From 19a1972e2a95c43ca42588725666dc77befd4a86 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:53:24 +0000 Subject: [PATCH 01/67] remove obsolete tests and data --- .../shadowserver/scan_rdpeudp.csv.license | 2 - .../parsers/shadowserver/test_blocklist.py | 103 ------- .../shadowserver/test_compromised_website.py | 88 ------ .../parsers/shadowserver/test_device_id.py | 116 -------- .../test_event4_ddos_participant.py | 131 --------- .../test_event4_honeypot_darknet.py | 106 ------- .../shadowserver/test_event4_honeypot_ddos.py | 148 ---------- .../test_event4_honeypot_ddos_target.py | 150 ---------- .../test_event4_honeypot_http_scan.py | 109 -------- .../shadowserver/test_event4_ip_spoofer.py | 182 ------------ .../test_event4_microsoft_sinkhole.py | 135 --------- .../test_event4_microsoft_sinkhole_http.py | 202 -------------- .../shadowserver/test_event4_sinkhole.py | 73 ----- .../shadowserver/test_event4_sinkhole_dns.py | 127 --------- .../shadowserver/test_event4_sinkhole_http.py | 189 ------------- .../test_event4_sinkhole_http_referer.py | 213 --------------- .../shadowserver/test_event6_sinkhole_http.py | 146 ---------- .../shadowserver/test_honeypot_brute_force.py | 72 ----- .../shadowserver/test_honeypot_ddos_amp.py | 91 ------ .../parsers/shadowserver/test_malware_url.py | 107 -------- .../parsers/shadowserver/test_phish_url.py | 106 ------- .../test_population_http_proxy.py | 130 --------- .../parsers/shadowserver/test_sandbox_conn.py | 99 ------- .../parsers/shadowserver/test_sandbox_dns.py | 95 ------- .../parsers/shadowserver/test_sandbox_url.py | 104 ------- .../parsers/shadowserver/test_scan_adb.py | 98 ------- .../parsers/shadowserver/test_scan_afp.py | 106 ------- .../parsers/shadowserver/test_scan_amqp.py | 144 ---------- .../parsers/shadowserver/test_scan_ard.py | 111 -------- .../parsers/shadowserver/test_scan_chargen.py | 110 -------- .../test_scan_cisco_smart_install.py | 82 ------ .../parsers/shadowserver/test_scan_coap.py | 121 -------- .../parsers/shadowserver/test_scan_couchdb.py | 128 --------- .../parsers/shadowserver/test_scan_cwmp.py | 103 ------- .../parsers/shadowserver/test_scan_db2.py | 91 ------ .../shadowserver/test_scan_ddos_middlebox.py | 119 -------- .../parsers/shadowserver/test_scan_dns.py | 91 ------ .../parsers/shadowserver/test_scan_docker.py | 159 ----------- .../test_scan_dvr_dhcpdiscover.py | 178 ------------ .../shadowserver/test_scan_elasticsearch.py | 126 --------- .../shadowserver/test_scan_exchange.py | 149 ---------- .../parsers/shadowserver/test_scan_ftp.py | 120 -------- .../parsers/shadowserver/test_scan_hadoop.py | 94 ------- .../parsers/shadowserver/test_scan_http.py | 100 ------- .../shadowserver/test_scan_http_proxy.py | 118 -------- .../shadowserver/test_scan_http_vulnerable.py | 125 --------- .../parsers/shadowserver/test_scan_ics.py | 125 --------- .../parsers/shadowserver/test_scan_ipmi.py | 106 ------- .../parsers/shadowserver/test_scan_ipp.py | 79 ------ .../parsers/shadowserver/test_scan_isakmp.py | 105 ------- .../shadowserver/test_scan_kubernetes.py | 214 --------------- .../shadowserver/test_scan_ldap_tcp.py | 154 ----------- .../shadowserver/test_scan_ldap_udp.py | 162 ----------- .../parsers/shadowserver/test_scan_mdns.py | 127 --------- .../shadowserver/test_scan_memcached.py | 130 --------- .../parsers/shadowserver/test_scan_mongodb.py | 103 ------- .../parsers/shadowserver/test_scan_mqtt.py | 89 ------ .../shadowserver/test_scan_mqtt_anon.py | 173 ------------ .../parsers/shadowserver/test_scan_mssql.py | 123 --------- .../parsers/shadowserver/test_scan_mysql.py | 258 ------------------ .../parsers/shadowserver/test_scan_nat_pmp.py | 116 -------- .../parsers/shadowserver/test_scan_netbios.py | 121 -------- .../shadowserver/test_scan_netis_router.py | 107 -------- .../parsers/shadowserver/test_scan_ntp.py | 161 ----------- .../shadowserver/test_scan_ntpmonitor.py | 108 -------- .../shadowserver/test_scan_portmapper.py | 120 -------- .../shadowserver/test_scan_postgres.py | 199 -------------- .../parsers/shadowserver/test_scan_qotd.py | 119 -------- .../parsers/shadowserver/test_scan_quic.py | 118 -------- .../parsers/shadowserver/test_scan_radmin.py | 236 ---------------- .../parsers/shadowserver/test_scan_rdp.py | 117 -------- .../parsers/shadowserver/test_scan_rdpeudp.py | 109 -------- .../parsers/shadowserver/test_scan_redis.py | 107 -------- .../parsers/shadowserver/test_scan_rsync.py | 116 -------- .../parsers/shadowserver/test_scan_sip.py | 124 --------- .../parsers/shadowserver/test_scan_slp.py | 137 ---------- .../parsers/shadowserver/test_scan_smb.py | 124 --------- .../shadowserver/test_scan_smb_json.py | 123 --------- .../shadowserver/test_scan_smtp_vulnerable.py | 92 ------- .../parsers/shadowserver/test_scan_snmp.py | 120 -------- .../parsers/shadowserver/test_scan_socks.py | 107 -------- .../parsers/shadowserver/test_scan_ssdp.py | 136 --------- .../parsers/shadowserver/test_scan_ssh.py | 182 ------------ .../parsers/shadowserver/test_scan_ssl.py | 218 --------------- .../shadowserver/test_scan_ssl_freak.py | 136 --------- .../shadowserver/test_scan_ssl_poodle.py | 91 ------ .../parsers/shadowserver/test_scan_stun.py | 146 ---------- .../shadowserver/test_scan_synfulknock.py | 117 -------- .../parsers/shadowserver/test_scan_telnet.py | 87 ------ .../parsers/shadowserver/test_scan_tftp.py | 121 -------- .../shadowserver/test_scan_ubiquiti.py | 124 --------- .../parsers/shadowserver/test_scan_vnc.py | 86 ------ .../shadowserver/test_scan_ws_discovery.py | 119 -------- .../parsers/shadowserver/test_scan_xdmcp.py | 117 -------- .../bots/parsers/shadowserver/test_special.py | 106 ------- .../parsers/shadowserver/test_testdata.py | 81 ------ .../shadowserver/testdata/blocklist.csv | 4 - .../testdata/blocklist.csv.license | 2 - .../testdata/botnet_drone.csv.license | 2 - .../testdata/caida_ip_spoofer.csv.license | 2 - .../testdata/compromised_website.csv | 4 - .../testdata/compromised_website.csv.license | 2 - .../shadowserver/testdata/darknet.csv.license | 2 - .../testdata/ddos_amplification.csv.license | 2 - .../shadowserver/testdata/device_id.csv | 4 - .../testdata/device_id.csv.license | 2 - .../testdata/drone_brute_force.csv.license | 2 - .../testdata/event4_ddos_participant.csv | 4 - .../event4_ddos_participant.csv.license | 2 - .../testdata/event4_honeypot_brute_force.csv | 7 - .../event4_honeypot_brute_force.csv.license | 2 - .../testdata/event4_honeypot_darknet.csv | 9 - .../event4_honeypot_darknet.csv.license | 2 - .../testdata/event4_honeypot_ddos.csv | 4 - .../testdata/event4_honeypot_ddos.csv.license | 2 - .../testdata/event4_honeypot_ddos_amp.csv | 6 - .../event4_honeypot_ddos_amp.csv.license | 2 - .../testdata/event4_honeypot_ddos_target.csv | 4 - .../event4_honeypot_ddos_target.csv.license | 2 - .../testdata/event4_honeypot_http_scan.csv | 3 - .../event4_honeypot_http_scan.csv.license | 2 - .../testdata/event4_ip_spoofer.csv | 7 - .../testdata/event4_ip_spoofer.csv.license | 2 - .../testdata/event4_microsoft_sinkhole.csv | 7 - .../event4_microsoft_sinkhole.csv.license | 2 - .../event4_microsoft_sinkhole_http.csv | 6 - ...event4_microsoft_sinkhole_http.csv.license | 2 - .../shadowserver/testdata/event4_sinkhole.csv | 4 - .../testdata/event4_sinkhole.csv.license | 2 - .../testdata/event4_sinkhole_dns.csv | 4 - .../testdata/event4_sinkhole_dns.csv.license | 2 - .../testdata/event4_sinkhole_http.csv | 6 - .../testdata/event4_sinkhole_http.csv.license | 2 - .../testdata/event4_sinkhole_http_referer.csv | 6 - .../event4_sinkhole_http_referer.csv.license | 2 - .../testdata/event6_sinkhole_http.csv | 4 - .../testdata/event6_sinkhole_http.csv.license | 2 - .../testdata/hp_http_scan.csv.license | 2 - .../testdata/hp_ics_scan.csv.license | 2 - .../shadowserver/testdata/malware_url.csv | 4 - .../testdata/malware_url.csv.license | 2 - .../testdata/outdated_dnssec_key.csv.license | 2 - .../shadowserver/testdata/phish_url.csv | 4 - .../testdata/phish_url.csv.license | 2 - .../testdata/population_http_proxy.csv | 4 - .../population_http_proxy.csv.license | 2 - .../shadowserver/testdata/sandbox_conn.csv | 4 - .../testdata/sandbox_conn.csv.license | 2 - .../shadowserver/testdata/sandbox_dns.csv | 4 - .../testdata/sandbox_dns.csv.license | 2 - .../shadowserver/testdata/sandbox_url.csv | 4 - .../testdata/sandbox_url.csv.license | 2 - .../shadowserver/testdata/scan_adb.csv | 3 - .../testdata/scan_adb.csv.license | 2 - .../shadowserver/testdata/scan_afp.csv | 3 - .../testdata/scan_afp.csv.license | 2 - .../shadowserver/testdata/scan_amqp.csv | 4 - .../testdata/scan_amqp.csv.license | 2 - .../shadowserver/testdata/scan_ard.csv | 4 - .../testdata/scan_ard.csv.license | 2 - .../shadowserver/testdata/scan_chargen.csv | 4 - .../testdata/scan_chargen.csv.license | 2 - .../testdata/scan_cisco_smart_install.csv | 3 - .../scan_cisco_smart_install.csv.license | 2 - .../shadowserver/testdata/scan_coap.csv | 4 - .../testdata/scan_coap.csv.license | 2 - .../shadowserver/testdata/scan_couchdb.csv | 4 - .../testdata/scan_couchdb.csv.license | 2 - .../shadowserver/testdata/scan_cwmp.csv | 3 - .../testdata/scan_cwmp.csv.license | 2 - .../shadowserver/testdata/scan_db2.csv | 3 - .../testdata/scan_db2.csv.license | 2 - .../testdata/scan_ddos_middlebox.csv | 4 - .../testdata/scan_ddos_middlebox.csv.license | 2 - .../shadowserver/testdata/scan_dns.csv | 101 ------- .../testdata/scan_dns.csv.license | 2 - .../shadowserver/testdata/scan_docker.csv | 4 - .../testdata/scan_docker.csv.license | 2 - .../testdata/scan_dvr_dhcpdiscover.csv | 4 - .../scan_dvr_dhcpdiscover.csv.license | 2 - .../testdata/scan_elasticsearch.csv | 4 - .../testdata/scan_elasticsearch.csv.license | 2 - .../shadowserver/testdata/scan_exchange.csv | 8 - .../testdata/scan_exchange.csv.license | 2 - .../shadowserver/testdata/scan_ftp.csv | 3 - .../testdata/scan_ftp.csv.license | 2 - .../shadowserver/testdata/scan_hadoop.csv | 3 - .../testdata/scan_hadoop.csv.license | 2 - .../shadowserver/testdata/scan_http.csv | 3 - .../testdata/scan_http.csv.license | 2 - .../shadowserver/testdata/scan_http_proxy.csv | 4 - .../testdata/scan_http_proxy.csv.license | 2 - .../testdata/scan_http_vulnerable.csv | 4 - .../testdata/scan_http_vulnerable.csv.license | 2 - .../shadowserver/testdata/scan_ics.csv | 4 - .../testdata/scan_ics.csv.license | 2 - .../shadowserver/testdata/scan_ipmi.csv | 96 ------- .../testdata/scan_ipmi.csv.license | 2 - .../shadowserver/testdata/scan_ipp.csv | 2 - .../testdata/scan_ipp.csv.license | 2 - .../shadowserver/testdata/scan_isakmp.csv | 3 - .../testdata/scan_isakmp.csv.license | 2 - .../shadowserver/testdata/scan_kubernetes.csv | 4 - .../testdata/scan_kubernetes.csv.license | 2 - .../shadowserver/testdata/scan_ldap_tcp.csv | 4 - .../testdata/scan_ldap_tcp.csv.license | 2 - .../shadowserver/testdata/scan_ldap_udp.csv | 4 - .../testdata/scan_ldap_udp.csv.license | 2 - .../shadowserver/testdata/scan_mdns.csv | 4 - .../testdata/scan_mdns.csv.license | 2 - .../shadowserver/testdata/scan_memcached.csv | 4 - .../testdata/scan_memcached.csv.license | 2 - .../shadowserver/testdata/scan_mongodb.csv | 11 - .../testdata/scan_mongodb.csv.license | 2 - .../shadowserver/testdata/scan_mqtt.csv | 2 - .../testdata/scan_mqtt.csv.license | 2 - .../shadowserver/testdata/scan_mqtt_anon.csv | 4 - .../testdata/scan_mqtt_anon.csv.license | 2 - .../shadowserver/testdata/scan_mssql.csv | 4 - .../testdata/scan_mssql.csv.license | 2 - .../shadowserver/testdata/scan_mysql.csv | 4 - .../testdata/scan_mysql.csv.license | 2 - .../shadowserver/testdata/scan_nat_pmp.csv | 4 - .../testdata/scan_nat_pmp.csv.license | 2 - .../shadowserver/testdata/scan_netbios.csv | 4 - .../testdata/scan_netbios.csv.license | 2 - .../testdata/scan_netis_router.csv | 4 - .../testdata/scan_netis_router.csv.license | 2 - .../shadowserver/testdata/scan_ntp.csv | 4 - .../testdata/scan_ntp.csv.license | 2 - .../shadowserver/testdata/scan_ntpmonitor.csv | 4 - .../testdata/scan_ntpmonitor.csv.license | 2 - .../shadowserver/testdata/scan_portmapper.csv | 4 - .../testdata/scan_portmapper.csv.license | 2 - .../shadowserver/testdata/scan_postgres.csv | 4 - .../testdata/scan_postgres.csv.license | 2 - .../shadowserver/testdata/scan_qotd.csv | 4 - .../testdata/scan_qotd.csv.license | 2 - .../shadowserver/testdata/scan_quic.csv | 4 - .../testdata/scan_quic.csv.license | 2 - .../shadowserver/testdata/scan_radmin.csv | 10 - .../testdata/scan_radmin.csv.license | 2 - .../shadowserver/testdata/scan_rdp.csv | 3 - .../testdata/scan_rdp.csv.license | 2 - .../shadowserver/testdata/scan_rdpeudp.csv | 4 - .../testdata/scan_rdpeudp.csv.license | 2 - .../shadowserver/testdata/scan_redis.csv | 94 ------- .../testdata/scan_redis.csv.license | 2 - .../shadowserver/testdata/scan_rsync.csv | 4 - .../testdata/scan_rsync.csv.license | 2 - .../shadowserver/testdata/scan_sip.csv | 4 - .../testdata/scan_sip.csv.license | 2 - .../shadowserver/testdata/scan_slp.csv | 4 - .../testdata/scan_slp.csv.license | 2 - .../shadowserver/testdata/scan_smb.csv | 4 - .../testdata/scan_smb.csv.license | 2 - .../testdata/scan_smtp_vulnerable.csv | 3 - .../testdata/scan_smtp_vulnerable.csv.license | 2 - .../shadowserver/testdata/scan_snmp.csv | 4 - .../testdata/scan_snmp.csv.license | 2 - .../shadowserver/testdata/scan_socks.csv | 4 - .../testdata/scan_socks.csv.license | 2 - .../shadowserver/testdata/scan_ssdp.csv | 4 - .../testdata/scan_ssdp.csv.license | 2 - .../shadowserver/testdata/scan_ssh.csv | 4 - .../testdata/scan_ssh.csv.license | 2 - .../shadowserver/testdata/scan_ssl.csv | 4 - .../testdata/scan_ssl.csv.license | 2 - .../shadowserver/testdata/scan_ssl_freak.csv | 46 ---- .../testdata/scan_ssl_freak.csv.license | 2 - .../shadowserver/testdata/scan_ssl_poodle.csv | 32 --- .../testdata/scan_ssl_poodle.csv.license | 2 - .../shadowserver/testdata/scan_stun.csv | 4 - .../testdata/scan_stun.csv.license | 2 - .../testdata/scan_synfulknock.csv | 4 - .../testdata/scan_synfulknock.csv.license | 2 - .../shadowserver/testdata/scan_telnet.csv | 3 - .../testdata/scan_telnet.csv.license | 2 - .../shadowserver/testdata/scan_tftp.csv | 4 - .../testdata/scan_tftp.csv.license | 2 - .../shadowserver/testdata/scan_ubiquiti.csv | 4 - .../testdata/scan_ubiquiti.csv.license | 2 - .../shadowserver/testdata/scan_vnc.csv | 3 - .../testdata/scan_vnc.csv.license | 2 - .../testdata/scan_ws_discovery.csv | 4 - .../testdata/scan_ws_discovery.csv.license | 2 - .../shadowserver/testdata/scan_xdmcp.csv | 4 - .../testdata/scan_xdmcp.csv.license | 2 - .../testdata/sinkhole_http_drone.csv.license | 2 - .../parsers/shadowserver/testdata/special.csv | 4 - .../shadowserver/testdata/special.csv.license | 2 - 291 files changed, 12939 deletions(-) delete mode 100644 intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_blocklist.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_device_id.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_malware_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_phish_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_special.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_testdata.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/special.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license diff --git a/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license b/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license deleted file mode 100644 index 043ed079f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py b/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py deleted file mode 100644 index 48509eea0..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/blocklist.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - 'feed.name': 'Block Listed IP Addresses', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-blocklist-test-geo.csv", -} -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.134", - "source.reverse_dns": "host.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.171", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.network": "198.123.245.0/24", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -},] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py b/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py deleted file mode 100644 index 53c5b247b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py +++ /dev/null @@ -1,88 +0,0 @@ -# SPDX-FileCopyrightText: 2017 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/compromised_website.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Compromised Website", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-compromised_website-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'ShadowServer Compromised Website', - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'classification.identifier': 'compromised-website', - 'extra.server': 'Microsoft-IIS/7.5', - 'extra.system': 'WINNT', - 'extra.detected_since': '2015-05-09 05:51:12', - 'protocol.application': 'http', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 64496, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '203.0.113.1', - 'source.port': 80, - 'source.url': 'http://example.com/header.php', - 'source.fqdn': 'example.com', - 'source.reverse_dns': 'example.com', - 'malware.name': 'hacked-webserver-stealrat-t1', - 'event_description.text': 'spam', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-01-16T00:43:48+00:00'}, - {'__type': 'Event', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'classification.identifier': 'compromised-website', - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'event_description.text': 'phishing', - 'feed.name': 'ShadowServer Compromised Website', - 'malware.name': 'phishing', - 'protocol.application': 'http', - 'source.asn': 64496, - 'source.fqdn': 'example.com', - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'GRAZ', - 'source.geolocation.region': 'STEIERMARK', - 'source.ip': '203.0.113.1', - 'source.port': 80, - 'source.url': 'http://example.com/', - 'time.source': '2018-04-09T15:43:41+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_device_id.py b/intelmq/tests/bots/parsers/shadowserver/test_device_id.py deleted file mode 100644 index e8954e03c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_device_id.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/device_id.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Device ID', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-device_id-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'extra.naics' : 517311, - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 2116, - 'source.geolocation.cc' : 'NO', - 'source.geolocation.city' : 'TROMVIK', - 'source.geolocation.region' : 'TROMS OG FINNMARK', - 'source.ip' : '88.84.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 27843, - 'source.geolocation.cc' : 'PE', - 'source.geolocation.city' : 'LIMA', - 'source.geolocation.region' : 'METROPOLITANA DE LIMA', - 'source.ip' : '170.231.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'extra.naics' : 517311, - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 4181, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MILWAUKEE', - 'source.geolocation.region' : 'WISCONSIN', - 'source.ip' : '96.60.0.0', - 'source.port' : 10443, - 'source.reverse_dns' : '96-60-66-218.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py deleted file mode 100644 index badc53a73..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py +++ /dev/null @@ -1,131 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_ddos_participant.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DDoS Participant', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_ddos_participant-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.1', - 'destination.port': 443, - 'destination.reverse_dns': 'node01.example.net', - 'extra.application': 'https', - 'extra.domain': 'www.example.com', - 'extra.http_method': 'GET', - 'extra.http_path': '/??=GovpfOoaWYlk', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 38055, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.2', - 'destination.port': 53, - 'destination.reverse_dns': 'node02.example.net', - 'extra.application': 'dns', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 53, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.3', - 'destination.port': 53, - 'destination.reverse_dns': 'node03.example.net', - 'extra.application': 'dns', - 'extra.device_model': 'Exchange', - 'extra.device_type': 'email', - 'extra.device_vendor': 'Microsoft', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 53, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py deleted file mode 100644 index 1d020f473..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_darknet.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Darknet", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_darknet.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.port': 23, - 'extra.source.naics': 518210, - 'extra.tag': 'mirai', - 'protocol.transport': 'tcp', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 9829, - 'source.geolocation.cc': 'IN', - 'source.geolocation.city': 'CHENGANNUR', - 'source.geolocation.region': 'KERALA', - 'source.ip': '61.3.1.2', - 'source.port': 4717, - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - {'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.port': 23, - 'protocol.transport': 'tcp', - 'extra.source.naics': 517311, - 'extra.tag': 'mirai', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 4766, - 'source.geolocation.cc': 'KR', - 'source.geolocation.city': 'PYEONGCHANG-EUP', - 'source.geolocation.region': 'GANGWON-DO', - 'source.ip': '211.218.3.4', - 'source.port': 4405, - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - }, - {'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.tag': 'mirai', - 'destination.port': 23, - 'protocol.transport': 'tcp', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 266915, - 'source.geolocation.cc': 'BR', - 'source.geolocation.city': 'VITORIA DA CONQUISTA', - 'source.geolocation.region': 'BAHIA', - 'source.ip': '45.225.5.6', - 'source.port': 59777, - 'source.reverse_dns': 'static-45-225-x-x.example.net', - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py deleted file mode 100644 index c62a610fa..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py +++ /dev/null @@ -1,148 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_ddos.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot DDoS Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_honeypot_ddos-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.1', - 'destination.port' : 88, - 'destination.reverse_dns' : 'node01.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '121.12.110.28/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 61234, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.2', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node02.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '180.97.183.94/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 61234, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.3', - 'destination.reverse_dns' : 'node03.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk7', - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '104.237.138.135/32', - 'extra.duration' : 10, - 'extra.family' : 'mirai', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 6379, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py deleted file mode 100644 index f379d1c88..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py +++ /dev/null @@ -1,150 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_ddos_target.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot DDoS Target Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_honeypot_ddos_target-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.1', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node01.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk0', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '115.238.198.85/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 61234, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.2', - 'destination.port' : 43437, - 'destination.reverse_dns' : 'node02.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk0', - 'extra.destination.sector' : 'Information', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '52.184.50.250/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 61234, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.3', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node03.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '211.99.102.216/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 61234, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py deleted file mode 100644 index bcf268ba7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_http_scan.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot-HTTP-Scan', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-08-01T12:00:00+00:00", - "extra.file_name": "2021-08-01-event4_honeypot_http_scan.csv", - } - -EVENTS = [{'__type': 'Event', - 'feed.name': 'Honeypot-HTTP-Scan', - 'classification.identifier': 'honeypot-http-scan', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'destination.asn': 5678, - 'destination.geolocation.cc': 'UK', - 'destination.geolocation.city': 'MAIDENHEAD', - 'destination.geolocation.region': 'WINDSOR AND MAIDENHEAD', - 'destination.ip': '109.87.65.43', - 'destination.port': 80, - 'extra.http_url': '/js/ueditor/wwwroot/way-board.cgi', - 'extra.destination.naics': 518210, - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'extra.public_source': 'CAPRICA-EU', - 'extra.request_raw': 'R0VUIC9qcy91ZWRpdG9yL3d3d3Jvb3Qvd2F5LWJvYXJkLmNnaSBIVFRQLzEuMHJuQWNjZXB0OiB0ZXh0L2h0bWwsYXBwbGljYXRpb24veGh0bWwreG1sLGFwcGxpY2F0aW9uL3htbDtxPTAuOSwqLyo7cT0wLjhybkFjY2VwdC1FbmNvZGluZzogZ3ppcCwgZGVmbGF0ZXJuQWNjZXB0LUxhbmd1YWdlOiBlbi1VUyxlbjtxPTAuNXJuQ29ubmVjdGlvbjogY2xvc2VybkRudDogMXJuSG9zdDogMTA5Ljg3LjY1LjQzcm5PcmlnaW46IGh0dHA6Ly8xMDkuODcuNjUuNDNyblJlZmVyZXI6IGh0dHA6Ly8xMDkuODcuNjUuNDMvcm5Vc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA2LjE7IFdPVzY0KSBBcHBsZVdlYktpdC81MzcuMzYgKEtIVE1MLCBsaWtlIEdlY2tvKSBDaHJvbWUvNTMuMC4yNzg1LjEwNCBTYWZhcmkvNTM3LjM2IENvcmUvMS41My4zMDg0LjQwMCBRUUJyb3dzZXIvOS42LjExMzQ2LjQwMA==', - 'extra.source.naics': 518210, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.version': '3.1.3-dev', - 'malware.name': 'http-scan', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 1234, - 'source.geolocation.cc': 'EE', - 'source.geolocation.city': 'TALLINN', - 'source.geolocation.region': 'HARJUMAA', - 'source.ip': '191.23.45.67', - 'source.port': 36455, - 'source.reverse_dns': '191-23-45-67-host.example.com', - 'time.observation': '2021-08-01T12:00:00+00:00', - 'time.source': '2021-08-01T00:24:08+00:00'}, - {'__type': 'Event', - 'feed.name': 'Honeypot-HTTP-Scan', - 'classification.identifier': 'honeypot-http-scan', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'destination.asn': 23456, - 'destination.geolocation.cc': 'UA', - 'destination.geolocation.city': 'KHARKIV', - 'destination.geolocation.region': "KHARKIVS'KA OBLAST'", - 'destination.ip': '82.41.20.10', - 'destination.port': 8080, - 'extra.http_url': '/', - 'extra.method': 'GET', - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'extra.public_source': 'CAPRICA-EU', - 'extra.request_raw': 'R0VUIC8gSFRUUC8xLjENCkhvc3Q6IDgyLjQxLjIwLjEwOjgwODANCkFjY2VwdDogdGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksKi8qO3E9MC44DQpBY2NlcHQtRW5jb2Rpbmc6IGRlZmxhdGUsIGd6aXAsIGlkZW50aXR5DQpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTO3E9MC42LGVuO3E9MC40DQpVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA1LjE7IHJ2OjkuMC4xKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzkuMC4xDQoNCg==', - 'extra.url_scheme': 'http', - 'extra.user_agent': 'Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1', - 'malware.name': 'http-scan', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 12345, - 'source.geolocation.cc': 'EE', - 'source.geolocation.city': 'TALLINN', - 'source.geolocation.region': 'HARJUMAA', - 'source.ip': '45.67.89.123', - 'source.port': 58610, - 'time.observation': '2021-08-01T12:00:00+00:00', - 'time.source': '2021-08-01T05:21:59+00:00'}, -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py deleted file mode 100644 index d21fb10c5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py +++ /dev/null @@ -1,182 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/event4_ip_spoofer.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "CAIDA", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-event4_ip_spoofer.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T00:42:59+00:00", - "source.ip": "98.191.250.0", - - "source.asn": 22898, - - "source.geolocation.cc": "US", - "source.geolocation.region": "OKLAHOMA", - "source.geolocation.city": "OKLAHOMA CITY", - "source.network": "98.191.250.0/24", - "source.reverse_dns": 'ip-98.191.250.0.atlinkservices.com', - "extra.routedspoof": "received", - "extra.session": '1112907', - "extra.nat": True, - "extra.public_source": "caida", - "extra.source.naics": 517311, - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T01:36:22+00:00", - "source.ip": "191.7.16.0", - - "source.asn": 262485, - - "source.geolocation.cc": "BR", - "source.geolocation.region": "RIO DE JANEIRO", - "source.geolocation.city": "NOVA IGUACU", - "source.network": "191.7.16.0/24", - "extra.routedspoof": "received", - "extra.session": '1112914', - "extra.nat": False, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T02:10:58+00:00", - "source.ip": "202.53.160.0", - - "source.asn": 23923, - - "source.geolocation.cc": "BD", - "source.geolocation.region": "DHAKA", - "source.geolocation.city": "DHAKA", - "source.network": "202.53.160.0/24", - "extra.routedspoof": "received", - "extra.session": '1112931', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T03:41:51+00:00", - "source.ip": "87.121.75.0", - - "source.asn": 134697, - - "source.geolocation.cc": "AU", - "source.geolocation.region": "QUEENSLAND", - "source.geolocation.city": "BRISBANE", - "source.network": "87.121.75.0/24", - "extra.routedspoof": "received", - "extra.session": '1112953', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T06:07:17+00:00", - "source.ip": "189.201.194.0", - - "source.asn": 262944, - - "source.network": "189.201.194.0/24", - "source.geolocation.cc": 'MX', - "source.geolocation.city": 'SALTILLO', - "source.geolocation.region": 'COAHUILA', - "source.reverse_dns": 'ip-189-201-194-0.slw.spectro.mx', - "extra.routedspoof": "received", - "extra.session": '1113015', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py deleted file mode 100644 index f008fd18e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py +++ /dev/null @@ -1,135 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_microsoft_sinkhole.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Microsoft Sinkhole", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_microsoft_sinkhole.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'HK', - 'destination.geolocation.city': 'HONG KONG', - 'destination.geolocation.region': 'HONG KONG', - 'destination.ip': '168.63.134.179', - 'destination.port': 16464, - 'extra.destination.naics': 334111, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 7303, - 'source.geolocation.cc': 'AR', - 'source.geolocation.city': 'CASEROS', - 'source.geolocation.region': 'BUENOS AIRES', - 'source.ip': '190.229.1.2', - 'source.port': 52955, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - {'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'IE', - 'destination.geolocation.city': 'DUBLIN', - 'destination.geolocation.region': 'DUBLIN', - 'destination.ip': '52.169.3.4', - 'destination.port': 16464, - 'extra.destination.naics': 334111, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 5769, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'LAVAL', - 'source.geolocation.region': 'QUEBEC', - 'source.ip': '96.20.3.4', - 'source.port': 16464, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - }, - {'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'HK', - 'destination.geolocation.city': 'HONG KONG', - 'destination.geolocation.region': 'HONG KONG', - 'destination.ip': '168.63.134.179', - 'destination.port': 16464, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 8151, - 'source.geolocation.cc': 'MX', - 'source.geolocation.city': 'MEXICO CITY', - 'source.geolocation.region': "CIUDAD DE MEXICO", - 'source.ip': '187.222.5.6', - 'source.port': 55049, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py deleted file mode 100644 index 2f8c3d8e2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py +++ /dev/null @@ -1,202 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_microsoft_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'HTTP Microsoft Sinkhole IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_microsoft_sinkhole_http.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.infection': 'necurs', - 'extra.tag': 'necurs', - 'protocol.application': 'http', - 'malware.name': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 8386, - 'source.geolocation.cc': 'TR', - 'source.geolocation.city': 'KEPEZ', - 'source.geolocation.region': 'ANTALYA', - 'source.ip': '31.206.1.2', - 'source.port': 49245, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'caphaw', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.fqdn': '3fo8jrthz3y.rgk.cc', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'REDMOND', - 'destination.geolocation.region': 'WASHINGTON', - 'destination.ip': '204.95.99.204', - 'destination.port': 443, - 'destination.url': 'http://3fo8jrthz3y.rgk.cc/index.php', - 'protocol.application': 'http', - 'extra.infection': 'caphaw', - 'extra.tag': 'caphaw', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.http_agent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.6103)', - 'extra.http_referer': 'null', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517312, - 'malware.name': 'caphaw', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 28573, - 'source.geolocation.cc': 'BR', - 'source.geolocation.city': 'SAO PAULO', - 'source.geolocation.region': 'SAO PAULO', - 'source.ip': '177.140.3.4', - 'source.port': 35919, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'protocol.application': 'http', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'malware.name': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn': 132199, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'MANDAUE', - 'source.geolocation.region': 'CEBU', - 'source.ip': '180.190.5.6', - 'source.port': 49264, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.ip': '40.121.206.97', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/news/stream.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'malware.name': 'necurs', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'source.asn': 37129, - 'source.geolocation.cc': 'KE', - 'source.geolocation.city': 'NAIROBI', - 'source.geolocation.region': 'NAIROBI CITY', - 'source.ip': '197.157.7.8', - 'source.port': 55307, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'necurs', - 'protocol.application': 'http', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'source.asn': 812, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'OTTAWA', - 'source.geolocation.region': 'ONTARIO', - 'source.ip': '174.114.9.10', - 'source.port': 59000, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py deleted file mode 100644 index 2bb8aa698..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Sinkhole", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'victorygate.b', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 28753, - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.ip': '178.162.1.2', - 'destination.port': 4455, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.public_source': 'eset', - 'feed.name': 'ShadowServer Sinkhole', - 'malware.name': 'victorygate.b', - 'extra.infection': 'victorygate.b', - 'protocol.transport': 'tcp', - 'source.asn': 12252, - 'source.geolocation.cc': 'PE', - 'source.geolocation.city': 'LIMA', - 'source.geolocation.region': 'METROPOLITANA DE LIMA', - 'source.ip': '190.113.1.2', - 'source.port': 17409, - 'time.source': '2021-03-04T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py deleted file mode 100644 index cf3bdb162..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_sinkhole_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "Sinkhole DNS", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole_dns-test-geo.csv", - } - -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'msexchange', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'calypso', - 'extra.dns_query' : 'YolkIsh.COM', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'extra.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'calypso', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 8220, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'FRANKFURT AM MAIN', - 'source.geolocation.region' : 'HESSEN', - 'source.ip' : '217.110.0.0', - 'source.port' : 29614, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'rat', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'orcus', - 'extra.dns_query' : 'verble.rocks', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'orcus', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 40934, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '209.66.0.0', - 'source.port' : 46189, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'msexchange', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'calypso', - 'extra.dns_query' : 'RAwFuNS.COM', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'extra.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'calypso', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 8220, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'FRANKFURT AM MAIN', - 'source.geolocation.region' : 'HESSEN', - 'source.ip' : '217.110.0.0', - 'source.port' : 3590, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -} -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py deleted file mode 100644 index 60cd6b6ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py +++ /dev/null @@ -1,189 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'HTTP Sinkhole IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole_http.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.1.2', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 134707, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'DEL PILAR', - 'source.geolocation.region': 'NUEVA ECIJA', - 'source.ip': '103.196.1.2', - 'source.port': 60902, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.3.4', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 8708, - 'source.geolocation.cc': 'RO', - 'source.geolocation.city': 'CONSTANTA', - 'source.geolocation.region': 'CONSTANTA', - 'source.ip': '5.14.3.4', - 'source.port': 55002, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'disorderstatus.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.5.6', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.source.naics': 517311, - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn': 9299, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'CEBU', - 'source.geolocation.region': 'CEBU', - 'source.ip': '49.145.5.6', - 'source.port': 31350, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.ip': '184.105.7.8', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.source.naics': 517311, - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'source.asn': 8048, - 'source.geolocation.cc': 'VE', - 'source.geolocation.city': 'VALENCIA', - 'source.geolocation.region': 'CARABOBO', - 'source.ip': '200.44.7.8', - 'source.port': 28063, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.9.10', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'source.asn': 17072, - 'source.geolocation.cc': 'MX', - 'source.geolocation.city': 'JUAREZ', - 'source.geolocation.region': 'CHIHUAHUA', - 'source.ip': '187.189.9.10', - 'source.port': 45335, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py deleted file mode 100644 index b1ccacd31..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py +++ /dev/null @@ -1,213 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole_http_referer.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-03-05T00:00:00+00:00", - "extra.file_name": "2021-03-04-event4_sinkhole_http_referer.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'kovter', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': '12106.mobapptrack.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.ip': '85.17.31.82', - 'destination.port': 80, - 'destination.url': 'http://12106.mobapptrack.com/favicon.ico', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.event_id': '1614816002', - 'malware.name': 'kovter', - 'extra.http_referer': 'http://12106.mobapptrack.com/click/redirect?feed_id=12106&sub_id=7&q=8A5491983C8FBE7743E2D2C36E45EBC4-18307118D2626C9BD756B3F09D14BB910E381EE4', - 'extra.http_referer_asn': 28753, - 'extra.http_referer_city': 'FRANKFURT AM MAIN', - 'extra.http_referer_geo': 'DE', - 'extra.http_referer_hostname': '12106.mobapptrack.com', - 'extra.http_referer_ip': '178.162.203.211', - 'extra.http_referer_naics': 518210, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'HESSEN', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:02+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 28753, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/animalally.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '178.162.1.2', - 'extra.event_id': '1614816011', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://x.noizm.com/jump.php?u=http://freescanonline.com/animalally.com', - 'extra.http_referer_asn': 9370, - 'extra.http_referer_city': 'OSAKA', - 'extra.http_referer_geo': 'JP', - 'extra.http_referer_hostname': 'x.noizm.com', - 'extra.http_referer_naics': 518210, - 'extra.http_referer_port': 80, - 'extra.http_referer_ip': '59.106.1.2', - 'extra.http_referer_region': 'OSAKA', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.source': '2021-03-04T00:00:11+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'kovter', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 28753, - 'destination.fqdn': 'rxrtb.bid', - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.port': 80, - 'destination.url': 'http://rxrtb.bid/getjs?r=0.6393021999392658', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '178.162.1.2', - 'extra.event_id': '1614816012', - 'malware.name': 'kovter', - 'extra.http_referer': 'http://x.blogspot.com/', - 'extra.http_referer_ip': '142.250.3.4', - 'extra.http_referer_asn': 15169, - 'extra.http_referer_city': 'MOUNTAIN VIEW', - 'extra.http_referer_geo': 'US', - 'extra.http_referer_hostname': 'x.blogspot.com', - 'extra.http_referer_naics': 519130, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'CALIFORNIA', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'time.source': '2021-03-04T00:00:12+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.ip': '5.79.71.225', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/personalationmall.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'extra.event_id': '1614816013', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://www.example.com/teams/default.asp?u=EKL&t=c&s=lacrosse&p=remote&url=http://freescanonline.com/personalationmall.com', - 'extra.http_referer_asn': 14618, - 'extra.http_referer_city': 'ASHBURN', - 'extra.http_referer_geo': 'US', - 'extra.http_referer_hostname': 'www.example.com', - 'extra.http_referer_ip': '34.232.5.6', - 'extra.http_referer_naics': 454110, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'VIRGINIA', - 'extra.http_referer_sector': 'Retail Trade', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'time.source': '2021-03-04T00:00:13+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/raftcomply.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '5.79.1.2', - 'extra.event_id': '1614816086', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://x.communes.jp/?url=http://freescanonline.com/raftcomply.com', - 'extra.http_referer_asn': 2516, - 'extra.http_referer_city': 'SAPPORO', - 'extra.http_referer_geo': 'JP', - 'extra.http_referer_hostname': 'x.communes.jp', - 'extra.http_referer_ip': '210.172.7.8', - 'extra.http_referer_naics': 517312, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'HOKKAIDO', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'time.source': '2021-03-04T00:01:26+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py deleted file mode 100644 index d6ff35dc1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event6_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "Sinkhole-Events-HTTP IPv6", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event6_sinkhole_http-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::fe', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)', - 'extra.infection' : 'boaxxe', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 7713, - 'source.geolocation.cc' : 'ID', - 'source.geolocation.city' : 'JAKARTA', - 'source.geolocation.region' : 'JAKARTA RAYA', - 'source.ip' : '2001:448a:1082:4d9b:7491:bf9e:3d5f:a634', - 'source.port' : 49431, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T09:14:19+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::ef', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)', - 'extra.infection' : 'boaxxe', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 7713, - 'source.geolocation.cc' : 'ID', - 'source.geolocation.city' : 'JAKARTA', - 'source.geolocation.region' : 'JAKARTA RAYA', - 'source.ip' : '2001:448a:1082:4d9b:7491:bf9e:3d5f:a634', - 'source.port' : 49460, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T09:15:10+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::fe', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/WMoUNCvuKGzdqSCeQcadP1%2f0B%2f3bzpOmyKBU85Z25HVOhvDQUPFl%2fk8uOcLewS%2b1BsuHXalRAOIgGOYYs2igj6UX8FkdCAmDewWPvfDhPD45nwd2tx1lLf2IoIfuOtIpGR6bN5Q6hGpSBgfERqCa0ImHcwfcZ2EdO%2fWvg7R8H6SLcTiuUC0I4pzvlWt1CRLgLdIEU1hZ0nnFHIHhchb6D7ITEgBQ2chQDxy5TJMrGjm4Dac6dKl%2ft5uYhRhSjAHkLLtgrJjsqVtVbelTAkt5kdcqLlO09m1SH%2fvtAb%2fOvR2DbhBss7%2f64DG7g6cAnghNA6JrFn1uW7sw%2bnKH8koKQwzUjdSsbrQAvmg4r0KDDW8Diq64gfDzxFWkzCLOYifc%2fwlinXPCl7aJiNCoieDC1U98RNQg%2f5td4SZmJnDQ2%2f96CPbFeSpCez5WD1rCjrxLj1h2cqzIgkydEWACceWP9ztxc4QaObzEcgOGxbRckWC7H2aaLeT8jaYEYdKi1pwEKChSL3YdEt4ZIb2IFrWwzNaXEpQzFXf07f902OEdI9vVA1ZdEOBPG6rAIkzMdebfprfVyhKEWtrCd3Skg3COUFtRQks5jzG1nv4sVGijTfSgyn6xE9Taka668Nycik6nmHy8Huj3oC01j3tee%2f1Z3eI6tV7lgM5d3uFJ84slRGHUCwMfVozOGmZRwNo%2fz%2bA', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko', - 'extra.infection' : 'boaxxe', - 'extra.source.naics' : 517311, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 11427, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'GARLAND', - 'source.geolocation.region' : 'TEXAS', - 'source.ip' : '2603:8080:b20a:dc00:f06e:8304:71f6:27e2', - 'source.port' : 62932, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T14:15:10+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py b/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py deleted file mode 100644 index c376a73fb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_honeypot_brute_force.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot-Brute-Force-Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_brute_force.csv" - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'ssh', - 'classification.taxonomy': 'intrusion-attempts', - 'classification.type': 'brute-force', - 'extra.client_version': "b'SSH-2.0-Go'", - 'destination.asn': 26832, - 'destination.geolocation.cc': 'CA', - 'destination.geolocation.city': 'MONTREAL', - 'destination.geolocation.region': 'QUEBEC', - 'destination.ip': '162.250.1.2', - 'destination.port': 22, - 'extra.application': 'ssh', - 'extra.end_time': '2021-03-27T00:00:01.710968+00:00', - 'extra.public_source': 'CAPRICA-EU', - 'extra.start_time': '2021-03-27T00:00:00.521730+00:00', - 'malware.name': 'ssh-brute-force', - 'feed.name': 'Honeypot-Brute-Force-Events', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 209588, - 'source.geolocation.cc': 'NL', - 'source.geolocation.city': 'AMSTERDAM', - 'source.geolocation.region': 'NOORD-HOLLAND', - 'source.ip': '141.98.1.2', - 'source.port': 30123, - 'time.source': '2021-03-27T00:00:00+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py b/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py deleted file mode 100644 index e95e59dcb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_honeypot_ddos_amp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Amplification DDoS Victim', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_ddos_amp.csv" - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Amplification DDoS Victim', - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation': '2019-01-01T00:00:00+00:00', - 'time.source': '2021-03-28T00:00:02+00:00', - 'source.ip': '107.141.1.2', - 'destination.port': 389, - 'source.reverse_dns': '192-0-2-10.example.net', - 'source.asn': 7018, - 'source.geolocation.cc': 'US', - 'source.geolocation.region': 'VISALIA', - 'source.geolocation.city': 'VISALIA', - 'source.geolocation.region': 'CALIFORNIA', - 'extra.end_time': '2021-03-28T00:20:22+00:00', - 'extra.public_source': 'CISPA', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'ddos-amplification', - 'source.reverse_dns': '107-141-x-x.lightspeed.frsnca.sbcglobal.net', - }, - {'__type': 'Event', - 'feed.name': 'Amplification DDoS Victim', - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.observation': '2019-01-01T00:00:00+00:00', - 'time.source': '2021-03-28T00:00:02+00:00', - 'source.ip': '74.59.3.4', - 'destination.port': 389, - 'source.reverse_dns': 'modemcablex-x-59-74.mc.videotron.ca', - 'source.asn': 5769, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'CHICOUTIMI', - 'source.geolocation.region': 'QUEBEC', - 'extra.end_time': '2021-03-28T00:13:50+00:00', - 'extra.public_source': 'CISPA', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'ddos-amplification', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py b/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py deleted file mode 100644 index b19b200b5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/malware_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Malware URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-malware_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'source.url' : 'http://41.86.0.0:50008/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.hash.sha256' : '12013662c71da69de977c04cd7021f13a70cf7bed4ca6c82acbc100464d4b0ef', - 'malware.name' : 'cve-2016-10372', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37203, - 'source.geolocation.cc' : 'LR', - 'source.geolocation.city' : 'MONROVIA', - 'source.geolocation.region' : 'MONTSERRADO', - 'source.ip' : '41.86.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:02:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'extra.source.naics' : 517311, - 'source.url' : 'http://42.225.0.0:38173/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.name' : 'cve-2018-10562', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 4837, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'ZHUMADIAN', - 'source.geolocation.region' : 'HENAN SHENG', - 'source.ip' : '42.225.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:03:14+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'extra.source.naics' : 517311, - 'source.url' : 'http://211.52.0.0:53029/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.name' : 'cve-2018-10562', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 4766, - 'source.geolocation.cc' : 'KR', - 'source.geolocation.city' : 'SAGOK-MYEON', - 'source.geolocation.region' : 'CHUNGCHEONGNAM-DO', - 'source.ip' : '211.52.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:10:26+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py b/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py deleted file mode 100644 index 0783372f9..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/phish_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Phish URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-phish_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'priceless-pare.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.naics' : 518210, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'source.url' : 'https://priceless-pare.example.net/Postal-/acec6/', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'BUFFALO', - 'source.geolocation.region' : 'NEW YORK', - 'source.ip' : '172.245.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'mailyahooattt.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.sector' : 'Professional, Scientific, and Technical Services', - 'source.url' : 'https://mailyahooattt.example.net/', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'SAN FRANCISCO', - 'source.geolocation.region' : 'CALIFORNIA', - 'source.ip' : '199.34.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'www.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.naics' : 519130, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'source.url' : 'https://www.example.net/viewer/vbid-730ec2b1-omsttuer', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'DRAPER', - 'source.geolocation.region' : 'UTAH', - 'source.ip' : '216.58.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py b/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py deleted file mode 100644 index e9f11a47c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py +++ /dev/null @@ -1,130 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/population_http_proxy.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible HTTP Proxy', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-population_http_proxy-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 3741, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"Squid proxy-caching web ' - 'server\\"\\""', - 'extra.server': 'squid/4.10', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3128, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 3833, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"00:23:24:43:1c:34\\"\\""', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3128, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 179, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"Proxy\\"\\""', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3128, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py deleted file mode 100644 index c5da82346..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py +++ /dev/null @@ -1,99 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_conn.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox Connections', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_conn-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'time.windows.com', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'b575ce6dcce6502a8431db5610135c25', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 8075, - 'source.geolocation.cc' : 'US', - 'source.ip' : '40.119.6.228', - 'source.port' : 123, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -}, - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'c0d947f9a8685b0d9f3efdba966389c2', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 3356, - 'source.geolocation.cc' : 'US', - 'source.ip' : '8.252.70.126', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'c0d947f9a8685b0d9f3efdba966389c2', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 8075, - 'source.geolocation.cc' : 'US', - 'source.ip' : '52.109.8.22', - 'source.port' : 443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py deleted file mode 100644 index 70cf1eee5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py +++ /dev/null @@ -1,95 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox DNS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_dns-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'time.windows.com', - 'extra.response' : '40.119.6.228', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : 'b575ce6dcce6502a8431db5610135c25', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:02+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'time.windows.com', - 'extra.response' : '40.119.6.228', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : '807679198a39c80d3ca07e60fd51b581', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:08+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'client-office365-tas.msedge.net', - 'extra.response' : '13.107.5.88', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : 'd97e973b9bf073bd3a217425259cea26', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:20+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py deleted file mode 100644 index 91b0154b8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py +++ /dev/null @@ -1,104 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'www.msftncsi.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://www.msftncsi.com/ncsi.txt', - 'extra.user_agent' : 'Microsoft NCSI', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : '37514b54e679a5313334e830ad780ec7', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'US', - 'source.ip' : '23.196.47.89', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'www.download.windowsupdate.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://www.download.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab', - 'extra.user_agent' : 'Microsoft-CryptoAPI/6.1', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : '37514b54e679a5313334e830ad780ec7', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 15133, - 'source.geolocation.cc' : 'US', - 'source.ip' : '72.21.81.240', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:28+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'crl.microsoft.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://crl.microsoft.com/pki/crl/products/MicTimStaPCA_2010-07-01.crl', - 'extra.user_agent' : 'Microsoft-CryptoAPI/6.1', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : 'e97ea2820c0d79f3f3ca241d4dcd1060', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'US', - 'source.ip' : '23.56.4.57', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:08:24+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py deleted file mode 100644 index 6bc6e6146..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py +++ /dev/null @@ -1,98 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_adb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible ADB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_adb-test-test.csv", - - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible ADB', - 'time.observation': '2018-07-30T00:00:00+00:00', - 'time.source': '2018-07-26T02:07:16+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-adb', - 'protocol.application': 'adb', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 3462, - 'source.geolocation.cc': 'TW', - 'source.geolocation.city': 'TAOYUAN CITY', - 'source.geolocation.region': 'TAOYUAN COUNTY', - 'source.ip': '36.239.124.210', - 'source.port': 5555, - 'extra.name': 'hlteuc', - 'extra.model': 'SAMSUNG-SM-N900A', - 'extra.device': 'hlteatt', - 'extra.tag': 'adb', - 'extra.naics': 518210, - 'extra.sic': 737415, - 'source.reverse_dns': '36-239-124-210.dynamic-ip.hinet.net', - }, - {'__type': 'Event', - 'feed.name': 'Accessible ADB', - 'time.observation': '2018-07-30T00:00:00+00:00', - 'time.source': '2018-07-26T02:07:16+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-adb', - 'protocol.application': 'adb', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 3462, - 'source.geolocation.cc': 'TW', - 'source.geolocation.city': 'TAIPEI', - 'source.geolocation.region': 'TAIPEI CITY', - 'source.ip': '36.236.108.107', - 'source.port': 5555, - 'extra.name': 'marlin', - 'extra.model': 'Pixel XL', - 'extra.device': 'marlin', - 'extra.features': 'cmd,shell_v2', - 'extra.naics': 518210, - 'extra.sic': 737415, - 'extra.tag': 'adb', - 'source.reverse_dns': '36-236-108-107.dynamic-ip.hinet.net', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py deleted file mode 100644 index cc30b1e4c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_afp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible AFP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_afp-test-test.csv", - - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible AFP', - "classification.identifier": "accessible-afp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.afp_versions": "AFP3.3,AFP3.2,AFP3.1", - "extra.flags": "SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient", - "extra.machine_type": "TimeCapsule8,119", - "extra.naics": 517311, - "extra.network_address": "198.33.24.165:548,10.0.1.1:548,fe80:0008:0000:0000:6e70:9fff:fed4::548,fe80:0009:0000:0000:6e70:9fff:fed4::548,179.24.24.165 (DNS address),", - "extra.server_name": "airport-time-capsule-de-jack", - "extra.signature": "4338364e37364442463948350069672d", - "extra.tag": "afp", - "extra.uams": "DHCAST128,DHX2,SRP,Recon1", - "extra.utf8_servername": "AirPort Time Capsule de jack", - "protocol.application": "afp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 6057, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.13.34.22", - "source.port": 548, - "source.reverse_dns": "host.local", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T05:05:53+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Accessible AFP', - "classification.identifier": "accessible-afp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.afp_versions": "AFP3.3,AFP3.2,AFP3.1", - "extra.flags": "SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient", - "extra.machine_type": "TimeCapsule8,119", - "extra.naics": 517311, - "extra.network_address": "0.0.0.1:548,10.0.1.1:548,198.33.42.1:548,fe80:000b:0000:0000:dea4:caff:feba::548,fe80:000c:0000:0000:dea4:caff:feba::548,fe80:000d:0000:0000:4c7d:ffff:fec7::548,0.0.0.1 (DNS address),", - "extra.server_name": "time-capsule-del-jack", - "extra.signature": "433836544b303147463948360069672d", - "extra.tag": "afp", - "extra.uams": "DHCAST128,DHX2,SRP,Recon1", - "extra.utf8_servername": "Time Capsule del Jack", - "protocol.application": "afp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 6057, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.40.27.212", - "source.port": 548, - "source.reverse_dns": "host.local", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T05:05:56+00:00" - },] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py deleted file mode 100644 index df707f30b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py +++ /dev/null @@ -1,144 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_amqp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible AMQP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_amqp-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@iZuf63m0nnq9bwf7lhjxrkZ', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'PLAIN AMQPLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.3.5', - 'extra.naics' : 518210, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37963, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'SHANGHAI', - 'source.geolocation.region' : 'SHANGHAI SHI', - 'source.ip' : '47.103.0.0', - 'source.port' : 5672, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@mtk-breizh', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'AMQPLAIN PLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP 24.0.3', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.8.19', - 'extra.naics' : 518210, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 16276, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'SAARBRUCKEN', - 'source.geolocation.region' : 'SAARLAND', - 'source.ip' : '141.95.0.0', - 'source.port' : 5672, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@1397a0e9629b', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'PLAIN AMQPLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP 24.2', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.9.11', - 'extra.naics' : 454110, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 14618, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '54.234.0.0', - 'source.port' : 5672, - 'source.reverse_dns' : 'ec2-54.234.0.0.compute-1.amazonaws.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py deleted file mode 100644 index 4d8420c3b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py +++ /dev/null @@ -1,111 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Tomas Bellus -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ard.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible ARD', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-07-20T00:00:00+00:00", - "extra.file_name": "2020-01-01-scan_ard-test-test.csv", - - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': 'Macmini (radio)', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3283, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': 'biuro-rip-org-pl', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3283, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': '127.0.0.1', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3283, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py deleted file mode 100644 index 3b72baa8d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py +++ /dev/null @@ -1,110 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_chargen.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Chargen', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_chargen-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 19, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 19, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.sector': 'Government', - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 19, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py deleted file mode 100644 index 46c963a79..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py +++ /dev/null @@ -1,82 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_cisco_smart_install.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Cisco Smart Install', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_cisco_smart_install-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible Cisco Smart Install', - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.type': 'vulnerable-system', - 'classification.taxonomy': 'vulnerable', - 'protocol.application': 'cisco-smart-install', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 8559, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '198.51.100.103', - 'source.port': 4786, - 'extra.tag': 'cisco-smart-install', - 'source.reverse_dns': '198-51-100-103.example.net', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-11-18T08:42:45+00:00'}, - {'__type': 'Event', - 'feed.name': 'Accessible Cisco Smart Install', - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.type': 'vulnerable-system', - 'classification.taxonomy': 'vulnerable', - 'protocol.application': 'cisco-smart-install', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 35609, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '198.51.100.218', - 'source.port': 4786, - 'extra.tag': 'cisco-smart-install', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-11-18T08:47:54+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py deleted file mode 100644 index 773fc04d5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_coap.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-CoAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-06-29T00:00:00+00:00", - "extra.file_name": "2020-06-28-scan_coap-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 2.05, - 'extra.response': ',,', - 'extra.response_size': 43, - 'extra.tag': 'coap', - 'extra.version': '2', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5683, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 5.38, - 'extra.response': ',,,,,,,,,', - 'extra.response_size': 113, - 'extra.tag': 'coap', - 'extra.version': '2', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5683, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 113.5, - 'extra.response': '`EsjAy************************************************************|CoAP ' - 'RFC 7252 ' - '|************************************************************|This ' - 'server is using the Eclipse Californium (Cf) CoAP ' - 'framework|published under EPL+EDL: ' - 'http://www.eclipse.org/californium/||(c) 2014, 2015, 2016 ' - 'Institute for Pervasive Computing, ETH Zurich and ' - 'others|************************************************************', - 'extra.response_size': 454, - 'extra.tag': 'coap', - 'extra.version': '1', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5683, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py deleted file mode 100644 index 1bf6f321c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py +++ /dev/null @@ -1,128 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_couchdb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible CouchDB Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_couchdb-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '1.6.1', - 'extra.server_version' : 'CouchDB/1.6.1 (Erlang OTP/18)', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'Ubuntu 16.04', - 'extra.visible_databases' : '_replicator;_users;test;shops;god', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5984, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '3.2.1', - 'extra.features' : 'access-ready,partitioned,pluggable-storage-engines,reshard,scheduler', - 'extra.git_sha' : '244d428af', - 'extra.server_version' : 'CouchDB/3.2.1 (Erlang OTP/23)', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'The Apache Software Foundation', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5984, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '3.2.1', - 'extra.features' : 'access-ready,partitioned,pluggable-storage-engines,reshard,scheduler', - 'extra.git_sha' : '244d428af', - 'extra.server_version' : 'CouchDB/3.2.1 (Erlang OTP/20)', - 'extra.source.sector' : 'Retail Trade', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'The Apache Software Foundation', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5984, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py deleted file mode 100644 index b508b6450..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_cwmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible CWMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_cwmp-test-test.csv", - - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible CWMP', - "classification.identifier": "open-cwmp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.connection": "keep-alive", - "extra.content_length": 5678, - "extra.content_type": "text/html", - "extra.date": "Wed, 04 Sep 2019 07:42:37 GMT", - "extra.http": "HTTP/1.1", - "extra.http_code": 200, - "extra.http_reason": "OK", - "extra.naics": 517311, - "extra.server": "DNVRS-Webs", - "extra.tag": "cwmp", - "protocol.application": "cwmp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.142", - "source.port": 30005, - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T10:44:55+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Accessible CWMP', - "classification.identifier": "open-cwmp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.content_type": "text/html", - "extra.http": "HTTP/1.1", - "extra.http_code": 404, - "extra.http_reason": "Not Found", - "extra.naics": 517311, - "extra.server": "RomPager/4.07 UPnP/1.0", - "extra.tag": "cwmp", - "extra.transfer_encoding": "chunked", - "protocol.application": "cwmp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.162", - "source.port": 5678, - "source.reverse_dns": "localhost.localdomain", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T11:06:50+00:00" - },] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py deleted file mode 100644 index 423ebe8c5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_db2.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Open-DB2-Discovery-Service", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_db2-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 14.9, - 'extra.db2_hostname': 'NOWAK_SERWER', - 'extra.servername': 'node01.example.com', - 'extra.size': 298, - 'extra.tag': 'db2', - 'feed.name': 'ShadowServer Open-DB2-Discovery-Service', - 'protocol.application': 'db2', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 523, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 14.9, - 'extra.db2_hostname': 'SPZOZ-DZIEWIN', - 'extra.servername': 'node02.example.com', - 'extra.size': 298, - 'extra.tag': 'db2', - 'feed.name': 'ShadowServer Open-DB2-Discovery-Service', - 'protocol.application': 'db2', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 523, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py deleted file mode 100644 index 9038a79ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ddos_middlebox.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DDoS Middlebox', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_ddos_middlebox-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source_port' : '49002', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 80, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.source_port' : '41200', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 80, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source_port' : '47492', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 80, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py deleted file mode 100644 index 3492f82ce..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DNS Open Resolvers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_dns-test-test.csv", - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'DNS Open Resolvers', - "classification.identifier": "dns-open-resolver", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.66", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.179", - "source.port": 53, - "source.reverse_dns": "198-51-100-189.example.net", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:34+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'DNS Open Resolvers', - "classification.identifier": "dns-open-resolver", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.51", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.8", - "source.port": 53, - "source.reverse_dns": "198-51-100-111.example.net", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:36+00:00" -},] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py deleted file mode 100644 index 31d0e4417..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py +++ /dev/null @@ -1,159 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_docker.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Docker Service', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_docker-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.37', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2018-05-09T22:18:36.000000000+00:00', - 'extra.content_type' : 'application/json; charset=UTF-8', - 'extra.date' : 'Fri, 06 May 2022 14:06:30 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : 'f150324', - 'extra.go_version' : 'go1.9.5', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-514.26.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.server' : 'Docker/18.05.0-ce (linux)', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'docker', - 'extra.version' : '18.05.0-ce', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 2375, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.26', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2022-03-02T15:25:43.414574467+00:00', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Fri, 06 May 2022 14:08:07 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : '7d71120/1.13.1', - 'extra.go_version' : 'go1.10.3', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-693.2.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.pkg_version' : 'docker-1.13.1-209.git7d71120.el7.centos.x86_64', - 'extra.server' : 'Docker/1.13.1 (linux)', - 'extra.tag' : 'docker', - 'extra.version' : '1.13.1', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 2375, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.37', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2018-05-09T22:18:36.000000000+00:00', - 'extra.content_type' : 'application/json; charset=UTF-8', - 'extra.date' : 'Fri, 06 May 2022 14:08:06 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : 'f150324', - 'extra.go_version' : 'go1.9.5', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-514.26.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.server' : 'Docker/18.05.0-ce (linux)', - 'extra.tag' : 'docker', - 'extra.version' : '18.05.0-ce', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 2375, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py deleted file mode 100644 index 01e68db94..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py +++ /dev/null @@ -1,178 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dvr_dhcpdiscover.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible DVR DHCPDiscover', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_dvr_dhcpdiscover-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 794.0, - 'extra.device_model': 'BCS-TIP3401IR-E-V', - 'extra.device_serial': '6J0E022PAG35073', - 'extra.device_type': 'IPC', - 'extra.device_vendor': 'General', - 'extra.device_version': '2.800.106F004.0.R', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.1', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::1', - 'extra.ipv6_dhcp_enable': False, - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3ac4:e8ff:fe03:b3e2/64', - 'extra.mac_address': '38:c4:e8:03:b3:e2', - 'extra.machine_name': '6J0E022PAG35073', - 'extra.manufacturer': 'General', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 0, - 'extra.response_size': 794, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.video_input_channels': 1, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 37810, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 761.0, - 'extra.device_model': 'HCVR', - 'extra.device_serial': '2K0488CPAGS0ND6', - 'extra.device_type': 'HCVR', - 'extra.device_vendor': 'Private', - 'extra.device_version': '3.210.1.4', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.2', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::2', - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3eef:8cff:fe18:a507/64', - 'extra.mac_address': '3c:ef:8c:18:a5:07', - 'extra.machine_name': 'HCVR', - 'extra.manufacturer': 'Private', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 9, - 'extra.response_size': 761, - 'extra.video_input_channels': 3, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 37810, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 711.0, - 'extra.device_model': 'BCS-XVR0401-IV', - 'extra.device_serial': '5L034FAPAZA0E30', - 'extra.device_type': 'HCVR', - 'extra.device_vendor': 'General', - 'extra.device_version': '4.000.0000002.11', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.3', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::3', - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3ac4:e8ff:fe02:74da/64', - 'extra.mac_address': '38:c4:e8:02:74:da', - 'extra.machine_name': 'XVR', - 'extra.manufacturer': 'General', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 0, - 'extra.response_size': 711, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.video_input_channels': 4, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 37810, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py deleted file mode 100644 index 4e12a1b07..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py +++ /dev/null @@ -1,126 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_elasticsearch.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Elasticsearch', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_elasticsearch-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': '90f439ff60a3c0f497f91663701e64ccd01edbb4', - 'extra.build_snapshot': False, - 'extra.build_timestamp': '2016-07-27T10:36:52Z', - 'extra.cluster_name': 'elasticsearch', - 'extra.lucene_version': '5.5.0', - 'extra.name': 'Red Skull', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '2.3.5', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 9200, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': 'bee86328705acaa9a6daede7140defd4d9ec56bd', - 'extra.build_snapshot': False, - 'extra.cluster_name': 'docker-cluster', - 'extra.lucene_version': '8.11.1', - 'extra.name': 'allinonepod', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '7.17.0', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 9200, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': '79d65f6e357953a5b3cbcc5e2c7c21073d89aa29', - 'extra.build_snapshot': False, - 'extra.cluster_name': 'docker-cluster', - 'extra.lucene_version': '8.9.0', - 'extra.name': 'f547c2952610', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '7.15.0', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 9200, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py deleted file mode 100644 index aeeffa3c2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py +++ /dev/null @@ -1,149 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/scan_exchange.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "Shadowserver CVE-2021-26855", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-scan_exchange.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:30+00:00", - "source.ip": "12.237.1.2", - "source.port": 443, - "source.asn": 7018, - "source.geolocation.cc": "US", - "source.geolocation.region": "CALIFORNIA", - "source.geolocation.city": "TURLOCK", - "source.reverse_dns": 'afs-exch-cas2.xxx.com', - "extra.version": '15.2.721', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 517311, - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "extra.servername": "AFS-EXCH2019", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:37+00:00", - "source.ip": "98.153.3.4", - "source.port": 443, - "source.asn": 20001, - "source.geolocation.cc": "US", - "source.geolocation.region": "CALIFORNIA", - "source.geolocation.city": "LOS ANGELES", - "source.reverse_dns": 'rrcs-98-153-x-x.west.biz.rr.com', - "extra.version": '15.0.847', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 517311, - "extra.tag": "exchange;webshell", - "classification.taxonomy": "intrusions", - "classification.type": "system-compromise", - "classification.identifier": "exchange-server-webshell", - "extra.servername": "SSAMAIL", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "206.210.5.6", - "source.port": 443, - "source.asn": 17054, - "source.geolocation.cc": "US", - "source.geolocation.region": "PENNSYLVANIA", - "source.geolocation.city": "PITTSBURGH", - "source.reverse_dns": 'webmail.xxx.com', - "extra.source.naics": 518210, - "extra.version": '15.0.1178', - "extra.servername": "OMNYXEXCH02", - "extra.tag": "exchange;webshell", - "classification.taxonomy": "intrusions", - "classification.type": "system-compromise", - "classification.identifier": "exchange-server-webshell", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "12.33.7.8", - "source.port": 443, - "source.asn": 7018, - "source.geolocation.cc": "US", - "source.geolocation.region": "ARKANSAS", - "source.geolocation.city": "LITTLE ROCK", - "source.reverse_dns": 'mail.xxx.org', - "extra.version": '15.1.2176', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 921120, - "extra.servername": "MHASVR02", - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "41.204.9.10", - "source.port": 443, - "source.asn": 21042, - "source.geolocation.cc": 'MG', - "source.geolocation.city": 'ANTANANARIVO', - "source.geolocation.region": 'ANTANANARIVO', - "source.reverse_dns": 'mail.xxx.mg', - "extra.servername": "SABMHQE0232", - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py deleted file mode 100644 index 33daefd75..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ftp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible FTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ftp-test-test.csv", - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible FTP', - 'time.observation': '2019-03-25T00:00:00+00:00', - 'time.source': '2019-03-06T06:37:00+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-ftp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.ip': '61.126.3.70', - 'source.port': 21, - 'protocol.transport': 'tcp', - 'protocol.application': 'ftp', - 'source.reverse_dns': 'arcus-net.co.jp', - 'extra.tag': 'ftp', - 'source.asn': 4713, - 'source.geolocation.cc': 'JP', - 'source.geolocation.region': 'TOKYO', - 'source.geolocation.city': 'TOKYO', - 'extra.naics': 517311, - 'extra.sic': 737401, - 'extra.banner': '220 FTP Server ready.|', - 'extra.handshake': 'TLSv1.2', - 'extra.cipher_suite': 'TLS_RSA_WITH_AES_128_CBC_SHA', - 'extra.cert_length': 2048, - 'extra.subject_common_name': '*.bizmw.com', - 'extra.issuer_common_name': 'GlobalSign Organization Validation CA - SHA256 - G2', - 'extra.cert_issue_date': 'Jan 14 08:04:50 2015 GMT', - 'extra.cert_expiration_date': 'Jan 14 08:04:50 2020 GMT', - 'extra.sha1_fingerprint': 'D9:98:3F:2E:F9:D1:BE:9A:10:1E:DE:51:2C:C1:DF:01:18:0A:20:65', - 'extra.cert_serial_number': '1121DC7421AB7924C3B1D396AEA3707E9E29', - 'extra.ssl_version': 2, - 'extra.signature_algorithm': 'sha256WithRSAEncryption', - 'extra.key_algorithm': 'rsaEncryption', - 'extra.subject_organization_name': 'NTT Communications Corporation', - 'extra.subject_country': 'JP', - 'extra.subject_state_or_province_name': 'Tokyo', - 'extra.subject_locality_name': 'Minato-ku', - 'extra.issuer_organization_name': 'GlobalSign nv-sa', - 'extra.issuer_country': 'BE', - 'extra.sha256_fingerprint': '27:4A:8A:3A:A7:DF:82:D0:43:03:0E:6F:48:30:30:C9:24:77:11:1A:08:EF:F7:B9:74:0C:CE:40:87:03:D2:51', - 'extra.sha512_fingerprint': 'E5:93:8B:72:84:0F:35:52:8E:7A:6C:E3:EF:36:90:4C:F2:86:A7:4D:B2:DD:C0:C6:23:83:18:EF:DD:86:34:92:91:57:22:29:75:45:71:8B:3A:CD:F1:27:A9:CA:5F:70:5E:AC:15:A5:E6:63:FD:6F:BB:C5:E2:45:99:73:E9:E6', - 'extra.md5_fingerprint': 'D1:A7:BC:96:78:1D:16:D0:24:A8:62:7C:3A:95:5A:4A', - 'extra.cert_valid': False, - 'extra.self_signed': False, - 'extra.cert_expired': False, - 'extra.validation_level': 'OV', - 'extra.auth_tls_response': '234 AUTH TLS successful', - }, - { - '__type': 'Event', - 'feed.name': 'Accessible FTP', - 'time.observation': '2019-03-25T00:00:00+00:00', - 'time.source': '2019-03-06T06:37:00+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-ftp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.ip': '62.48.156.65', - 'source.port': 21, - 'protocol.transport': 'tcp', - 'protocol.application': 'ftp', - 'source.reverse_dns': 'dial-62-48-156-65.ptprime.net', - 'extra.tag': 'ftp', - 'source.asn': 15525, - 'source.geolocation.cc': 'PT', - 'source.geolocation.region': 'LISBOA', - 'source.geolocation.city': 'FRIELAS', - 'extra.banner': '220-================================================================| PT Empresas| Acesso Reservado| Acesso nao autorizado punido por lei: 109/91; 67/98| ----------------------------------------------------------------| HENNES & MAURITZ LDA - 149093| SITE: PT303 - Cascais Shopping| MORADA: | NIR: EWS1822940| ================================================================|220 FTP server ready, 1 active clients of 4 simultaneous clients allowed.|', - 'extra.auth_tls_response': '500 Syntax error, command unrecognized.', - 'extra.auth_ssl_response': '500 Syntax error, command unrecognized.' - } - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py deleted file mode 100644 index 0b5794cb7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py +++ /dev/null @@ -1,94 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_hadoop.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Accessible-Hadoop", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_hadoop-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'ShadowServer Accessible-Hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-hadoop', - 'extra.version': '2.7.3, rbaa91f7c6bc9cb92be5982de4719c1c8af91ccff', - 'extra.server_type': 'namenode', - 'extra.clusterid': 'CID-64471a53-60cb-4302-9832-92f321f111fe', - 'extra.total_disk': 41567956992, - 'extra.used_disk': 53248, - 'extra.free_disk': 25160089600, - 'extra.livenodes': 'edmonton:50010', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 15296, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'CALGARY', - 'source.geolocation.region': 'ALBERTA', - 'source.ip': '199.116.235.200', - 'source.port': 50070, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-09-13T02:06:05+00:00'}, - {'__type': 'Event', - 'feed.name': 'ShadowServer Accessible-Hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-hadoop', - 'extra.version': '2.7.1.2.4.0.0-169', - 'extra.naics': 334111, - 'extra.sic': 357101, - 'extra.server_type': 'datanode', - 'extra.clusterid': 'CID-771bae52-9e4f-4ec4-bc1a-c867585751f0', - 'extra.namenodeaddress': 'sandbox.hortonworks.com', - 'extra.volumeinfo': '/hadoop/hdfs/data/current', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 8075, - 'source.geolocation.cc': 'US', - 'source.geolocation.city': 'DES MOINES', - 'source.geolocation.region': 'IOWA', - 'source.ip': '104.43.235.92', - 'source.port': 50075, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-09-13T02:07:48+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py deleted file mode 100644 index 793a95f22..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible HTTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_http-test-test.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible HTTP', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'accessible-http', - 'extra.source.naics': 518111, - 'extra.source.sic': 737401, - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_reason': 'OK', - 'extra.content_type': 'text/html', - 'extra.server': 'lighttpd', - 'extra.transfer_encoding': 'chunked', - 'extra.http_date': '2018-04-19T00:02:28+00:00', - 'extra.tag': 'http', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.reverse_dns': 'c-75-74-78-113.hsd1.fl.comcast.net', - 'source.asn': 7922, - 'source.geolocation.cc': 'US', - 'source.geolocation.city': 'MIAMI', - 'source.geolocation.region': 'FLORIDA', - 'source.ip': '75.74.78.113', - 'source.port': 8080, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2018-04-19T00:02:26+00:00'}, - {'__type': 'Event', - 'feed.name': 'Accessible HTTP', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'accessible-http', - 'extra.source.naics': 518210, - 'extra.source.sic': 737415, - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_reason': 'OK', - 'extra.content_type': 'text/html', - 'extra.content_length': 17729, - 'extra.http_date': '2018-04-19T02:02:28+00:00', - 'extra.tag': 'http', - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.reverse_dns': 'sto95-3-88-162-174-130.fbx.proxad.net', - 'source.asn': 12322, - 'source.geolocation.cc': 'FR', - 'source.geolocation.city': 'SAINT-OUEN-LAUMONE', - 'source.ip': '88.162.174.130', - 'source.port': 8080, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2018-04-19T00:02:26+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py deleted file mode 100644 index dc5e94e5e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http_proxy.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open HTTP Proxy', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_http_proxy-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3128, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'extra.via': 'HTTP/1.1 s_proxy_den1', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3128, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'extra.via': 'HTTP/1.1 s_proxy_yvr', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3128, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py deleted file mode 100644 index d15232eaf..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py +++ /dev/null @@ -1,125 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http_vulnerable.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable HTTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-08-01T09:00:00+00:00", - "extra.file_name": "2021-08-01-scan_http_vulnerable-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.content_length': 149, - 'extra.content_type': 'text/html; charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 401, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Unauthorized', - 'extra.server': 'TwistedWeb/19.7.0', - 'extra.set_cookie': 'TWISTED_SESSION=5473ad3faa3de66685fb3a53bffb390b4fcec2039893009a06caf38e1bec8aa8', - 'extra.tag': 'basic-auth,http', - 'extra.www_authenticate': 'Basic realm=\\\\"OpenWebif\\"\\""', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 8080, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.content_length': 149, - 'extra.content_type': 'text/html; charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 401, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Unauthorized', - 'extra.server': 'TwistedWeb/19.7.0', - 'extra.set_cookie': 'TWISTED_SESSION=d2460d37b7fdbdd6c27dd74423ead5704e553d4f2c230672313edc5602059e33', - 'extra.tag': 'basic-auth,http', - 'extra.www_authenticate': 'Basic realm=\\\\"OpenWebif\\"\\""', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 80, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.detail': 'repositoryformatversion = 0;filemode = false;bare = ' - 'false;logallrefupdates = true;symlinks = false;ignorecase = ' - 'true', - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.tag': 'git-config-file', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 443, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py deleted file mode 100644 index f673f40c8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py +++ /dev/null @@ -1,125 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ics.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Acessible ICS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ics-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 1', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDE=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.1', - 'source.port' : 502, - 'source.reverse_dns' : 'host1.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 2', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDI=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64513, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.2', - 'source.port' : 502, - 'source.reverse_dns' : 'host2.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 3', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDM=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64514, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.3', - 'source.port' : 502, - 'source.reverse_dns' : 'host3.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py deleted file mode 100644 index 08a9082af..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ipmi.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open IPMI', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ipmi-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open IPMI', - "classification.identifier": "open-ipmi", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.anon_login": False, - "extra.defaultkg": "-", - "extra.ipmi_version": "1.5", - "extra.md2_auth": False, - "extra.md5_auth": True, - "extra.none_auth": True, - "extra.nulluser": True, - "extra.oem_auth": False, - "extra.passkey_auth": True, - "extra.permessage_auth": True, - "extra.tag": "ipmi", - "extra.userlevel_auth": True, - "extra.usernames": False, - "protocol.application": "ipmi", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 2914, - "source.geolocation.cc": "DE", - "source.geolocation.city": "BERLIN", - "source.geolocation.region": "BERLIN", - "source.ip": "198.51.100.4", - "source.port": 623, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:09:42+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open IPMI', - "classification.identifier": "open-ipmi", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.anon_login": False, - "extra.defaultkg": "default", - "extra.ipmi_version": "2.0", - "extra.md2_auth": False, - "extra.md5_auth": False, - "extra.none_auth": False, - "extra.nulluser": False, - "extra.oem_auth": False, - "extra.passkey_auth": False, - "extra.permessage_auth": False, - "extra.tag": "ipmi", - "extra.userlevel_auth": True, - "extra.usernames": True, - "protocol.application": "ipmi", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 28753, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.182", - "source.port": 623, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:09:43+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py deleted file mode 100644 index 9adc8485e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ipp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open-IPP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-06-09T00:00:00+00:00", - "extra.file_name": "2020-06-08-scan_ipp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open-IPP', - "classification.identifier": "open-ipp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "ipp", - "extra.ipp_version": "IPP/2.1", - "extra.cups_version": "CUPS/2.0", - "extra.printer_uris": "ipp://123.45.67.89:631/ipp/print", - "extra.printer_name": "NPI3F0D22", - "extra.printer_info": "HP Color LaserJet MFP M277dw", - "extra.printer_more_info": "http://123.45.67.89:631/hp/device/info_config_AirPrint.html?tab=Networking&menu=AirPrintStatus", - "extra.printer_make_and_model": "HP Color LaserJet MFP M277dw", - "extra.printer_firmware_name": "20191203", - "extra.printer_firmware_string_version": "20191203", - "extra.printer_firmware_version": "20191203", - "extra.printer_organization": "org", - "extra.printer_organization_unit": "unit", - "extra.printer_uuid": "urn:uuid:456e4238-4a44-4643-4c42-10e1813f0a18", - "extra.printer_wifi_ssid": "wifissid", - "protocol.application": "ipp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 12345, - "source.geolocation.cc": "AA", - "source.geolocation.city": "CITY", - "source.geolocation.region": "REGION", - "source.ip": "123.45.67.89", - "source.port": 631, - 'source.reverse_dns': 'some.host.com', - "time.observation": "2020-06-09T00:00:00+00:00", - "time.source": "2020-06-08T11:30:14+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py deleted file mode 100644 index 3192f508f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py +++ /dev/null @@ -1,105 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_isakmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable ISAKMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_isakmp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Vulnerable ISAKMP', - "classification.identifier": "open-ike", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.domain_of_interpretation": '00', - "extra.exchange_type": '05', - "extra.flags": '00', - "extra.initiator_spi": "3e35c70729dfedef", - "extra.message_id": "00000000", - "extra.naics": 517311, - "extra.next_payload": '11', - "extra.next_payload2": '00', - "extra.notify_message_type": '14', - "extra.responder_spi": "253acab7cbfda607", - "extra.spi_size": 0, - "extra.tag": "isakmp-vulnerable", - "protocol.application": "ipsec", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.42", - "source.port": 500, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T00:17:25+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Vulnerable ISAKMP', - "classification.identifier": "open-ike", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.domain_of_interpretation": '00', - "extra.exchange_type": '05', - "extra.flags": '00', - "extra.initiator_spi": "3e35c70729dfedef", - "extra.message_id": "00000000", - "extra.next_payload": '11', - "extra.next_payload2": '00', - "extra.notify_message_type": '14', - "extra.responder_spi": "b274460e7adc1bf0", - "extra.spi_size": 0, - "extra.tag": "isakmp-vulnerable", - "protocol.application": "ipsec", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.67", - "source.port": 500, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T00:17:28+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py deleted file mode 100644 index 2bac336a7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py +++ /dev/null @@ -1,214 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_kubernetes.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Kubernetes API Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_kubernetes-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2021-11-17T13:00:29Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:13 GMT', - 'extra.git_commit' : '2444b3347a2c45eb965b182fb836e1f51dc61b70', - 'extra.git_tree_state' : 'clean', - 'extra.git_version' : 'v1.20.13', - 'extra.go_version' : 'go1.15.15', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '20', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 6443, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2022-02-25T06:26:46Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:12 GMT', - 'extra.git_commit' : '6f5a5295923a614a4202a7ad274b38b69f9ca8c0', - 'extra.git_tree_state' : 'clean', - 'extra.git_version' : 'v1.23.3+e419edf', - 'extra.go_version' : 'go1.17.5', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '23', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 6443, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2020-05-08T07:29:59Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:12 GMT', - 'extra.git_commit' : '4f7ea78', - 'extra.git_version' : 'v1.16.9-aliyun.1', - 'extra.go_version' : 'go1.13.9', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '16+', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 6443, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py deleted file mode 100644 index b6abf6eba..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py +++ /dev/null @@ -1,154 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ldap_tcp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open LDAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ldap_tcp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 2, - 'extra.ldap_service_name': 'node01.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'node01.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821124435.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 25029662, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node02.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'node02.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821124539.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.ldap_service_name': 'node03.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'node03.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py deleted file mode 100644 index aa4deefb8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py +++ /dev/null @@ -1,162 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ldap_udp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open LDAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ldap_udp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 58.42, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821044533.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 222537, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node01.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 3038, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'node01.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 58.88, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821044948.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 1478714, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node02.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 3062, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'node02.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 0.69, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.ldap_service_name': 'node03.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 36, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'node03.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py deleted file mode 100644 index 9207aaf36..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mdns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open mDNS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mdns-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_ipv4' : '192.168.0.1', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::1', - 'extra.services' : '_smb._tcp.local.; _device-info._tcp.local.; _http._tcp.local.; _dacp._tcp.local.;', - 'extra.tag' : 'mdns', - 'extra.workstation_ipv4' : '192.168.0.1', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::1', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5353, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_ipv4' : '192.168.0.2', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::2', - 'extra.services' : '_home-assistant._tcp.local.;', - 'extra.tag' : 'mdns', - 'extra.workstation_ipv4' : '192.168.0.2', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::2', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5353, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_info' : '\\\\\"vendor=Synology\\\"\\\" \\\"\\\"model=DS218+\\\"\\\" \\\"\\\"serial=17A0PCN482002\\\"\\\" \\\"\\\"version_major=6\\\"\\\" \\\"\\\"version_minor=2\\\"\\\" \\\"\\\"version_build=25556\\\"\\\" \\\"\\\"admin_port=5000\\\"\\\" \\\"\\\"secure_admin_port=5001\\\"\\\" \\\"\\\"mac_address=00:11:32:80:fd:b5\\\"\\\"\"', - 'extra.http_ipv4' : '192.168.0.3', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::3', - 'extra.http_name' : 'snmeijer.local.', - 'extra.http_port' : 5000, - 'extra.http_ptr' : 'snmeijer._http._tcp.local.', - 'extra.http_target' : 'snmeijer.local.', - 'extra.services' : '_webdav._tcp.local.; _adisk._tcp.local.; _smb._tcp.local.; _http._tcp.local.; _dacp._tcp.local.; _afpovertcp._tcp.local.; _device-info._tcp.local.;', - 'extra.tag' : 'mdns,iot', - 'extra.workstation_ipv4' : '192.168.0.3', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::3', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5353, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py deleted file mode 100644 index b54fc0ea5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py +++ /dev/null @@ -1,130 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_memcached.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Memcached', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_memcached-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 81.71, - 'extra.curr_connections': 243, - 'extra.pid': 1010, - 'extra.pointer_size': 64, - 'extra.response_size': 1144, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:34:06', - 'extra.total_connections': 6106, - 'extra.uptime': 32908114, - 'extra.version': '1.4.15', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 50260, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 75.21, - 'extra.curr_connections': 9, - 'extra.pid': 5316, - 'extra.pointer_size': 64, - 'extra.response_size': 1053, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:39:21', - 'extra.total_connections': 2962, - 'extra.uptime': 9618498, - 'extra.version': '1.4.13', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 11211, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 31.57, - 'extra.curr_connections': 2, - 'extra.pid': 1460, - 'extra.pointer_size': 32, - 'extra.response_size': 442, - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:39:39', - 'extra.total_connections': 534, - 'extra.uptime': 1375159, - 'extra.version': '1.2.6', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 11211, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py deleted file mode 100644 index 3ecf7b21f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mongodb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open MongoDB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mongodb-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open MongoDB', - "classification.identifier": "open-mongodb", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.allocator": "tcmalloc", - "extra.bits": "64", - "extra.gitversion": "a2ddc68ba7c9cee17bfe69ed840383ec3506602b", - "extra.javascriptengine": "V8", - "extra.maxbsonobjectsize": "16777216", - "extra.ok": True, - "extra.sysinfo": "Linux ip-198-51-100-100 198.51.100.103-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49", - "extra.tag": "mongodb", - "extra.version": "2.4.5", - "extra.visible_databases": "local | countly | admin", - "protocol.application": "mongodb", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 20773, - "source.geolocation.cc": "DE", - "source.geolocation.city": "WEEZE", - "source.geolocation.region": "NORDRHEIN-WESTFALEN", - "source.ip": "198.51.100.203", - "source.port": 27017, - "source.reverse_dns": "198-51-100-203.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:40:07+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open MongoDB', - "classification.identifier": "open-mongodb", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.allocator": "tcmalloc", - "extra.bits": "64", - "extra.gitversion": "d73c92b1c85703828b55c2916a5dd4ad46535f6a", - "extra.javascriptengine": "V8", - "extra.maxbsonobjectsize": "16777216", - "extra.ok": True, - "extra.sector": "Information Technology", - "extra.sysinfo": "Linux build5.ny.cbi.10gen.cc 2.6.32-431.3.1.el6.x86_64 #1 SMP Fri Jan 3 21:39:27 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49", - "extra.tag": "mongodb", - "extra.version": "2.6.12", - "extra.visible_databases": "none visible", - "protocol.application": "mongodb", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 24940, - "source.geolocation.cc": "DE", - "source.geolocation.city": "GUNZENHAUSEN", - "source.geolocation.region": "BAYERN", - "source.ip": "198.51.100.42", - "source.port": 27017, - "source.reverse_dns": "198-51-100-208.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:40:07+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py deleted file mode 100644 index 45d19f9ee..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py +++ /dev/null @@ -1,89 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mqtt.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open-MQTT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-03-15T00:00:00+00:00", - "extra.file_name": "2020-03-14-scan_mqtt-test-geo.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.anonymous_access' : False, - 'extra.cert_expiration_date' : '2022-11-14 00:00:00', - 'extra.cert_issue_date' : '2020-08-12 00:00:00', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '085699743A23114C9B6B8DC975A8AF42', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Refused, not authorized', - 'extra.hex_code' : '05', - 'extra.issuer_common_name' : 'Sectigo RSA Domain Validation Secure Server CA', - 'extra.issuer_country' : 'GB', - 'extra.issuer_locality_name' : 'Salford', - 'extra.issuer_organization_name' : 'Sectigo Limited', - 'extra.issuer_state_or_province_name' : 'Greater Manchester', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'DE:2C:98:30:27:2E:7D:C9:ED:A3:9D:AF:9E:CE:14:CC', - 'extra.raw_response' : '20020005', - 'extra.sha1_fingerprint' : '70:84:F1:6D:28:DA:B6:E6:27:60:13:8B:2C:93:52:B6:7B:4B:13:7B', - 'extra.sha256_fingerprint' : 'D2:D7:54:52:EB:86:4E:2D:34:4D:FC:CE:CD:CF:39:41:E1:06:5C:8B:B8:54:E6:0C:DF:FD:6E:E3:F1:B5:41:00', - 'extra.sha512_fingerprint' : '17:57:FB:88:9D:BE:A7:F0:29:A5:31:FC:79:DF:F7:8A:1C:D6:4A:DF:1B:4A:DC:BF:05:E7:E8:2F:79:9A:FA:FE:F7:E8:66:22:CB:B9:4C:72:F7:FB:6C:1D:59:8C:54:63:70:05:DE:7F:3C:2F:BA:B8:37:18:CE:29:6F:11:E8:AB', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.naics' : 454110, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : '*.tracesafe.io', - 'extra.tag' : 'mqtt', - 'feed.name' : 'Open-MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 12345, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'COLUMBUS', - 'source.geolocation.region' : 'OHIO', - 'source.ip' : '18.220.0.0', - 'source.port' : 8883, - 'source.reverse_dns' : '18-220-0-0.example.com', - 'time.observation' : '2020-03-15T00:00:00+00:00', - 'time.source' : '2022-02-07T12:56:53+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py deleted file mode 100644 index 461895724..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py +++ /dev/null @@ -1,173 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_mqtt_anon.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Anonymous MQTT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_mqtt_anon-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2030-05-06 08:07:05', - 'extra.cert_issue_date' : '2020-05-08 08:07:05', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '02', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'RootCA', - 'extra.issuer_country' : 'CN', - 'extra.issuer_organization_name' : 'EMQ', - 'extra.issuer_state_or_province_name' : 'hangzhou', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'AB:A8:E0:2C:EF:AE:BF:9D:DD:FA:70:BA:2F:F2:CA:5C', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '70:1A:1E:1F:EC:5F:7E:A9:12:32:B2:C9:8A:C9:EE:91:8E:0B:82:45', - 'extra.sha256_fingerprint' : '85:26:A2:F2:A2:50:CD:96:33:19:A6:2D:12:2E:97:6B:D3:06:3C:11:EA:01:B4:B7:25:2A:B7:4F:0A:8F:45:40', - 'extra.sha512_fingerprint' : '72:50:07:30:9A:6F:CB:FD:E2:80:69:02:65:62:77:16:C3:B4:0C:98:44:4E:D4:2C:AC:6B:AF:F8:9E:AB:51:C2:FA:A8:72:A3:45:DF:81:09:50:08:18:EB:03:34:FC:92:33:A7:12:46:FE:90:20:91:86:C5:4D:89:48:86:4C:CD', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 518210, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'Server', - 'extra.subject_country' : 'CN', - 'extra.subject_organization_name' : 'EMQ', - 'extra.subject_state_or_province_name' : 'hangzhou', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37963, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'SHENZHEN', - 'source.geolocation.region' : 'GUANGDONG SHENG', - 'source.ip' : '47.106.0.0', - 'source.port' : 8883, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2022-03-06 13:48:03', - 'extra.cert_issue_date' : '2021-12-06 13:48:04', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '06B25BEAD1F43266ABCFCDDE408D3544D04B', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'R3', - 'extra.issuer_country' : 'US', - 'extra.issuer_organization_name' : 'Lets Encrypt', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '23:99:39:C6:77:D8:9F:55:90:FC:A5:FB:BA:72:8B:42', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '20:0E:AC:E7:AF:07:8D:D3:16:7C:63:D1:B9:12:AD:1D:2C:F0:46:86', - 'extra.sha256_fingerprint' : 'DD:7A:4C:8A:1D:66:1D:7C:F5:17:04:5B:A0:B4:C4:E0:80:58:44:B4:DB:A7:5E:61:AE:43:9D:85:4C:9E:DC:83', - 'extra.sha512_fingerprint' : '55:B6:3D:56:A4:39:6E:99:B6:AF:72:AF:4D:3C:7C:C5:A8:C5:4F:A1:79:92:D0:46:8A:A2:9B:2A:48:0D:00:68:39:F0:B8:67:B4:E0:88:51:2A:D7:55:46:83:BD:ED:1E:09:6E:DB:3D:21:E2:AA:DB:42:6A:33:45:1A:2A:DB:4C', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 518210, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 24940, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'WERNIGERODE', - 'source.geolocation.region' : 'SACHSEN-ANHALT', - 'source.ip' : '144.76.0.0', - 'source.port' : 8883, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2030-08-05 16:51:57', - 'extra.cert_issue_date' : '2020-08-07 16:51:57', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'A71541EFAE529B03', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'ClearView2Dev', - 'extra.issuer_organization_name' : 'Sohonet', - 'extra.issuer_organization_unit_name' : 'ClearView2Dev', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '43:0D:A7:89:9E:76:8D:6E:D5:AD:95:CC:F2:91:87:56', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '32:4B:66:98:FA:5B:D2:D1:F2:53:83:21:19:11:5A:A9:BE:85:56:16', - 'extra.sha256_fingerprint' : 'AE:0D:65:34:2F:51:F7:32:1E:DF:B1:DA:12:C7:6A:DE:42:B5:4B:FF:80:2C:E5:EF:99:F6:CC:01:4B:C9:77:68', - 'extra.sha512_fingerprint' : '44:C4:B8:19:FA:39:55:51:EC:E4:6D:C4:6D:0F:A5:46:BB:D5:F9:FD:A6:8D:DF:F3:2D:D2:92:6C:0B:D5:D3:25:CB:19:50:9D:A6:A4:D4:D3:2E:53:10:F5:8D:77:F7:90:F8:65:A7:79:AB:14:62:72:01:F3:EA:38:E2:68:C7:25', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 0, - 'extra.subject_common_name' : 'foo.example.com', - 'extra.subject_locality_name' : '<', - 'extra.subject_organization_name' : 'Sohonet', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 5555, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'BURBANK', - 'source.geolocation.region' : 'CALIFORNIA', - 'source.ip' : '173.0.0.0', - 'source.port' : 8883, - 'source.reverse_dns' : 'example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py deleted file mode 100644 index 0f12014e6..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mssql.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open MSSQL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mssql-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 310.0, - 'extra.instance_name': 'OPTIMA', - 'extra.named_pipe': '\\\\\\\\ERPOPTIMA\\\\pipe\\\\MSSQL$OPTIMA\\\\sql\\\\query', - 'extra.response_size': 310, - 'extra.tag': 'mssql', - 'extra.tcp_port': 49729, - 'extra.version': '13.2.5026.0', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'ERPOPTIMA', - 'source.port': 1434, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 226.0, - 'extra.instance_name': 'MSSQLSERVER', - 'extra.response_size': 226, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'mssql', - 'extra.tcp_port': 1433, - 'extra.version': '13.0.1601.5', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'SERWER', - 'source.port': 1434, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 304.0, - 'extra.instance_name': 'INSERTGT', - 'extra.named_pipe': '\\\\\\\\ILONY\\\\pipe\\\\MSSQL$INSERTGT\\\\sql\\\\query', - 'extra.response_size': 304, - 'extra.tag': 'mssql', - 'extra.tcp_port': 49358, - 'extra.version': '10.50.2500.0', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'ILONY', - 'source.port': 1434, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py deleted file mode 100644 index 3e008f950..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py +++ /dev/null @@ -1,258 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_mysql.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible MySQL Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_mysql-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '5.7.37-0ubuntu0.18.04.1', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 3306, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '5.7.30-0ubuntu0.18.04.1-log', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 3306, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '8.0.23', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 3306, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py deleted file mode 100644 index beeac2717..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_nat_pmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open NATPMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_nat_pmp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.1', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 291278940, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5351, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.2', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 768416, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5351, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.3', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 19629454, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5351, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py deleted file mode 100644 index febe8305c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_netbios.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Netbios', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_netbios-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 4.58, - 'extra.mac_address': '00-00-00-00-00-00', - 'extra.machine_name': 'NBG6503', - 'extra.response_size': 229, - 'extra.tag': 'netbios', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.account': 'NBG6503', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 137, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.86, - 'extra.mac_address': '00-00-00-00-00-00', - 'extra.machine_name': 'NAS-OLD', - 'extra.response_size': 193, - 'extra.tag': 'netbios', - 'extra.workgroup': 'PRACOWNIAELN.', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.account': 'NAS-OLD', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 137, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.14, - 'extra.mac_address': '00-25-90-F0-64-64', - 'extra.machine_name': 'HR-SRV01', - 'extra.response_size': 157, - 'extra.sector': 'Government', - 'extra.tag': 'netbios', - 'extra.workgroup': 'HRSIGMA', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': 'InRpbWVzdGFtcCIsImlwIiwicHJvdG9jb2wiLCJwb3J0IiwiaG9zdG5hbWUiLCJ0YWciLCJtYWNfYWRkcmVzcyIsImFzbiIsImdlbyIsInJlZ2lvbiIsImNpdHkiLCJ3b3JrZ3JvdXAiLCJtYWNoaW5lX25hbWUiLCJ1c2VybmFtZSIsIm5haWNzIiwic2ljIiwic2VjdG9yIiwicmVzcG9uc2Vfc2l6ZSIsImFtcGxpZmljYXRpb24iCiIyMDEwLTAyLTEwIDAwOjAwOjAyIiwxOTIuMTY4LjAuMyx1ZHAsMTM3LG5vZGUwMy5leGFtcGxlLmNvbSxuZXRiaW9zLDAwLTI1LTkwLUYwLTY0LTY0LDY0NTEyLFpaLFJlZ2lvbixDaXR5LEhSU0lHTUEsSFItU1JWMDEsLDAsMCxHb3Zlcm5tZW50LDE1NywzLjE0', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 137, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py deleted file mode 100644 index 043cdf1aa..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_netis_router.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_netis_router-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 53413, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 53413, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 53413, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py deleted file mode 100644 index 85ef710d4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py +++ /dev/null @@ -1,161 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ntp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'NTP Version', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ntp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.0, - 'extra.clock': '0xe6ac3809.363028e7', - 'extra.frequency': 2.018, - 'extra.jitter': 0.977, - 'extra.leap': 0.0, - 'extra.noise': '0.984', - 'extra.offset': 0.557, - 'extra.peer': 18986, - 'extra.poll': 10, - 'extra.precision': -10, - 'extra.refid': '81.15.252.130', - 'extra.reftime': '0xe6ac35ba.2d2e8f2b', - 'extra.response_size': 324, - 'extra.rootdelay': 17.685, - 'extra.rootdispersion': 61.254, - 'extra.stability': '0.027', - 'extra.state': '4', - 'extra.stratum': 4, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 123, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.33, - 'extra.clk_wander': 0.007, - 'extra.clock': '0xE6AC3806.7DF3B7A0', - 'extra.frequency': -20.407, - 'extra.jitter': 8.776, - 'extra.leap': 0.0, - 'extra.mintc': '3', - 'extra.offset': -14.502, - 'extra.peer': 19244, - 'extra.precision': -10, - 'extra.refid': '10.48.21.21', - 'extra.reftime': '0xE6AC3431.B3B64790', - 'extra.response_size': 328, - 'extra.rootdelay': 32.25, - 'extra.rootdispersion': 105.778, - 'extra.sector': 'Transportation and Warehousing', - 'extra.stratum': 8, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.tc': 10, - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 123, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.0, - 'extra.clk_wander': 0.001, - 'extra.clock': '0xE6AC380A.5A1CAD00', - 'extra.frequency': -24.01, - 'extra.jitter': 2.343, - 'extra.leap': 0.0, - 'extra.mintc': '3', - 'extra.offset': 0.49, - 'extra.peer': 51892, - 'extra.precision': -10, - 'extra.refid': '172.28.0.1', - 'extra.reftime': '0xE6AC3020.0C49BA80', - 'extra.response_size': 324, - 'extra.rootdelay': 7.749, - 'extra.rootdispersion': 81.612, - 'extra.stratum': 4, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.tc': 10, - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 123, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py deleted file mode 100644 index ff0e95f3e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py +++ /dev/null @@ -1,108 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ntpmonitor.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'NTP Monitor', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ntpmonitor-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 55.33, - 'extra.packets': 2, - 'extra.size': 664, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 123, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3666.67, - 'extra.packets': 100, - 'extra.size': 44000, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 123, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3666.67, - 'extra.packets': 100, - 'extra.size': 44000, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 123, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py deleted file mode 100644 index 11caec78a..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_portmapper.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Portmapper', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_portmapper-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 111, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 111, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Government', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 111, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py deleted file mode 100644 index 43a297f78..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py +++ /dev/null @@ -1,199 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_postgres.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-PostgreSQL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_postgres-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5432, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5432, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5432, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py deleted file mode 100644 index de52af625..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_qotd.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open QOTD', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_qotd-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 166.0, - 'extra.quote': '_The secret of being miserable is to have leisure to bother ' - 'about whether?? you are happy or not. The cure for it is ' - 'occupation._?? George Bernard Shaw (1856-1950)?', - 'extra.response_size': 166, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 17, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 162.0, - 'extra.quote': '_Oh the nerves, the nerves; the mysteries of this machine ' - 'called man!?? Oh the little that unhinges it, poor creatures ' - 'that we are!_?? Charles Dickens (1812-70)?', - 'extra.response_size': 162, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 17, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 162.0, - 'extra.quote': '_Oh the nerves, the nerves; the mysteries of this machine ' - 'called man!?? Oh the little that unhinges it, poor creatures ' - 'that we are!_?? Charles Dickens (1812-70)?', - 'extra.response_size': 162, - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 17, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py deleted file mode 100644 index 23d11ce99..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_quic.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible QUIC Report', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_quic-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517311, - 'extra.tag' : 'quic', - 'extra.version_field_1' : 'Q050', - 'extra.version_field_3' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 5607, - 'source.geolocation.cc' : 'UK', - 'source.geolocation.city' : 'LONDON', - 'source.geolocation.region' : 'LONDON', - 'source.ip' : '176.255.0.0', - 'source.port' : 443, - 'source.reverse_dns' : 'test1.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517311, - 'extra.tag' : 'quic', - 'extra.version_field_1' : 'Q050', - 'extra.version_field_2' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 6327, - 'source.geolocation.cc' : 'CA', - 'source.geolocation.city' : 'MEACHAM', - 'source.geolocation.region' : 'SASKATCHEWAN', - 'source.ip' : '24.244.0.0', - 'source.port' : 443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517919, - 'extra.tag' : 'quic', - 'extra.version_field_2' : 'Q050', - 'extra.version_field_3' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'JP', - 'source.geolocation.city' : 'OSAKA', - 'source.geolocation.region' : 'OSAKA', - 'source.ip' : '23.60.0.0', - 'source.port' : 443, - 'source.reverse_dns' : 'test3.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py deleted file mode 100644 index 7c052c451..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py +++ /dev/null @@ -1,236 +0,0 @@ -# SPDX-FileCopyrightText: 2020 sinus-x -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/scan_radmin.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "Accessible Radmin", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-scan_radmin-test-test.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin (Details Unknown)", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.asn": 701, - "source.geolocation.cc": "US", - "source.geolocation.city": "BROOKLYN", - "source.geolocation.region": "NEW YORK", - "source.ip": "74.101.218.75", - "source.port": 4899, - "source.reverse_dns": "static-74-101-218-75.nycmny.fios.verizon.net", - "time.source": "2020-07-06T13:55:26+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.asn": 56618, - "source.geolocation.cc": "RU", - "source.geolocation.city": "MURMANSK", - "source.geolocation.region": "MURMANSKAYA OBLAST", - "source.ip": "192.162.189.171", - "source.port": 4899, - "source.reverse_dns": "rubin.an.ru", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin (Details Unknown)", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "CN", - "source.geolocation.city": "BEIJING", - "source.geolocation.region": "BEIJING SHI", - "source.asn": 4808, - "source.ip": "111.197.143.69", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "KR", - "source.geolocation.city": "DAEIN-DONG", - "source.geolocation.region": "GWANGJU-GWANGYEOKSI", - "source.asn": 4766, - "source.ip": "121.147.215.220", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "KR", - "source.geolocation.city": "DAEIN-DONG", - "source.geolocation.region": "GWANGJU-GWANGYEOKSI", - "source.asn": 4766, - "source.ip": "121.147.215.178", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "CN", - "source.geolocation.city": "CHONGQING", - "source.geolocation.region": "CHONGQING SHI", - "source.asn": 9808, - "source.ip": "183.230.5.219", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[6]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "RU", - "source.geolocation.city": "MOSCOW", - "source.geolocation.region": "MOSKVA", - "source.asn": 34300, - "source.ip": "85.93.154.74", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[7]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "BE", - "source.geolocation.city": "BRASSCHAAT", - "source.geolocation.region": "ANTWERPEN", - "source.asn": 5432, - "source.ip": "81.246.135.247", - "source.port": 4899, - "source.reverse_dns": "247.135-246-81.adsl-dyn.isp.belgacom.be", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[8]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "ES", - "source.geolocation.city": "LAS PALMAS DE GRAN CANARIA", - "source.geolocation.region": "LAS PALMAS", - "source.asn": 12430, - "source.ip": "46.27.146.22", - "source.port": 4899, - "source.reverse_dns": "static-22-146-27-46.ipcom.comunitel.net", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[9]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py deleted file mode 100644 index 28a4a02c2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rdp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible RDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rdp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible RDP', - "classification.identifier": "open-rdp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.bluekeep_vulnerable": False, - "extra.cert_expiration_date": "2019-10-29 02:22:06", - "extra.cert_issue_date": "2019-04-29 02:22:06", - "extra.cert_length": 5678, - "extra.cert_serial_number": "1EF2B37AF850C9BF4E88F18177001D6B", - "extra.cve20190708_vulnerable": False, - "extra.issuer_common_name": "KABESRV.KABE.local", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "BC:6E:C3:E2:98:22:EC:BA:5B:30:E2:53:FD:4A:9D:FF", - "extra.naics": 517311, - "extra.rdp_protocol": "RDP", - "extra.sha1_fingerprint": "EC:BB:4D:DB:9F:0C:D3:FF:5B:49:EA:B1:56:62:B6:A7:5D:60:54:42", - "extra.sha256_fingerprint": "B7:C9:F4:07:D5:C0:75:1D:EA:0C:40:E7:26:39:C2:30:C6:13:83:7E:18:46:D8:E9:4C:45:3F:88:1B:0B:70:76", - "extra.sha512_fingerprint": "08:AC:75:FA:EB:A3:2B:44:15:DE:6D:A7:0B:C0:AE:17:94:F3:55:D9:EC:70:AC:5B:B7:94:79:F0:D7:84:83:89:CB:A9:11:E0:08:D7:54:4D:33:85:89:D2:A8:DD:9D:15:F4:CC:95:DE:6A:E3:DF:6B:FA:8B:27:E3:DA:16:AF:0A", - "extra.signature_algorithm": "sha256WithRSAEncryption", - "extra.ssl_version": 2, - "extra.subject_common_name": "KABESRV.KABE.local", - "extra.tag": "rdp", - "protocol.application": "rdp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.178", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T15:45:51+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible RDP', - "classification.identifier": "open-rdp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.bluekeep_vulnerable": False, - "extra.cert_expiration_date": "2019-10-16 06:15:20", - "extra.cert_issue_date": "2019-04-16 06:15:20", - "extra.cert_length": 5678, - "extra.cert_serial_number": "3FF3EBC5CF154BA54D128A8548C8AAF5", - "extra.cve20190708_vulnerable": False, - "extra.issuer_common_name": "RAMBLA01.rambla.local", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "38:73:6A:B3:AA:41:69:C9:BA:E7:3D:D7:40:16:F8:AA", - "extra.naics": 517311, - "extra.rdp_protocol": "RDP", - "extra.sector": "Information Technology", - "extra.sha1_fingerprint": "7A:67:1F:F8:87:C6:B0:AC:A9:84:15:B7:40:EC:CB:19:AA:E3:19:52", - "extra.sha256_fingerprint": "8F:CD:7D:C4:80:2D:8D:9B:06:A0:40:18:9F:ED:73:7A:BA:83:55:BE:1B:56:83:A2:97:DF:BB:B4:06:57:CB:F1", - "extra.sha512_fingerprint": "E8:9B:9A:93:69:B4:58:01:D8:46:C2:DC:01:20:1E:DD:93:E1:EB:E3:9D:6B:65:A0:C5:00:6C:A4:44:08:FE:A4:A6:19:FF:55:79:F2:AA:61:68:C8:1C:B0:CE:78:EB:84:DD:29:9D:64:2F:4E:25:31:3A:6C:B8:02:C9:AF:F5:1F", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.ssl_version": 2, - "extra.subject_common_name": "RAMBLA01.rambla.local", - "extra.tag": "rdp", - "protocol.application": "rdp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.233", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T15:45:51+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py deleted file mode 100644 index 54be35a26..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rdpeudp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible MS RDPEUDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rdpeudp-test-geo.csv", - } - -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '05b28c0c', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '053d355f', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '0567a8cb', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py deleted file mode 100644 index 04552e2ec..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_redis.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Redis', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_redis-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open Redis', - "classification.identifier": "open-redis", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.build_id": "26069fb482f6334b", - "extra.connected_clients": "50", - "extra.gcc_version": "4.7.2", - "extra.git_sha1": "00000000", - "extra.mode": "standalone", - "extra.multiplexing_api": "epoll", - "extra.naics": 541512, - "extra.os.name": "Linux 3.2.0-4-amd64 x86_64", - "extra.process_id": "2127", - "extra.run_id": "d440b0b2fb3d1db655ad607e11e6f38011a0f599", - "extra.sic": 737999, - "extra.tag": "redis", - "extra.uptime": 27946314, - "extra.version": "2.8.19", - "protocol.application": "redis", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 201229, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.152", - "source.port": 6379, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:42:33+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open Redis', - "classification.identifier": "open-redis", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.build_id": "e41bf84a0cecf09d", - "extra.connected_clients": "25376", - "extra.gcc_version": "4.8.4", - "extra.git_sha1": "00000000", - "extra.mode": "standalone", - "extra.multiplexing_api": "epoll", - "extra.os.name": "Linux 3.18.24-sirzion x86_64", - "extra.process_id": "343519", - "extra.run_id": "53d63f23511dc0080b49aaa8e8203d65619f1c8c", - "extra.tag": "redis", - "extra.uptime": 310556, - "extra.version": "3.0.6", - "protocol.application": "redis", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 12586, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.67", - "source.port": 6379, - "source.reverse_dns": "198-51-100-67.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:42:43+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py deleted file mode 100644 index e2a961f71..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rsync.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Rsync', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rsync-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 873, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 873, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 873, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py deleted file mode 100644 index 6b972ec5d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_sip.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-SIP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_sip-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.sip_allow': 'INVITE,ACK,BYE,CANCEL,REGISTER', - 'extra.amplification': 15.57, - 'extra.content_length': 0, - 'extra.response_size': 109, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '489', - 'extra.sip_reason': 'Event Package Not Supported', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5060, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 62.57, - 'extra.content_length': 364, - 'extra.content_type': 'text/plain', - 'extra.response_size': 438, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '400', - 'extra.sip_reason': 'Bad Request', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5060, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.57, - 'extra.content_length': 0, - 'extra.response_size': 46, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '400', - 'extra.sip_reason': 'Bad Request', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5060, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py deleted file mode 100644 index f05973cf5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py +++ /dev/null @@ -1,137 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_slp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SLP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_slp-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 427, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 427, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 427, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py deleted file mode 100644 index 921525122..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_smb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SMB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_smb-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py deleted file mode 100644 index cae83d273..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest -import json - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser_json import ShadowserverJSONParserBot -from intelmq.tests.bots.parsers.shadowserver.test_testdata import csvtojson - -EXAMPLE_FILE = csvtojson(os.path.join(os.path.dirname(__file__), 'testdata/scan_smb.csv')) - -EXAMPLE_REPORT = { - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_smb-test-geo.json", - } - -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[0]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverJSONParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverJSONParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverJSONParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py deleted file mode 100644 index 4428420cf..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_smtp_vulnerable.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable SMTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-07-08T00:00:00+00:00", - "extra.file_name": "2021-07-08-scan_smtp_vulnerable-test-test.csv", - } - -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'vulnerable-smtp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.banner' : '220 smtp-server.invalid ESMTP Exim 4.80 Wed, 11 Jun 2021 10:00:00 +0300|', - 'extra.tag' : 'smtp;21nails', - 'feed.name' : 'Vulnerable SMTP', - 'protocol.application' : 'smtp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 12345, - 'source.geolocation.cc' : 'EE', - 'source.geolocation.city' : 'TALLINN', - 'source.geolocation.region' : 'HARJUMAA', - 'source.ip' : '1.2.3.4', - 'source.port' : 25, - 'source.reverse_dns' : 'smtp-server.invalid', - 'time.observation' : '2021-07-08T00:00:00+00:00', - 'time.source' : '2021-07-08T11:58:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'vulnerable-smtp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.banner' : '220 smtp-out.invalid, ESMTP EXIM 4.86_2|', - 'extra.tag' : 'smtp;21nails', - 'feed.name' : 'Vulnerable SMTP', - 'protocol.application' : 'smtp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 23456, - 'source.geolocation.cc' : 'EE', - 'source.geolocation.city' : 'TALLINN', - 'source.geolocation.region' : 'HARJUMAA', - 'source.ip' : '5.6.7.8', - 'source.port' : 25, - 'source.reverse_dns' : 'smtp-out.invalid', - 'time.observation' : '2021-07-08T00:00:00+00:00', - 'time.source' : '2021-07-08T11:58:44+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py deleted file mode 100644 index e6da5b34f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_snmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SNMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_snmp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.94, - 'extra.community': 'public', - 'extra.response_size': 165, - 'extra.sysdesc': 'Linux localhost 3.18.20 #1 SMP Mon Jul 9 14:11:21 CST 2018 ' - 'armv7l', - 'extra.tag': 'snmp', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 161, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.35, - 'extra.community': 'public', - 'extra.device_sector': 'consumer', - 'extra.device_type': 'router', - 'extra.device_vendor': 'MikroTik', - 'extra.response_size': 115, - 'extra.sysdesc': 'RouterOS CCR1009-8G-1S-1S+', - 'extra.tag': 'snmp,iot', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 161, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.0, - 'extra.community': 'public', - 'extra.response_size': 85, - 'extra.tag': 'snmp', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 161, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py deleted file mode 100644 index 067602aa1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_socks.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SOCKS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_socks-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks4', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 1080, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks5', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 1080, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Retail Trade', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks4', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 1080, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py deleted file mode 100644 index 0811f15ed..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py +++ /dev/null @@ -1,136 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssdp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SSDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssdp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.35, - 'extra.cache_control': 'max-age=100', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node01.example.com', - 'extra.location': 'http://192.168.200.254:49152/description.xml', - 'extra.response_size': 325, - 'extra.search_target': 'upnp:rootdevice', - 'extra.sector': 'Government', - 'extra.server': 'Linux/2.6.26, UPnP/1.0, Portable SDK for UPnP devices/1.3.1', - 'extra.systime': 'Sun, 21 Aug 2022 09:51:13 GMT', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:28802880-2880-1880-a880-001bc502f600::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 60194, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 2.71, - 'extra.cache_control': 'max-age = 1800', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node02.example.com', - 'extra.location': 'http://95.160.216.14:52235/dmr/SamsungMRDesc.xml', - 'extra.response_size': 263, - 'extra.search_target': 'upnp:rootdevice', - 'extra.server': 'Linux/9.0 UPnP/1.0 PROTOTYPE/1.0', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:f144ca92-6816-94b5-b95f-b58180834044::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 38732, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 4.79, - 'extra.cache_control': 'max-age=1800', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node03.example.com', - 'extra.location': 'http://192.168.1.3:8008/ssdp/device-desc.xml', - 'extra.response_size': 465, - 'extra.search_target': 'upnp:rootdevice', - 'extra.sector': 'Government', - 'extra.server': 'Linux/3.10.79, UPnP/1.0, Portable SDK for UPnP ' - 'devices/1.6.18', - 'extra.systime': 'Sun, 03 Jan 2016 21:37:50 GMT', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:62fa0fc8-079d-d00f-2e22-59b49fb488f9::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 57626, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py deleted file mode 100644 index a01383713..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py +++ /dev/null @@ -1,182 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssh.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SSH', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ssh-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ecdsa-sha2-nistp256', - 'extra.available_ciphers' : 'chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com, aes128-cbc, aes192-cbc, aes256-cbc, blowfish-cbc, cast128-cbc, 3des-cbc', - 'extra.available_compression' : 'none, zlib@openssh.com', - 'extra.available_kex' : 'curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha256, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1', - 'extra.available_mac' : 'umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1', - 'extra.ecdsa_curve' : 'P-256', - 'extra.ecdsa_curve25519' : '1xx7ASut7BF4ED8b592bebZBMBKTCzOsmbH4cjwx/0U=', - 'extra.ecdsa_public_key_b' : 'WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=', - 'extra.ecdsa_public_key_gx' : 'axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=', - 'extra.ecdsa_public_key_gy' : 'T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=', - 'extra.ecdsa_public_key_length' : '256', - 'extra.ecdsa_public_key_n' : '/////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=', - 'extra.ecdsa_public_key_p' : '/////wAAAAEAAAAAAAAAAAAAAAD///////////////8=', - 'extra.ecdsa_public_key_x' : 'NIQdotpy2HAHfSu0DjEmA3cb3NeQKabZaFX9OU0GsPY=', - 'extra.ecdsa_public_key_y' : '0fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=', - 'extra.selected_cipher' : 'aes128-ctr', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'curve25519-sha256@libssh.org', - 'extra.selected_mac' : 'hmac-sha2-256', - 'extra.server_cookie' : 'bGjsifbPIDWT7tAu8BMjyg==', - 'extra.server_host_key' : 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDSEHaLacthwB30rtA4xJgN3G9zXkCmm2WhV/TlNBrD20fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=', - 'extra.server_host_key_sha256' : 'a6e4e1c16ba25d51bcddc58a6e16797144575dd18d02d9dedf75093d2b15c557', - 'extra.server_signature_raw' : 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAABKAAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=', - 'extra.server_signature_value' : 'AAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=', - 'extra.serverid_raw' : 'SSH-2.0-OpenSSH_7.4', - 'extra.serverid_software' : 'OpenSSH_7.4', - 'extra.serverid_version' : '2.0', - 'extra.source.naics' : 454110, - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 16509, - 'source.geolocation.cc' : 'JP', - 'source.geolocation.city' : 'TOKYO', - 'source.geolocation.region' : 'TOKYO', - 'source.ip' : '18.179.0.0', - 'source.port' : 22, - 'source.reverse_dns' : 'ec2-18-179-0-0.ap-northeast-1.compute.amazonaws.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ssh-rsa', - 'extra.available_ciphers' : 'aes128-cbc, 3des-cbc, aes256-cbc, twofish256-cbc, twofish-cbc, twofish128-cbc, blowfish-cbc', - 'extra.available_compression' : 'none', - 'extra.available_kex' : 'diffie-hellman-group1-sha1', - 'extra.available_mac' : 'hmac-sha1-96, hmac-sha1, hmac-md5', - 'extra.device_vendor' : 'Arris', - 'extra.rsa_exponent' : '65537', - 'extra.rsa_length' : '1040', - 'extra.rsa_modulus' : 'g6tZBqqsHUFa8gjXKMg2waZy/6JLzNnfTZkdDo6a3E6Amdk2eVlGbdgRsDxNUCyWLJI4b+xsJX0OKoRKhw4N+cDK5Du4/S00Auylt8YrDi7lIpeBZjMT5KJerN/feOcwRqY8NJfwZiZ1A5brcWr5HXfOH6aRTMFzadr248VdUmFXPQ==', - 'extra.selected_cipher' : 'aes128-cbc', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'diffie-hellman-group1-sha1', - 'extra.selected_mac' : 'hmac-sha1', - 'extra.server_cookie' : 'Y4RQS9sdRgEFwNJKVP6bZg==', - 'extra.server_host_key' : 'AAAAB3NzaC1yc2EAAAADAQABAAAAgwCDq1kGqqwdQVryCNcoyDbBpnL/okvM2d9NmR0OjprcToCZ2TZ5WUZt2BGwPE1QLJYskjhv7GwlfQ4qhEqHDg35wMrkO7j9LTQC7KW3xisOLuUil4FmMxPkol6s39945zBGpjw0l/BmJnUDlutxavkdd84fppFMwXNp2vbjxV1SYVc9', - 'extra.server_host_key_sha256' : 'd53fedbfe92e631264629882b2e85bfd213ca4b07b824cd31f8de1fcb8d0ddcb', - 'extra.server_signature_raw' : 'AAAAB3NzaC1yc2EAAACCLQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==', - 'extra.server_signature_value' : 'LQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==', - 'extra.serverid_raw' : 'SSH-2.0-ARRIS_0.50', - 'extra.serverid_software' : 'ARRIS_0.50', - 'extra.serverid_version' : '2.0', - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey, password', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 11976, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MARSHALL', - 'source.geolocation.region' : 'TEXAS', - 'source.ip' : '170.10.0.0', - 'source.port' : 22, - 'source.reverse_dns' : '170-10-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ssh-rsa', - 'extra.available_ciphers' : 'aes128-cbc, 3des-cbc, aes192-cbc, aes256-cbc', - 'extra.available_compression' : 'none', - 'extra.available_kex' : 'diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1', - 'extra.available_mac' : 'hmac-sha1, hmac-sha1-96, hmac-md5, hmac-md5-96', - 'extra.device_sector' : 'enterprise', - 'extra.device_vendor' : 'Cisco', - 'extra.rsa_exponent' : '65537', - 'extra.rsa_length' : '4096', - 'extra.rsa_modulus' : 'yFVwcChoYt+YGm8BzWYugcZbNQRrQ1VWRYcL4U6SSkyoVeE9h5wxRu/hQaWHo3PdsB9Nuln/riRyKZypFUEZ5zlffMyl1uvE8/jp8E/GgUHSyPkGAwu8C8BkX/nDolxAJKTK6djiZnvhsEPe6AXHBMHbto/b3GABUNPngjzX8D63GYcFW9NJLf5qC1UsVkXbAzM0IjQ2X9s3pfhUCAJeXAn2i0gEGtUyF8vEjNdwdG655aXciKrpEEtM1L/zy/+gLH4YC13kAYI7NVyH+qi/mXbULLOQClA7iYK1g3Et58jWUIPwgLfF3SLC57bt2wp/lRgNTv4FBi0tWvRqBnf5UQK5ZjgzbW3bO+Ju4cWgH/4M4NCxSceh4cLm5lQs01xB5feSh2ByqA7wrVDoFJu81LoMVo4bCz30+lH2QsLwmNtUhlWLKBD4k09g4bgBa4jPj0/Nya3rBR4GQ6LG6ltFQotm8wCkgbv76YWqk20nQ6NMYZFvSQm981JFtoHv3vxq48VeHDV0QvV0P12BCFprRf4B0otIvSsHl+LDeUxJAf+Nbw78gzncjyfCbWtCPbwaJQ8CeqnTBzj5TluaFvN8goG5lCTWJGfjIrwAZXOokv9NOqmIiMJJx3s22OX6GHfJAzje2ALLDsAiXBub4iCOdGdTfVbBpFL+bGTK9qfa8vE=', - 'extra.selected_cipher' : 'aes128-cbc', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'diffie-hellman-group14-sha1', - 'extra.selected_mac' : 'hmac-sha1', - 'extra.server_cookie' : 'Z2fOfWsrLlh76Y0bOqa1cw==', - 'extra.server_host_key' : 'AAAAB3NzaC1yc2EAAAADAQABAAACAQDIVXBwKGhi35gabwHNZi6Bxls1BGtDVVZFhwvhTpJKTKhV4T2HnDFG7+FBpYejc92wH026Wf+uJHIpnKkVQRnnOV98zKXW68Tz+OnwT8aBQdLI+QYDC7wLwGRf+cOiXEAkpMrp2OJme+GwQ97oBccEwdu2j9vcYAFQ0+eCPNfwPrcZhwVb00kt/moLVSxWRdsDMzQiNDZf2zel+FQIAl5cCfaLSAQa1TIXy8SM13B0brnlpdyIqukQS0zUv/PL/6AsfhgLXeQBgjs1XIf6qL+ZdtQss5AKUDuJgrWDcS3nyNZQg/CAt8XdIsLntu3bCn+VGA1O/gUGLS1a9GoGd/lRArlmODNtbds74m7hxaAf/gzg0LFJx6HhwubmVCzTXEHl95KHYHKoDvCtUOgUm7zUugxWjhsLPfT6UfZCwvCY21SGVYsoEPiTT2DhuAFriM+PT83JresFHgZDosbqW0VCi2bzAKSBu/vphaqTbSdDo0xhkW9JCb3zUkW2ge/e/GrjxV4cNXRC9XQ/XYEIWmtF/gHSi0i9KweX4sN5TEkB/41vDvyDOdyPJ8Jta0I9vBolDwJ6qdMHOPlOW5oW83yCgbmUJNYkZ+MivABlc6iS/006qYiIwknHezbY5foYd8kDON7YAssOwCJcG5viII50Z1N9VsGkUv5sZMr2p9ry8Q==', - 'extra.server_host_key_sha256' : '06ff3cce443ed832927576d982b69d5a526d0e63334c72e87201deda61679406', - 'extra.server_signature_raw' : 'AAAAB3NzaC1yc2EAAAIAlrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=', - 'extra.server_signature_value' : 'lrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=', - 'extra.serverid_raw' : 'SSH-1.99-Cisco-1.25', - 'extra.serverid_software' : 'Cisco-1.25', - 'extra.serverid_version' : '1.99', - 'extra.source.naics' : 517311, - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey, keyboard-interactive, password', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 33363, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ORLANDO', - 'source.geolocation.region' : 'FLORIDA', - 'source.ip' : '72.17.0.0', - 'source.port' : 22, - 'source.reverse_dns' : '072-017-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py deleted file mode 100644 index f96c03e56..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py +++ /dev/null @@ -1,218 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssl.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SSL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ssl-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: unknown error', - 'extra.browser_trusted' : False, - 'extra.cert_expiration_date' : '2038-01-19 03:14:07', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2014-06-23 09:56:32', - 'extra.cert_length' : 1024, - 'extra.cert_serial_number' : '168CAE', - 'extra.cert_valid' : True, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.content_length' : 131, - 'extra.content_type' : 'text/html', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : 'support', - 'extra.issuer_country' : 'US', - 'extra.issuer_email_address' : 'support@fortinet.com', - 'extra.issuer_locality_name' : 'Sunnyvale', - 'extra.issuer_organization_name' : 'Fortinet', - 'extra.issuer_organization_unit_name' : 'Certificate Authority', - 'extra.issuer_state_or_province_name' : 'California', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '99:45:1F:2E:AE:EB:88:91:27:43:33:79:FA:93:7D:CA', - 'extra.self_signed' : False, - 'extra.server_type' : 'xxxxxxxx-xxxxx', - 'extra.sha1_fingerprint' : '5A:3D:FF:06:F9:E9:25:37:57:F9:09:52:33:A4:85:15:24:2D:88:7F', - 'extra.sha256_fingerprint' : '35:AB:B6:76:2A:3D:17:B2:FB:40:45:1B:FC:0A:99:0A:6E:48:57:F7:30:0A:3B:B1:1A:E6:99:70:5B:7C:32:41', - 'extra.sha512_fingerprint' : '88:7B:16:DB:39:44:0C:47:0E:4A:8F:0B:C5:FB:4D:45:BC:93:5A:00:43:A1:D9:7F:05:1D:86:33:02:F8:FC:57:67:A6:1D:C0:FF:F7:D2:40:D8:9A:21:AE:4E:6D:DC:E7:FF:72:BF:13:CB:EE:A7:5F:CD:83:EA:8A:5E:FB:87:DD', - 'extra.signature_algorithm' : 'sha1WithRSAEncryption', - 'extra.source.naics' : 517311, - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'FGT60D4614030700', - 'extra.subject_country' : 'US', - 'extra.subject_email_address' : 'support@fortinet.com', - 'extra.subject_locality_name' : 'Sunnyvale', - 'extra.subject_organization_name' : 'Fortinet', - 'extra.subject_organization_unit_name' : 'FortiGate', - 'extra.subject_state_or_province_name' : 'California', - 'extra.tag' : 'ssl,vpn', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 4181, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MILWAUKEE', - 'source.geolocation.region' : 'WISCONSIN', - 'source.ip' : '96.60.0.0', - 'source.port' : 10443, - 'source.reverse_dns' : '96-60-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: unknown error', - 'extra.browser_trusted' : False, - 'extra.cert_expiration_date' : '2023-02-06 01:01:34', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2022-01-04 01:01:34', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '36974C4C6B1B3785', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.content_type' : 'text/html; charset=UTF-8', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_connection' : 'keep-alive', - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : '1078-btb-tbi-HungHa-61d39c6d5a7e2', - 'extra.issuer_organization_name' : 'pfSense webConfigurator Self-Signed Certificate', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '16:93:9A:F4:35:7F:9A:85:45:71:91:C7:7C:80:88:00', - 'extra.self_signed' : True, - 'extra.server_type' : 'nginx', - 'extra.set_cookie' : 'PHPSESSID=e15bdfa5739c36877608eb4cf46cc388; path=/; secure; HttpO', - 'extra.sha1_fingerprint' : 'A9:00:BB:E1:54:4D:56:54:59:F1:B7:EA:F1:1A:D5:36:5C:63:90:8E', - 'extra.sha256_fingerprint' : '38:85:F0:44:1E:AD:84:B8:2F:43:68:BA:AC:EE:17:13:A4:BF:86:1D:48:75:7E:22:FA:08:4C:28:5F:AC:3E:5F', - 'extra.sha512_fingerprint' : 'AE:1B:4F:D1:E4:C0:35:9D:2A:4F:7A:37:B8:7B:11:9D:84:25:23:21:AB:EF:B2:0F:DC:C9:F2:A3:72:28:92:E1:74:72:FA:E1:09:6C:E1:F6:B6:E3:A7:61:1C:58:89:34:D7:06:5C:3D:0A:A7:F6:CC:8A:D6:24:D0:04:4C:03:02', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 517311, - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : '1078-btb-tbi-HungHa-61d39c6d5a7e2', - 'extra.subject_organization_name' : 'pfSense webConfigurator Self-Signed Certificate', - 'extra.tag' : 'ssl', - 'extra.transfer_encoding' : 'chunked', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 45899, - 'source.geolocation.cc' : 'VN', - 'source.geolocation.city' : 'THAI BINH', - 'source.geolocation.region' : 'THAI BINH', - 'source.ip' : '113.160.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_trusted' : True, - 'extra.cert_expiration_date' : '2022-11-06 15:30:28', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2021-10-07 15:30:28', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '7B388364A24B88E77E5553B5C6748100', - 'extra.cert_valid' : True, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.content_length' : 131, - 'extra.content_type' : 'text/html', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : 'Entrust Certification Authority - L1K', - 'extra.issuer_country' : 'US', - 'extra.issuer_organization_name' : 'Entrust, Inc.', - 'extra.issuer_organization_unit_name' : '(c) 2012 Entrust, Inc. - for authorized use only', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'E7:34:BC:92:84:FA:39:DE:E1:46:6C:27:DA:5A:01:F4', - 'extra.self_signed' : False, - 'extra.server_type' : 'xxxxxxxx-xxxxx', - 'extra.sha1_fingerprint' : 'AD:19:B2:1C:CB:88:70:9B:DB:8E:7E:F5:65:50:13:D6:43:6C:BE:6E', - 'extra.sha256_fingerprint' : '9A:64:73:0B:8A:FA:DE:22:D4:6D:5A:C6:C4:6F:D4:A4:2A:28:FA:41:1E:FF:81:DC:D4:D9:00:FD:78:DF:C4:DD', - 'extra.sha512_fingerprint' : '9A:B7:BD:68:7D:F3:E7:C1:B7:D3:F4:2F:01:B6:C4:77:90:A3:2B:1E:C0:89:F5:08:EC:43:87:35:60:36:D4:87:61:AA:B8:A8:B3:8A:E9:F1:04:AA:5B:67:12:FF:63:D5:14:80:77:6E:8F:7D:C3:E2:3A:F3:13:DF:08:43:6C:B0', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 454110, - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_country' : 'US', - 'extra.subject_locality_name' : 'Hanover', - 'extra.subject_organization_name' : 'Ciena Corporation', - 'extra.subject_state_or_province_name' : 'Maryland', - 'extra.tag' : 'ssl,vpn', - 'extra.validation_level' : 'OV', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 14618, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '34.224.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py deleted file mode 100644 index 42221bda2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py +++ /dev/null @@ -1,136 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssl_freak.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SSL FREAK Vulnerable Servers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssl_freak-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'SSL FREAK Vulnerable Servers', - 'protocol.application': 'https', - "classification.identifier": "ssl-freak", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.browser_error": "x509: unknown error", - "extra.browser_trusted": False, - "extra.cert_expiration_date": "2032-05-05 00:01:19", - "extra.cert_expired": False, - "extra.cert_issue_date": "2012-05-10 00:01:19", - "extra.cert_length": 1024, - "extra.cert_serial_number": "4FAB054F", - "extra.cert_valid": True, - "extra.cipher_suite": "TLS_RSA_WITH_RC4_128_SHA", - "extra.content_type": "text/html", - "extra.freak_cipher_suite": "TLS_RSA_EXPORT_WITH_RC4_40_MD5", - "extra.freak_vulnerable": True, - "extra.handshake": "TLSv1.0", - "extra.http_code": 200, - "extra.http_date": "2018-04-23T13:25:26+00:00", - "extra.http_reason": "OK", - "extra.http_response_type": "HTTP/1.1", - "extra.issuer_common_name": "usg50_B0B2DC2FA69D", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE", - "extra.self_signed": True, - "extra.sha1_fingerprint": "14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2", - "extra.sha256_fingerprint": "57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1", - "extra.sha512_fingerprint": "E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.subject_common_name": "usg50_B0B2DC2FA69D", - "extra.tag": "ssl-freak", - "extra.transfer_encoding": "chunked", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 8447, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.232", - "source.port": 443, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2018-04-23T13:25:21+00:00" - }, - {'__type': 'Event', - 'feed.name': 'SSL FREAK Vulnerable Servers', - 'protocol.application': 'https', - "classification.identifier": "ssl-freak", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.browser_error": "x509: unknown error", - "extra.browser_trusted": False, - "extra.cert_expiration_date": "2029-12-27 00:00:53", - "extra.cert_expired": False, - "extra.cert_issue_date": "2010-01-01 00:00:53", - "extra.cert_length": 1024, - "extra.cert_serial_number": "4B3D3B35", - "extra.cert_valid": True, - "extra.cipher_suite": "TLS_RSA_WITH_RC4_128_SHA", - "extra.content_type": "text/html", - "extra.freak_cipher_suite": "TLS_RSA_EXPORT_WITH_RC4_40_MD5", - "extra.freak_vulnerable": True, - "extra.handshake": "TLSv1.0", - "extra.http_code": 200, - "extra.http_date": "2018-04-23T13:25:29+00:00", - "extra.http_reason": "OK", - "extra.http_response_type": "HTTP/1.1", - "extra.issuer_common_name": "usg20w_C86C870287EC", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE", - "extra.self_signed": True, - "extra.sha1_fingerprint": "14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2", - "extra.sha256_fingerprint": "57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1", - "extra.sha512_fingerprint": "E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.subject_common_name": "usg20w_C86C870287EC", - "extra.tag": "ssl-freak", - "extra.transfer_encoding": "chunked", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 12577, - "source.geolocation.cc": "AT", - "source.geolocation.city": "BADEN", - "source.geolocation.region": "NIEDEROSTERREICH", - "source.ip": "198.51.100.224", - "source.port": 443, - "source.reverse_dns": "198-51-100-224.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2018-04-23T13:25:26+00:00" - }] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py deleted file mode 100644 index 41535e67a..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ssl_poodle.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SSL POODLE Vulnerable Servers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssl_poodle-test-geo.csv", - } -EVENTS = [{'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'ssl-poodle', - 'extra.browser_error': 'x509: unknown error', - 'extra.browser_trusted': False, - 'extra.cert_expiration_date': '2034-06-20 00:00:42', - 'extra.cert_expired': False, - 'extra.cert_issue_date': '2014-06-25 00:00:42', - 'extra.cert_length': 1024, - 'extra.cert_serial_number': '53AA112A', - 'extra.cert_valid': True, - 'extra.cipher_suite': 'TLS_RSA_WITH_RC4_128_SHA', - 'extra.content_type': 'text/html', - 'extra.handshake': 'TLSv1.0', - 'extra.http_code': 200, - 'extra.http_date': '2018-08-08T00:51:44+00:00', - 'extra.http_reason': 'OK', - 'extra.http_response_type': 'HTTP/1.1', - 'extra.issuer_common_name': 'usg20_107BEF394BA5', - 'extra.key_algorithm': 'rsaEncryption', - 'extra.md5_fingerprint': '33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC', - 'extra.self_signed': True, - 'extra.sha1_fingerprint': '04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3', - 'extra.sha256_fingerprint': '16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E', - 'extra.sha512_fingerprint': '0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE', - 'extra.signature_algorithm': 'sha1WithRSAEncryption', - 'extra.ssl_poodle': True, - 'extra.ssl_version': 2, - 'extra.subject_common_name': 'usg20_107BEF394BA5', - 'extra.tag': 'ssl-poodle', - 'extra.transfer_encoding': 'chunked', - 'feed.name': 'SSL POODLE Vulnerable Servers', - 'protocol.application': 'https', - 'source.asn': 65540, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '203.0.113.85', - 'source.port': 8443, - 'source.reverse_dns': 'example.com', - 'time.source': '2018-08-08T00:51:42+00:00', - "time.observation": "2015-01-01T00:00:00+00:00", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - '__type': 'Event', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py deleted file mode 100644 index 7fd5f6ec2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_stun.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_stun-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 5.4, - 'extra.fingerprint': '0xfaedd06e', - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '192.168.0.1', - 'extra.mapped_family': '01', - 'extra.mapped_port': 3243, - 'extra.message_length': 88, - 'extra.message_type': '0101', - 'extra.response_size': 108, - 'extra.software': "Coturn-4.5.1.1 'dan Eider'", - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '192.168.0.1', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 3243, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3478, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 5.4, - 'extra.fingerprint': '0x21128641', - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '51.77.39.195', - 'extra.mapped_family': '01', - 'extra.mapped_port': 45877, - 'extra.message_length': 88, - 'extra.message_type': '0101', - 'extra.response_size': 108, - 'extra.software': "Coturn-4.5.1.1 'dan Eider'", - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '192.168.0.2', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 45877, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3478, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 4.8, - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '192.168.0.3', - 'extra.mapped_family': '01', - 'extra.mapped_port': 16321, - 'extra.message_length': 76, - 'extra.message_type': '0101', - 'extra.response_size': 96, - 'extra.software': "ApolloProxy-1.20.1.28 'sunflower'", - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '188.68.240.32', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 16321, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3478, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py deleted file mode 100644 index 9b7e1fd3d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_synfulknock.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SYNful Knock', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_synfulknock-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '3cfdfec601e4700f6a9a2000080045000034c3780000f706789442099555b869f7ee0050b20800000000000c123e8012200002aa0000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 18885, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'JERSEY CITY', - 'source.geolocation.region' : 'NEW JERSEY', - 'source.ip' : '66.9.0.0', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:18:23+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '90e2baaf0b84700f6a9a200008004500003434100000f2064382d58337d2b8698b720050916200000000000c123e8012200059d50000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 35805, - 'source.geolocation.cc' : 'GE', - 'source.geolocation.city' : 'TBILISI', - 'source.geolocation.region' : 'TBILISI', - 'source.ip' : '213.131.0.0', - 'source.port' : 80, - 'source.reverse_dns' : 'host-213-131-55-210-customer.wanex.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:19:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '90e2bab9cfd4700f6a9a20000800450000340f1d0000ea068bdad5b2e6914a522f360050eb5200000000000c123e801220001b4a0000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 29256, - 'source.geolocation.cc' : 'SY', - 'source.geolocation.city' : 'DAMASCUS', - 'source.geolocation.region' : 'DIMASHQ', - 'source.ip' : '213.178.0.0', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:27:39+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py deleted file mode 100644 index 66408db4c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py +++ /dev/null @@ -1,87 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_telnet.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Telnet', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_telnet-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible Telnet', - "classification.identifier": "open-telnet", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "|MikroTik v6.5|Login:", - "extra.tag": "telnet-alt", - "protocol.application": "telnet", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.145", - "source.port": 5678, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T12:27:34+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible Telnet', - "classification.identifier": "open-telnet", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "|MikroTik v6.45.3 (stable)|Login:", - "extra.tag": "telnet-alt", - "protocol.application": "telnet", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.145", - "source.port": 5678, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T12:27:40+00:00" - }] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py deleted file mode 100644 index 3cf3688f9..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_tftp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open TFTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_tftp-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.57, - 'extra.error': 'Not defined', - 'extra.errormessage': 'Get not supported', - 'extra.opcode': '5', - 'extra.size': 22, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 35067, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.36, - 'extra.error': 'File not found', - 'extra.errorcode': '1', - 'extra.errormessage': 'File not found', - 'extra.opcode': '5', - 'extra.size': 19, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 56709, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.5, - 'extra.error': 'Access violation', - 'extra.errorcode': '2', - 'extra.errormessage': 'Access violation', - 'extra.opcode': '5', - 'extra.size': 21, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 32785, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py deleted file mode 100644 index 396bff1e3..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ubiquiti.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Ubiquiti', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-03-04T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ubiquiti-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 37.0, - 'extra.essid': 'Kachine-Meta-Lidia-Tereixa', - 'extra.firmwarerev': 'XS5.ar2313.v3.5.4494.091109.1459', - 'extra.mac_address': '00156db98c3a', - 'extra.model': 'NS5', - 'extra.radio_name': 'kachine.meta.lidia.tereixa', - 'extra.response_size': 148, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 10001, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 39.0, - 'extra.essid': 'Adana-Mason-Lanikai-Ozaner', - 'extra.firmwarerev': 'XM.ar7240.v5.6.3.28591.151130.1749', - 'extra.mac_address': '00156d7c9188', - 'extra.model': 'LM5', - 'extra.model_full': 'NanoStation Loco M5', - 'extra.radio_name': 'adana.mason.lanikai.ozaner', - 'extra.response_size': 156, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 10001, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 36.25, - 'extra.essid': 'Tailynn-Kadija-Noreen-Dinkar', - 'extra.firmwarerev': 'XW.ar934x.v5.6.5.29033.160515.2108', - 'extra.mac_address': '0418d6000fd5', - 'extra.model': 'P2B-400', - 'extra.model_full': 'PowerBeam M2 400', - 'extra.radio_name': 'tailynn.kadija.noreen.dinkar', - 'extra.response_size': 145, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 10001, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py deleted file mode 100644 index 457ec4425..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_vnc.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible VNC', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_vnc-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible VNC', - "classification.identifier": "open-vnc", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "RFB 003.889", - "extra.product": "Apple remote desktop vnc", - "protocol.application": "vnc", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.53", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T14:51:44+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible VNC', - "classification.identifier": "open-vnc", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "RFB 005.000", - "extra.naics": 517311, - "extra.product": "RealVNC Enterprise v5.3 or later", - "protocol.application": "vnc", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.112", - "source.port": 5678, - "source.reverse_dns": "localhost.localdomain", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T14:51:44+00:00"}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py deleted file mode 100644 index 41ab55e58..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ws_discovery.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-WS-Discovery-Service', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_ws_discovery-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 164.83, - 'extra.error': 'Validation constraint violation: SOAP message expected', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 989, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3702, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 183.6, - 'extra.error': 'Validation constraint violation: missing root element', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 918, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3702, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 197.8, - 'extra.error': 'Validation constraint violation: SOAP message expected', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 989, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3702, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py deleted file mode 100644 index d17482e71..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_xdmcp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer XDMCP", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_xdmcp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.29, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node01.example.com', - 'extra.size': 44, - 'extra.status': 'Linux 3.0.101-100-default', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 177, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.86, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node02.example.com', - 'extra.size': 48, - 'extra.status': 'Linux 2.6.9-103.ELsmp', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 47074, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.57, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node03.example.com', - 'extra.size': 46, - 'extra.status': '1 user, load: 6,5, 6,6, 6,6', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 177, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_special.py b/intelmq/tests/bots/parsers/shadowserver/test_special.py deleted file mode 100644 index abad86cac..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_special.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/special.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Special', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-special-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Professional, Scientific, and Technical Services', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_testdata.py b/intelmq/tests/bots/parsers/shadowserver/test_testdata.py deleted file mode 100644 index 19cbdd7d7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_testdata.py +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-FileCopyrightText: 2017 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import csv -import json -import os -import os.path -import unittest -import pathlib - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot -from intelmq.bots.parsers.shadowserver.parser_json import ShadowserverJSONParserBot - -def csvtojson(csvfile): - datalist = [] - - with open(csvfile) as fop: - reader = csv.DictReader(fop, restval="") - - for row in reader: - datalist.append(row) - - return json.dumps(datalist, indent=4) - -CSVREPORTS = {} -JSONREPORTS = {} -testdata = pathlib.Path(__file__).parent / 'testdata' -for filename in testdata.glob('*.csv'): - EXAMPLE_FILE = filename.read_text() - shortname = filename.stem - CSVREPORTS[shortname] = {"raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": f"2019-01-01-{shortname}-test-test.csv", - } - JSONREPORTS[shortname] = {"raw": utils.base64_encode(csvtojson(filename)), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": f"2019-01-01-{shortname}-test-test.json", - } - - -def generate_feed_function(feedname, reports): - def test_feed(self): - """ Test if no errors happen for feed %s. """ % feedname - self.input_message = reports[feedname] - self.run_bot() - return test_feed - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - -class TestShadowserverJSONParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverJSONParserBot - -for key in CSVREPORTS: - setattr(TestShadowserverParserBot, 'test_feed_%s' % key, generate_feed_function(key, CSVREPORTS)) -for key in JSONREPORTS: - setattr(TestShadowserverJSONParserBot, 'test_feed_%s' % key, generate_feed_function(key, JSONREPORTS)) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv deleted file mode 100644 index cfadcbb2d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","hostname","source","reason","asn","geo","region","city","naics","sic","sector","tag" -"2019-09-04 07:00:19","198.123.245.134",host.local,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,0, -"2019-09-04 07:00:19","198.123.245.171",,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,, -"2019-09-04 07:00:19","198.123.245.0/24",,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license deleted file mode 100644 index 476908eeb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license deleted file mode 100644 index 456b03316..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv deleted file mode 100644 index 117dd6560..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","application","asn","geo","region","city","url","http_host","category","system","detected_since","server","redirect_target","naics","sic","sector","cc_url","family" -"2017-01-16 00:43:48","203.0.113.1",80,"example.com","hacked-webserver-stealrat-t1","http",64496,"AT","WIEN","VIENNA","/header.php","example.com","spam","WINNT","2015-05-09 05:51:12","Microsoft-IIS/7.5",,0,0,,, -"2018-04-09 15:43:41","203.0.113.1","80","","phishing","http","64496","AT","STEIERMARK","GRAZ","/","example.com","phishing","","","","","0","0","",, -"2022-02-07 21:52:29","66.249.0.0",,"66-249-0-0.example.com","magecart",,1234,"US","CALIFORNIA","MOUNTAIN VIEW",,,"stealer",,,,,519130,,"Communications, Service Provider, and Hosting Service","https://lolfree.pw/ads.txt", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv deleted file mode 100644 index 22cfdd69e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model" -"2022-01-10 00:01:42","88.84.0.0","tcp",10443,,"ssl,vpn",2116,"NO","TROMS OG FINNMARK","TROMVIK",517311,,,"Fortinet","firewall","FortiGate" -"2022-01-10 00:01:42","170.231.0.0","tcp",10443,,"ssl,vpn",27843,"PE","METROPOLITANA DE LIMA","LIMA",,,,"Fortinet","firewall","FortiGate" -"2022-01-10 00:01:42","96.60.0.0","tcp",10443,"96-60-66-218.example.com","ssl,vpn",4181,"US","WISCONSIN","MILWAUKEE",517311,,,"Fortinet","firewall","FortiGate" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv deleted file mode 100644 index 3114c26b1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized","http_agent" -"2010-02-10 00:00:00",tcp,192.168.0.1,38055,64512,ZZ,Region,City,node01.example.com,0,,,,,172.16.0.1,443,65534,ZZ,Region,City,node01.example.net,0,"",,,ddos-participant,,,https,,,,,,,,,www.example.com,,,GET,/??=GovpfOoaWYlk,,,,,,,,,,,,,,,,,, -"2010-02-10 00:00:01",udp,192.168.0.2,53,64512,ZZ,Region,City,node02.example.com,0,,,,,172.16.0.2,53,65534,ZZ,Region,City,node02.example.net,0,"",,,ddos-participant,,,dns,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -"2010-02-10 00:00:02",udp,192.168.0.3,53,64512,ZZ,Region,City,node03.example.com,0,,Microsoft,email,Exchange,172.16.0.3,53,65534,ZZ,Region,City,node03.example.net,0,"",,,ddos-participant,,,dns,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license deleted file mode 100644 index 9f58c89ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv deleted file mode 100644 index 17ff15ee6..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","service","start_time","end_time","client_version","username","password","payload_url","payload_md5" -"2021-03-27 00:00:00","tcp","141.98.1.2",30123,209588,"NL","NOORD-HOLLAND","AMSTERDAM",,,,,,,"162.250.1.2",22,26832,"CA","QUEBEC","MONTREAL",,,,"CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.521730Z","2021-03-27T00:00:01.710968Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","5.188.3.4",55690,57172,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"162.250.3.4",22,26832,"CA","QUEBEC","MONTREAL",,,,"CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.520927Z","2021-03-27T00:00:01.670993Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","45.14.5.6",38636,44220,"RO","BIHOR","ORADEA",,,,,,,"82.118.5.6",23,204957,"PL","MAZOWIECKIE","WARSAW",,,,"CAPRICA-EU","telnet-brute-force",,,"telnet",,,,"2021-03-27T00:00:00.781774Z","2021-03-27T00:00:00.857244Z",,,,, -"2021-03-27 00:00:00","tcp","5.188.6.7",56385,49453,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"102.16.6.7",22,37054,"MG","ANTANANARIVO","ANTANANARIVO",,,"Communications, Service Provider, and Hosting Service","CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.163870Z","2021-03-27T00:00:02.896640Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","45.14.7.8",35802,44220,"RO","BIHOR","ORADEA",,,,,,,"82.118.7.8",23,204957,"PL","MAZOWIECKIE","WARSAW",,,,"CAPRICA-EU","telnet-brute-force",,,"telnet",,,,"2021-03-27T00:00:00.781272Z","2021-03-27T00:00:00.856606Z",,,,, -"2021-03-27 00:00:00","tcp","5.188.9.10",33289,49453,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"60.234.9.10",22,9790,"NZ","WELLINGTON","LOWER HUTT",,,"Communications, Service Provider, and Hosting Service","CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.044871Z","2021-03-27T00:00:00.077322Z","b'SSH-2.0-Go'",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license deleted file mode 100644 index 8b9580cf1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv deleted file mode 100644 index dc78c1c1a..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv +++ /dev/null @@ -1,9 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","count" -"2021-03-07 00:00:00","tcp","61.3.1.2",4717,9829,"IN","KERALA","CHENGANNUR",,518210,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","211.218.3.4",4405,4766,"KR","GANGWON-DO","PYEONGCHANG-EUP",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","45.225.5.6",59777,266915,"BR","BAHIA","VITORIA DA CONQUISTA","static-45-225-x-x.example.net",,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","125.122.7.8",8460,4134,"CN","ZHEJIANG SHENG","HANGZHOU",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","219.77.9.10",21867,4760,"HK","HONG KONG","HONG KONG","n219077092196.example.com",517311,,,,,,5555,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","24.137.11.12",4680,14638,"PR","PUERTO RICO","SAN JUAN","dynamic.libertypr.net",,,,,,,5555,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","119.182.13.14",13175,4837,"CN","SHANDONG SHENG","JINING",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","27.198.15.16",56133,4837,"CN","SHANDONG SHENG","JINAN",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv deleted file mode 100644 index f41cb508f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized","http_agent" -"2010-02-10 00:00:00",,192.168.0.1,61234,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.1,88,65534,ZZ,Region,City,node01.example.net,0,,,,ddos,mirai,mirai,mirai,,,121.12.110.28/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:01",,192.168.0.2,61234,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.2,80,65534,ZZ,Region,City,node02.example.net,0,,,,ddos,mirai,mirai,mirai,,,180.97.183.94/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:02",,192.168.0.3,6379,64512,ZZ,Region,City,node03.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.3,,65534,ZZ,Region,City,node03.example.net,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,104.237.138.135/32,32,atk7,10,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv deleted file mode 100644 index a7d0bc4f1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","request","count","bytes","end_time","duration","avg_pps","max_pps" -"2021-03-28 00:00:02",,"107.141.1.2",,7018,"US","CALIFORNIA","VISALIA","107-141-x-x.lightspeed.frsnca.sbcglobal.net",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,"2021-03-28 00:20:22",,, -"2021-03-28 00:00:02",,"74.59.3.4",,5769,"CA","QUEBEC","CHICOUTIMI","modemcablex-x-59-74.mc.videotron.ca",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,"2021-03-28 00:13:50",,, -"2021-03-28 00:00:02",,"65.131.5.6",,209,"US","WYOMING","CASPER","65-131-x-x.chyn.qwest.net",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, -"2021-03-28 00:00:02",,"104.162.7.8",,12271,"US","NEW YORK","KINGSTON","cpe-104-162-x-x.hvc.res.rr.com",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, -"2021-03-28 00:00:02",,"37.120.178.9.10",,197540,"DE","NIEDERSACHSEN","GIFHORN","v22020111328131649.ultrasrv.de",,,,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license deleted file mode 100644 index 8b9580cf1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv deleted file mode 100644 index 0e5b1e5e9..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","device_vendor","device_type","device_model","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized" -"2010-02-10 00:00:00",,172.16.0.1,80,65534,ZZ,Region,City,node01.example.net,0,,,,,192.168.0.1,61234,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,115.238.198.85/32,32,atk0,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:01",,172.16.0.2,43437,65534,ZZ,Region,City,node02.example.net,0,Information,,,,192.168.0.2,61234,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,52.184.50.250/32,32,atk0,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:02",,172.16.0.3,80,65534,ZZ,Region,City,node03.example.net,0,,,,,192.168.0.3,61234,64512,ZZ,Region,City,node03.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,211.99.102.216/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv deleted file mode 100644 index d9448bd83..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","pattern","http_url","http_agent","http_request_method","url_scheme","session_tags","vulnerability_enum","vulnerability_id","vulnerability_class","vulnerability_score","vulnerability_severity","vulnerability_version","threat_framework","threat_tactic_id","threat_technique_id","target_vendor","target_product","target_class","file_md5","file_sha256","request_raw","body_raw" -"2021-08-01 00:24:08","tcp","191.23.45.67",36455,1234,"EE","HARJUMAA","TALLINN","191-23-45-67-host.example.com",518210,"Communications, Service Provider, and Hosting Service",,,,"109.87.65.43",80,5678,"UK","WINDSOR AND MAIDENHEAD","MAIDENHEAD",,518210,,"CAPRICA-EU","http-scan",,,,"3.1.3-dev",,"unknown","/js/ueditor/wwwroot/way-board.cgi",,,,,,,,,,,,,,,,,,,"GET /js/ueditor/wwwroot/way-board.cgi HTTP/1.0rnAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8rnAccept-Encoding: gzip, deflaternAccept-Language: en-US,en;q=0.5rnConnection: closernDnt: 1rnHost: 109.87.65.43rnOrigin: http://109.87.65.43rnReferer: http://109.87.65.43/rnUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3084.400 QQBrowser/9.6.11346.400", -"2021-08-01 05:21:59","tcp","45.67.89.123",58610,12345,"EE","HARJUMAA","TALLINN",,,,,,,"82.41.20.10",8080,23456,"UA","KHARKIVS'KA OBLAST'","KHARKIV",,,,"CAPRICA-EU","http-scan",,,,,,,"/","Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1","GET","http",,,,,,,,,,,,,,,,"R0VUIC8gSFRUUC8xLjENCkhvc3Q6IDgyLjQxLjIwLjEwOjgwODANCkFjY2VwdDogdGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksKi8qO3E9MC44DQpBY2NlcHQtRW5jb2Rpbmc6IGRlZmxhdGUsIGd6aXAsIGlkZW50aXR5DQpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTO3E9MC42LGVuO3E9MC40DQpVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA1LjE7IHJ2OjkuMC4xKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzkuMC4xDQoNCg==", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license deleted file mode 100644 index c1900637f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv deleted file mode 100644 index 174360bbd..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","network","routedspoof","session","nat" -"2021-03-28 00:42:59","tcp","98.191.250.0",,22898,"US","OKLAHOMA","OKLAHOMA CITY","ip-98.191.250.0.atlinkservices.com",517311,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"98.191.250.0/24","received",1112907,"True" -"2021-03-28 01:36:22","tcp","191.7.16.0",,262485,"BR","RIO DE JANEIRO","NOVA IGUACU",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"191.7.16.0/24","received",1112914,"False" -"2021-03-28 02:10:58","tcp","202.53.160.0",,23923,"BD","DHAKA","DHAKA",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"202.53.160.0/24","received",1112931,"True" -"2021-03-28 03:41:51","tcp","87.121.75.0",,134697,"AU","QUEENSLAND","BRISBANE",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"87.121.75.0/24","received",1112953,"True" -"2021-03-28 06:07:17","tcp","189.201.194.0",,262944,"MX","COAHUILA","SALTILLO","ip-189-201-194-0.slw.spectro.mx",,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"189.201.194.0/24","received",1113015,"True" -"2021-03-28 06:59:53","tcp","197.15.48.0",,37671,"TN","TUNIS","TUNIS",,517311,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"197.15.48.0/24","received",1113035,"True" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv deleted file mode 100644 index eb0cbbab9..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id" -"2021-06-07 00:00:00","tcp","190.229.1.2",52955,7303,"AR","BUENOS AIRES","CASEROS",,517311,,,,,"168.63.134.179",16464,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","96.20.3.4",16464,5769,"CA","QUEBEC","LAVAL",,517311,"Communications, Service Provider, and Hosting Service",,,,"52.169.3.4",16464,8075,"IE","DUBLIN","DUBLIN",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","187.222.5.6",55049,8151,"MX","CIUDAD DE MEXICO","MEXICO CITY",,517311,,,,,"168.63.134.179",16464,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","75.84.7.8",64190,20001,"US","CALIFORNIA","NORTH HOLLYWOOD",,517311,"Communications, Service Provider, and Hosting Service",,,,"52.169.7.8",16464,8075,"IE","DUBLIN","DUBLIN",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","24.15.9.10",60373,7922,"US","ILLINOIS","HOMER GLEN",,517311,"Communications, Service Provider, and Hosting Service",,,,"104.40.6.5",16464,8075,"US","CALIFORNIA","SAN FRANCISCO",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","124.101.11.12",50386,4713,"JP","FUKUOKA","FUKUOKA",,517311,"Communications, Service Provider, and Hosting Service",,,,"23.99.101.165",16465,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-64bit","zeroaccess","b68-zeroaccess-2-64bit",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv deleted file mode 100644 index c56d1f218..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2021-06-07 00:00:00","tcp","31.206.1.2",49245,8386,"TR","ANTALYA","KEPEZ",,,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, -"2021-06-07 00:00:00","tcp","177.140.3.4",35919,28573,"BR","SAO PAULO","SAO PAULO",,517312,,,,,"204.95.99.204",443,8075,"US","WASHINGTON","REDMOND",,334111,"Information","MSDCU","caphaw","caphaw","caphaw",,,,"/index.php","3fo8jrthz3y.rgk.cc","Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.6103)",,,"null" -"2021-06-07 00:00:01","tcp","180.190.5.6",49264,132199,"PH","CEBU","MANDAUE",,517311,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, -"2021-06-07 00:00:01","tcp","197.157.7.8",55307,37129,"KE","NAIROBI CITY","NAIROBI",,,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/news/stream.php","40.121.206.97",,,, -"2021-06-07 00:00:01","tcp","174.114.9.10",59000,812,"CA","ONTARIO","OTTAWA",,517311,"Communications, Service Provider, and Hosting Service",,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv deleted file mode 100644 index c5126c843..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id" -"2021-03-04 00:00:00","tcp","190.113.1.2",17409,12252,"PE","METROPOLITANA DE LIMA","LIMA",,,,,,,"178.162.1.2",4455,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service","eset","victorygate.b","victorygate.b",,,, -"2021-03-04 00:00:00","tcp","35.205.9.10",44696,15169,"BE","BRUXELLES-CAPITALE","BRUSSELS","x.x.205.35.bc.googleusercontent.com",519130,"Communications, Service Provider, and Hosting Service",,,,"148.81.9.10",80,1887,"PL","MAZOWIECKIE","WARSAW",,,,,"virut","virut",,,, -"2021-03-04 00:00:00","tcp","35.197.11.12",36968,15169,"US","OREGON","THE DALLES","x.x.197.35.bc.googleusercontent.com",519130,"Communications, Service Provider, and Hosting Service",,,,"148.81.111.11.12",80,1887,"PL","MAZOWIECKIE","WARSAW",,,,,"virut","virut",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv deleted file mode 100644 index 3e85690d8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","infection","family","tag","query_type","query","count" -"2022-01-06 00:00:02","udp","217.110.0.0",29614,8220,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,,"calypso","calypso","msexchange","A","YolkIsh.COM",1 -"2022-01-06 00:00:02","udp","209.66.0.0",46189,40934,"US","VIRGINIA","ASHBURN",,518210,,,,,"orcus","orcus","rat","A","verble.rocks",1 -"2022-01-06 00:00:02","udp","217.110.0.0",3590,8220,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,,"calypso","calypso","msexchange","A","RAwFuNS.COM",1 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license deleted file mode 100644 index 662bb20b7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv deleted file mode 100644 index 4514f248e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2021-03-04 00:00:00","tcp","103.196.1.2",60902,134707,"PH","NUEVA ECIJA","DEL PILAR",,,,,,,"184.105.1.2",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","5.14.3.4",55002,8708,"RO","CONSTANTA","CONSTANTA",,517311,"Communications, Service Provider, and Hosting Service",,,,"184.105.3.4",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","49.145.5.6",31350,9299,"PH","CEBU","CEBU",,517311,,,,,"184.105.5.6",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"disorderstatus.ru",,,, -"2021-03-04 00:00:00","tcp","200.44.7.8",28063,8048,"VE","CARABOBO","VALENCIA",,517311,,,,,"184.105.7.8",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","187.189.9.10",45335,17072,"MX","CHIHUAHUA","JUAREZ",,,,,,,"184.105.9.10",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv deleted file mode 100644 index 23a3cb2b6..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","http_referer_ip","http_referer_port","http_referer_asn","http_referer_geo","http_referer_region","http_referer_city","http_referer_hostname","http_referer_naics","http_referer_sector","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_referer" -"2021-03-04 00:00:02","tcp","178.162.203.211",80,28753,"DE","HESSEN","FRANKFURT AM MAIN","12106.mobapptrack.com",518210,"Communications, Service Provider, and Hosting Service","85.17.31.82",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"kovter","kovter",,,1614816002,"GET /favicon.ico HTTP/1.1","12106.mobapptrack.com","http://12106.mobapptrack.com/click/redirect?feed_id=12106&sub_id=7&q=8A5491983C8FBE7743E2D2C36E45EBC4-18307118D2626C9BD756B3F09D14BB910E381EE4" -"2021-03-04 00:00:11","tcp","59.106.1.2",80,9370,"JP","OSAKA","OSAKA","x.noizm.com",518210,"Communications, Service Provider, and Hosting Service","178.162.1.2",80,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816011,"GET /animalally.com HTTP/1.1","freescanonline.com","http://x.noizm.com/jump.php?u=http://freescanonline.com/animalally.com" -"2021-03-04 00:00:12","tcp","142.250.3.4",80,15169,"US","CALIFORNIA","MOUNTAIN VIEW","x.blogspot.com",519130,"Communications, Service Provider, and Hosting Service","178.162.1.2",80,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,"kovter","kovter",,,1614816012,"GET /getjs?r=0.6393021999392658 HTTP/1.1","rxrtb.bid","http://x.blogspot.com/" -"2021-03-04 00:00:13","tcp","34.232.5.6",80,14618,"US","VIRGINIA","ASHBURN","www.example.com",454110,"Retail Trade","5.79.71.225",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816013,"GET /personalationmall.com HTTP/1.1","freescanonline.com","http://www.example.com/teams/default.asp?u=EKL&t=c&s=lacrosse&p=remote&url=http://freescanonline.com/personalationmall.com" -"2021-03-04 00:01:26","tcp","210.172.7.8",80,2516,"JP","HOKKAIDO","SAPPORO","x.communes.jp",517312,"Communications, Service Provider, and Hosting Service","5.79.1.2",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816086,"GET /raftcomply.com HTTP/1.1","freescanonline.com","http://x.communes.jp/?url=http://freescanonline.com/raftcomply.com" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license deleted file mode 100644 index c1900637f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv deleted file mode 100644 index 016d2f912..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2022-03-02 09:14:19","tcp","2001:448a:1082:4d9b:7491:bf9e:3d5f:a634",49431,7713,"ID","JAKARTA RAYA","JAKARTA",,,,,,,"2001:470:1:332::fe",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM HTTP/1.1","devps.net","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)",,, -"2022-03-02 09:15:10","tcp","2001:448a:1082:4d9b:7491:bf9e:3d5f:a634",49460,7713,"ID","JAKARTA RAYA","JAKARTA",,,,,,,"2001:470:1:332::ef",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM HTTP/1.1","devps.net","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)",,, -"2022-03-02 14:15:10","tcp","2603:8080:b20a:dc00:f06e:8304:71f6:27e2",62932,11427,"US","TEXAS","GARLAND",,517311,"Communications, Service Provider, and Hosting Service",,,,"2001:470:1:332::fe",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /WMoUNCvuKGzdqSCeQcadP1%2f0B%2f3bzpOmyKBU85Z25HVOhvDQUPFl%2fk8uOcLewS%2b1BsuHXalRAOIgGOYYs2igj6UX8FkdCAmDewWPvfDhPD45nwd2tx1lLf2IoIfuOtIpGR6bN5Q6hGpSBgfERqCa0ImHcwfcZ2EdO%2fWvg7R8H6SLcTiuUC0I4pzvlWt1CRLgLdIEU1hZ0nnFHIHhchb6D7ITEgBQ2chQDxy5TJMrGjm4Dac6dKl%2ft5uYhRhSjAHkLLtgrJjsqVtVbelTAkt5kdcqLlO09m1SH%2fvtAb%2fOvR2DbhBss7%2f64DG7g6cAnghNA6JrFn1uW7sw%2bnKH8koKQwzUjdSsbrQAvmg4r0KDDW8Diq64gfDzxFWkzCLOYifc%2fwlinXPCl7aJiNCoieDC1U98RNQg%2f5td4SZmJnDQ2%2f96CPbFeSpCez5WD1rCjrxLj1h2cqzIgkydEWACceWP9ztxc4QaObzEcgOGxbRckWC7H2aaLeT8jaYEYdKi1pwEKChSL3YdEt4ZIb2IFrWwzNaXEpQzFXf07f902OEdI9vVA1ZdEOBPG6rAIkzMdebfprfVyhKEWtrCd3Skg3COUFtRQks5jzG1nv4sVGijTfSgyn6xE9Taka668Nycik6nmHy8Huj3oC01j3tee%2f1Z3eI6tV7lgM5d3uFJ84slRGHUCwMfVozOGmZRwNo%2fz%2bA HTTP/1.1","devps.net","Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license deleted file mode 100644 index 662bb20b7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv deleted file mode 100644 index ccafbab3f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","url","host","ip","asn","geo","region","city","naics","sector","tag","source","sha256","application" -"2022-01-07 00:02:07","http://41.86.0.0:50008/Mozi.m","41.86.0.0","41.86.0.0",37203,"LR","MONTSERRADO","MONROVIA",,,"CVE-2016-10372",,"12013662c71da69de977c04cd7021f13a70cf7bed4ca6c82acbc100464d4b0ef","http" -"2022-01-07 00:03:14","http://42.225.0.0:38173/Mozi.m","42.225.0.0","42.225.0.0",4837,"CN","HENAN SHENG","ZHUMADIAN",517311,,"CVE-2018-10562",,,"http" -"2022-01-07 00:10:26","http://211.52.0.0:53029/Mozi.m","211.52.0.0","211.52.0.0",4766,"KR","CHUNGCHEONGNAM-DO","SAGOK-MYEON",517311,,"CVE-2018-10562",,,"http" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv deleted file mode 100644 index 965d763a3..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","url","host","ip","asn","geo","region","city","naics","sector","source" -"2022-02-01 08:00:07","https://priceless-pare.example.net/Postal-/acec6/","priceless-pare.example.net","172.245.0.0",64512,"US","NEW YORK","BUFFALO",518210,"Communications, Service Provider, and Hosting Service","openphish.com" -"2022-02-01 08:00:07","https://mailyahooattt.example.net/","mailyahooattt.example.net","199.34.0.0",64512,"US","CALIFORNIA","SAN FRANCISCO",,"Professional, Scientific, and Technical Services","openphish.com" -"2022-02-01 08:00:07","https://www.example.net/viewer/vbid-730ec2b1-omsttuer","www.example.net","216.58.0.0",64512,"US","UTAH","DRAPER",519130,"Communications, Service Provider, and Hosting Service","openphish.com" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv deleted file mode 100644 index d5baa730f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","proxy_authenticate","via","server","content_length","transfer_encoding","http_date" -"2010-02-10 00:00:00",192.168.0.1,tcp,3128,node01.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"Squid proxy-caching web server\"\"",,squid/4.10,3741,,"Wed, 10 Feb 2010 00:00:00 GMT" -"2010-02-10 00:00:01",192.168.0.2,tcp,3128,node02.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"00:23:24:43:1c:34\"\"",,,3833,,"Wed, 10 Feb 2010 00:00:01 GMT" -"2010-02-10 00:00:02",192.168.0.3,tcp,3128,node03.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"Proxy\"\"",,,179,,"Wed, 10 Feb 2010 00:00:02 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv deleted file mode 100644 index 4710af974..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","asn","geo","md5","protocol","port","host","bytes_in","bytes_out" -"2022-01-10 00:00:03","40.119.6.228",8075,"US","b575ce6dcce6502a8431db5610135c25","udp",123,"time.windows.com",0,0 -"2022-01-10 00:00:03","8.252.70.126",3356,"US","c0d947f9a8685b0d9f3efdba966389c2","tcp",80,,0,0 -"2022-01-10 00:00:03","52.109.8.22",8075,"US","c0d947f9a8685b0d9f3efdba966389c2","tcp",443,,0,0 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv deleted file mode 100644 index 697cb6209..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","md5hash","request","type","response","family","tag","source" -"2022-01-10 00:00:02","b575ce6dcce6502a8431db5610135c25","time.windows.com","A","40.119.6.228",,, -"2022-01-10 00:00:08","807679198a39c80d3ca07e60fd51b581","time.windows.com","A","40.119.6.228",,, -"2022-01-10 00:00:20","d97e973b9bf073bd3a217425259cea26","client-office365-tas.msedge.net","A","13.107.5.88",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv deleted file mode 100644 index bbfe596a2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","asn","geo","md5","url","user_agent","host","method" -"2022-01-10 00:01:13","23.196.47.89",20940,"US","37514b54e679a5313334e830ad780ec7","http://www.msftncsi.com/ncsi.txt","Microsoft NCSI","www.msftncsi.com","GET" -"2022-01-10 00:01:28","72.21.81.240",15133,"US","37514b54e679a5313334e830ad780ec7","http://www.download.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab","Microsoft-CryptoAPI/6.1","www.download.windowsupdate.com","GET" -"2022-01-10 00:08:24","23.56.4.57",20940,"US","e97ea2820c0d79f3f3ca241d4dcd1060","http://crl.microsoft.com/pki/crl/products/MicTimStaPCA_2010-07-01.crl","Microsoft-CryptoAPI/6.1","crl.microsoft.com","GET" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv deleted file mode 100644 index c0ff0bdf1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","name","model","device","features","device_vendor","device_type","device_model","device_version","device_sector" -"2018-07-26 02:07:16","36.239.124.210","tcp",5555,"36-239-124-210.dynamic-ip.hinet.net","adb",3462,"TW","TAOYUAN COUNTY","TAOYUAN CITY",518210,737415,"hlteuc","SAMSUNG-SM-N900A","hlteatt",,,,,, -"2018-07-26 02:07:16","36.236.108.107","tcp",5555,"36-236-108-107.dynamic-ip.hinet.net","adb",3462,"TW","TAIPEI CITY","TAIPEI",518210,737415,"marlin","Pixel XL","marlin","cmd,shell_v2",,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv deleted file mode 100644 index c5494d458..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","machine_type","afp_versions","uams","flags","server_name","signature","directory_service","utf8_servername","network_address" -"2019-09-04 05:05:53","198.13.34.22","tcp",548,"host.local","afp",6057,"AA","LOCATION","LOCATION",517311,0,"TimeCapsule8,119","AFP3.3,AFP3.2,AFP3.1","DHCAST128,DHX2,SRP,Recon1","SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient","airport-time-capsule-de-jack","4338364e37364442463948350069672d",,"AirPort Time Capsule de jack","198.33.24.165:548,10.0.1.1:548,fe80:0008:0000:0000:6e70:9fff:fed4::548,fe80:0009:0000:0000:6e70:9fff:fed4::548,179.24.24.165 (DNS address)," -"2019-09-04 05:05:56","198.40.27.212","tcp",548,"host.local","afp",6057,"AA","LOCATION","LOCATION",517311,0,"TimeCapsule8,119","AFP3.3,AFP3.2,AFP3.1","DHCAST128,DHX2,SRP,Recon1","SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient","time-capsule-del-jack","433836544b303147463948360069672d",,"Time Capsule del Jack","0.0.0.1:548,10.0.1.1:548,198.33.42.1:548,fe80:000b:0000:0000:dea4:caff:feba::548,fe80:000c:0000:0000:dea4:caff:feba::548,fe80:000d:0000:0000:4c7d:ffff:fec7::548,0.0.0.1 (DNS address)," diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv deleted file mode 100644 index 92f078af7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","channel","message_length","class","method","version_major","version_minor","capabilities","cluster_name","platform","product","product_version","mechanisms","locales","sector" -"2022-01-10 04:32:13","47.103.0.0","tcp",5672,,"amqp",37963,"CN","SHANGHAI SHI","SHANGHAI",518210,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos","rabbit@iZuf63m0nnq9bwf7lhjxrkZ","Erlang/OTP","RabbitMQ","3.3.5","PLAIN AMQPLAIN","en_US", -"2022-01-10 04:32:13","141.95.0.0","tcp",5672,,"amqp",16276,"DE","SAARLAND","SAARBRUCKEN",518210,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to","rabbit@mtk-breizh","Erlang/OTP 24.0.3","RabbitMQ","3.8.19","AMQPLAIN PLAIN","en_US", -"2022-01-10 04:32:13","54.234.0.0","tcp",5672,"ec2-54.234.0.0.compute-1.amazonaws.com","amqp",14618,"US","VIRGINIA","ASHBURN",454110,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to","rabbit@1397a0e9629b","Erlang/OTP 24.2","RabbitMQ","3.9.11","PLAIN AMQPLAIN","en_US", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv deleted file mode 100644 index 9c43f8598..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","machine_name","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,3283,node01.example.com,ard,64512,ZZ,Region,City,0,0,"Macmini (radio)",1006,201.20 -"2010-02-10 00:00:01",192.168.0.2,udp,3283,node02.example.com,ard,64512,ZZ,Region,City,0,0,biuro-rip-org-pl,1006,201.20 -"2010-02-10 00:00:02",192.168.0.3,udp,3283,node03.example.com,ard,64512,ZZ,Region,City,0,0,127.0.0.1,1006,201.20 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv deleted file mode 100644 index 7bd2b20e0..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","size","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,19,node01.example.com,chargen,,64512,ZZ,Region,City,0,0,,74,74.00 -"2010-02-10 00:00:01",192.168.0.2,udp,19,node02.example.com,chargen,,64512,ZZ,Region,City,0,0,,74,74.00 -"2010-02-10 00:00:02",192.168.0.3,udp,19,node03.example.com,chargen,,64512,ZZ,Region,City,0,0,Government,74,74.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv deleted file mode 100644 index 5182817c1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic" -"2017-11-18 08:42:45","198.51.100.103","tcp",4786,"198-51-100-103.example.net","cisco-smart-install",8559,"AT","WIEN","VIENNA",0,0 -"2017-11-18 08:47:54","198.51.100.218","tcp",4786,,"cisco-smart-install",35609,"AT","WIEN","VIENNA",0,0 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv deleted file mode 100644 index 6d72dac53..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","response","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,5683,node01.example.com,coap,2,64512,ZZ,Region,City,0,0,",,",43,2.05 -"2010-02-10 00:00:01",192.168.0.2,udp,5683,node02.example.com,coap,2,64512,ZZ,Region,City,0,0,",,,,,,,,,",113,5.38 -"2010-02-10 00:00:02",192.168.0.3,udp,5683,node03.example.com,coap,1,64512,ZZ,Region,City,0,0,"`EsjAy************************************************************|CoAP RFC 7252 |************************************************************|This server is using the Eclipse Californium (Cf) CoAP framework|published under EPL+EDL: http://www.eclipse.org/californium/||(c) 2014, 2015, 2016 Institute for Pervasive Computing, ETH Zurich and others|************************************************************",454,113.50 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv deleted file mode 100644 index f4074f3ed..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","server_version","couchdb_message","couchdb_version","git_sha","features","vendor","visible_databases","error","error_reason" -"2010-02-10 00:00:00",192.168.0.1,tcp,5984,node01.example.com,couchdb,64512,ZZ,Region,City,0,0,,"CouchDB/1.6.1 (Erlang OTP/18)",Welcome,1.6.1,,,"Ubuntu 16.04",_replicator;_users;test;shops;god,, -"2010-02-10 00:00:01",192.168.0.2,tcp,5984,node02.example.com,couchdb,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service","CouchDB/3.2.1 (Erlang OTP/23)",Welcome,3.2.1,244d428af,"access-ready,partitioned,pluggable-storage-engines,reshard,scheduler","The Apache Software Foundation",,, -"2010-02-10 00:00:02",192.168.0.3,tcp,5984,node03.example.com,couchdb,64512,ZZ,Region,City,0,0,"Retail Trade","CouchDB/3.2.1 (Erlang OTP/20)",Welcome,3.2.1,244d428af,"access-ready,partitioned,pluggable-storage-engines,reshard,scheduler","The Apache Software Foundation",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv deleted file mode 100644 index 5aebed050..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","date","sector" -"2019-09-04 10:44:55","198.123.245.142","tcp",30005,,"cwmp",5678,"AA","LOCATION","LOCATION",517311,0,"HTTP/1.1",200,"OK","text/html","keep-alive",,,"DNVRS-Webs",5678,,"Wed, 04 Sep 2019 07:42:37 GMT", -"2019-09-04 11:06:50","198.123.245.162","tcp",5678,"localhost.localdomain","cwmp",5678,"AA","LOCATION","LOCATION",517311,0,"HTTP/1.1",404,"Not Found","text/html",,,,"RomPager/4.07 UPnP/1.0",,"chunked",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv deleted file mode 100644 index c4bb32e57..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","db2_hostname","servername","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,523,node01.example.com,db2,64512,ZZ,Region,City,0,0,NOWAK_SERWER,node01.example.com,298,14.90 -"2010-02-10 00:00:01",192.168.0.2,udp,523,node02.example.com,db2,64512,ZZ,Region,City,0,0,SPZOZ-DZIEWIN,node02.example.com,298,14.90 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv deleted file mode 100644 index 25e6f11d0..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","source_port","bytes","amplification","method" -"2010-02-10 00:00:00",192.168.0.1,tcp,80,node01.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,,49002,99,2,SYN+ACK:PSH -"2010-02-10 00:00:01",192.168.0.2,tcp,80,node02.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",41200,99,2,SYN+ACK:PSH -"2010-02-10 00:00:02",192.168.0.3,tcp,80,node03.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,,47492,99,2,SYN+ACK:PSH diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv deleted file mode 100644 index 05b807883..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv +++ /dev/null @@ -1,101 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","dns_version","asn","geo","region","city","min_amplification","p0f_genre","p0f_detail","naics","sic","sector" -"2018-04-14 00:14:34","198.51.100.179","udp",53,"198-51-100-189.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.8","udp",53,"198-51-100-111.example.net","openresolver","dnsmasq-2.51",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:34","198.51.100.179","udp",53,"198-51-100-189.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.8","udp",53,"198-51-100-111.example.net","openresolver","dnsmasq-2.51",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.158","udp",53,,"openresolver",,8447,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:37","198.51.100.7","udp",53,"198-51-100-7.example.net","openresolver","9.9.4-rpz2.13269.14-P2",13292,"AT","STEIERMARK","EISENERZ","4.6190",,,0,0, -"2018-04-14 00:14:38","198.51.100.167","udp",53,"198-51-100-167.example.net","openresolver","ZyWALL DNS",6830,"AT","KARNTEN","VILLACH","4.6667",,,0,0, -"2018-04-14 00:14:40","198.51.100.10","udp",53,"198-51-100-10.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:14:41","198.51.100.191","udp",53,"198-51-100-63.example.net","openresolver",,25255,"AT","TIROL","LIENZ","4.6190",,,0,0, -"2018-04-14 00:14:43","198.51.100.25","udp",53,"198-51-100-187.example.net","openresolver","p.4.0",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:54","198.51.100.174","udp",53,"198-51-100-174.example.net","openresolver",,8445,"AT","SALZBURG","SALZBURG","6.4048",,,0,0, -"2018-04-14 00:14:54","198.51.100.181","udp",53,"198-51-100-181.example.net","openresolver","PowerDNS Recursor tele2/sil",8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:54","198.51.100.80","udp",53,"198-51-100-80.example.net","openresolver",,1901,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:14:57","198.51.100.43","udp",53,"198-51-100-43.example.net","openresolver","vi2zcnsat10, Customer DNS",6830,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:14:58","198.51.100.124","udp",53,"198-51-100-124.example.net","openresolver","dnsmasq-2.47",28919,"AT","TIROL","EIBERG","3.8095",,,0,0, -"2018-04-14 00:15:00","198.51.100.60","udp",53,"198-51-100-60.example.net","openresolver",,24992,"AT","VORARLBERG","DORNBIRN","3.4762",,,0,0, -"2018-04-14 00:15:00","198.51.100.201","udp",53,"198-51-100-201.example.net","openresolver",,1853,"AT","STEIERMARK","GRAZ","4.6190",,,0,0, -"2018-04-14 00:15:01","198.51.100.82","udp",53,"198-51-100-82.example.net","openresolver","9.6-ESV-R7-P2",20811,"AT","TIROL","INNSBRUCK","4.6190",,,0,0, -"2018-04-14 00:15:01","198.51.100.105","udp",53,"198-51-100-105.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:02","198.51.100.173","udp",53,"198-51-100-173.example.net","openresolver",,8445,"AT","NIEDEROSTERREICH","WALD","1.3810",,,0,0, -"2018-04-14 00:15:03","198.51.100.82","udp",53,"198-51-100-82.example.net","openresolver","PowerDNS Recursor tele2/sil",1257,"AT","NIEDEROSTERREICH","MODLING","1.3810",,,518111,737401, -"2018-04-14 00:15:05","198.51.100.39","udp",53,,"openresolver",,8437,"AT","VORARLBERG","LUSTENAU","1.3810",,,0,0, -"2018-04-14 00:15:09","198.51.100.33","udp",53,,"openresolver","dnsmasq-2.55",8447,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:15:09","198.51.100.248","udp",53,"198-51-100-248.example.net","openresolver",,39912,"AT","NIEDEROSTERREICH","HOLLABRUNN","3.8095",,,0,0, -"2018-04-14 00:15:10","198.51.100.119","udp",53,"198-51-100-172.example.net","openresolver","dnsmasq-2.62",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:12","198.51.100.135","udp",53,"198-51-100-135.example.net","openresolver","no access.",43848,"AT","NIEDEROSTERREICH","WIESELBURG","3.8095",,,0,0, -"2018-04-14 00:15:15","198.51.100.64","udp",53,"198-51-100-64.example.net","openresolver",,6830,"AT","VORARLBERG","UBERSAXEN","1.3810",,,0,0, -"2018-04-14 00:15:17","198.51.100.80","udp",53,"198-51-100-80.example.net","openresolver",,42473,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:18","198.51.100.60","udp",53,"198-51-100-60.example.net","openresolver","198-51-100-60.example.net",35369,"AT","OBEROSTERREICH","LINZ","1.3810",,,0,0, -"2018-04-14 00:15:21","198.51.100.50","udp",53,,"openresolver","ZyWALL DNS",8447,"AT","STEIERMARK","TAUPLITZ","4.6667",,,0,0, -"2018-04-14 00:15:23","198.51.100.93","udp",53,,"openresolver","Microsoft DNS 6.1.7601 (1DB15D39)",8447,"AT","NIEDEROSTERREICH","SCHWADORF","1.3810",,,0,0, -"2018-04-14 00:15:24","198.51.100.33","udp",53,,"openresolver",,8447,"AT","STEIERMARK","FURSTENFELD","4.6190",,,0,0, -"2018-04-14 00:15:31","198.51.100.45","udp",53,,"openresolver","dnsmasq-2.52",8245,"AT","BURGENLAND","EISENSTADT","1.3810",,,0,0, -"2018-04-14 00:15:34","198.51.100.13","udp",53,"198-51-100-13.example.net","openresolver",,8447,"AT","WIEN","VIENNA","6.4048",,,518210,737415, -"2018-04-14 00:15:36","198.51.100.190","udp",53,,"openresolver",,8447,"AT","BURGENLAND","PINKAFELD","1.3810",,,0,0, -"2018-04-14 00:15:41","198.51.100.104","udp",53,,"openresolver",,21013,"AT","OBEROSTERREICH","LEONDING","2.6667",,,0,0, -"2018-04-14 00:15:42","198.51.100.101","udp",53,"198-51-100-101.example.net","openresolver",,8447,"AT","STEIERMARK","KAINACH BEI VOITSBERG","1.3810",,,0,0, -"2018-04-14 00:15:44","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver",,1901,"AT","OBEROSTERREICH","GMUNDEN","1.3810",,,518210,737415, -"2018-04-14 00:15:46","198.51.100.186","udp",53,"198-51-100-186.example.net","openresolver",,31239,"AT","WIEN","VIENNA","6.4048",,,0,0, -"2018-04-14 00:15:46","198.51.100.197","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","KIRCHDORF AN DER KREMS","4.6190",,,0,0, -"2018-04-14 00:15:49","198.51.100.16","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","LAAKIRCHEN","4.6190",,,0,0, -"2018-04-14 00:15:50","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver",,6830,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","4.6190",,,0,0, -"2018-04-14 00:15:53","198.51.100.7","udp",53,"198-51-100-7.example.net","openresolver",,198950,"AT","TIROL","REUTTE","4.6190",,,518210,737415, -"2018-04-14 00:15:53","198.51.100.177","udp",53,"198-51-100-177.example.net","openresolver","Microsoft DNS 6.1.7601 (1DB1446A)",12605,"AT","OBEROSTERREICH","LINZ","1.3810",,,0,0, -"2018-04-14 00:15:57","198.51.100.47","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","KOTTINGBRUNN","1.3810",,,0,0, -"2018-04-14 00:15:59","198.51.100.95","udp",53,"198-51-100-67.example.net","openresolver","GNS DNS Version 3",57169,"AT","WIEN","VIENNA","1.3810",,,0,0,"Information Technology" -"2018-04-14 00:16:02","198.51.100.104","udp",53,"198-51-100-104.example.net","openresolver",,6830,"AT","OBEROSTERREICH","BAD WIMSBACH-NEYDHARTING","1.3810",,,0,0, -"2018-04-14 00:16:04","198.51.100.106","udp",53,,"openresolver",,8447,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:16:05","198.51.100.204","udp",53,"198-51-100-204.example.net","openresolver",,12605,"AT","OBEROSTERREICH","LINZ","4.6190",,,0,0, -"2018-04-14 00:16:05","198.51.100.111","udp",53,"198-51-100-111.example.net","openresolver",,8447,"AT","OBEROSTERREICH","LINZ","1.3810",,,518210,737415, -"2018-04-14 00:16:06","198.51.100.131","udp",53,"198-51-100-139.example.net","openresolver","p.4.0",25255,"AT","OBEROSTERREICH","TRAUN","1.3810",,,0,0, -"2018-04-14 00:16:10","198.51.100.240","udp",53,"198-51-100-240.example.net","openresolver",,6830,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:13","198.51.100.9","udp",53,"198-51-100-42.example.net","openresolver",,13026,"AT","STEIERMARK","LEIBNITZ","6.4048",,,0,0, -"2018-04-14 00:16:15","198.51.100.231","udp",53,"198-51-100-74.example.net","openresolver",,25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:17","198.51.100.228","udp",53,"198-51-100-227.example.net","openresolver","u.1.0",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:19","198.51.100.152","udp",53,"198-51-100-152.example.net","openresolver",,34694,"AT","TIROL","WORGL","4.6190",,,0,0, -"2018-04-14 00:16:21","198.51.100.88","udp",53,,"openresolver",,8447,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:22","198.51.100.97","udp",53,"198-51-100-97.example.net","openresolver",,8447,"AT","TIROL","INNSBRUCK","1.3810",,,518210,737415, -"2018-04-14 00:16:23","198.51.100.208","udp",53,"198-51-100-208.example.net","openresolver","dnsmasq-2.62",8447,"AT","TIROL","OTZTAL-BAHNHOF","1.3810",,,0,0, -"2018-04-14 00:16:33","198.51.100.113","udp",53,"198-51-100-121.example.net","openresolver","dnsmasq-2.62",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:35","198.51.100.34","udp",53,"198-51-100-44.example.net","openresolver",,25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:37","198.51.100.236","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","ST. ANDRAE-WOERDERN","4.6190",,,0,0, -"2018-04-14 00:16:40","198.51.100.46","udp",53,"198-51-100-46.example.net","openresolver",,21013,"AT","OBEROSTERREICH","LEONDING","2.6667",,,0,0, -"2018-04-14 00:16:45","198.51.100.72","udp",53,"198-51-100-5.example.net","openresolver",,25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:50","198.51.100.179","udp",53,"198-51-100-179.example.net","openresolver",,31125,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:50","198.51.100.107","udp",53,"198-51-100-107.example.net","openresolver","dnsmasq-2.66",18845,"AT","WIEN","VIENNA","1.3810",,,0,0,"Information Technology" -"2018-04-14 00:16:51","198.51.100.188","udp",53,,"openresolver","9.9.4-RedHat-9.9.4-51.el7_4.2",49322,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:54","198.51.100.232","udp",53,"198-51-100-232.example.net","openresolver",,6830,"AT","SALZBURG","SALZBURG","1.3810",,,0,0, -"2018-04-14 00:16:55","198.51.100.102","udp",53,"198-51-100-102.example.net","openresolver","ZyWALL DNS",6830,"AT","KARNTEN","WERNBERG","3.4762",,,0,0, -"2018-04-14 00:16:59","198.51.100.162","udp",53,"198-51-100-162.example.net","openresolver",,8445,"AT","SALZBURG","SALZBURG","1.3810",,,0,0, -"2018-04-14 00:17:00","198.51.100.110","udp",53,"198-51-100-110.example.net","openresolver",,31543,"AT","TIROL","SOLDEN","4.6190",,,0,0, -"2018-04-14 00:17:02","198.51.100.193","udp",53,"198-51-100-193.example.net","openresolver",,8447,"AT","STEIERMARK","FOHNSDORF","1.3810",,,0,0, -"2018-04-14 00:17:06","198.51.100.45","udp",53,"198-51-100-45.example.net","openresolver",,61201,"AT","KARNTEN","KLAGENFURT AM WORTHERSEE","1.3810",,,0,0, -"2018-04-14 00:17:06","198.51.100.219","udp",53,"198-51-100-219.example.net","openresolver",,8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:10","198.51.100.47","udp",53,"198-51-100-47.example.net","openresolver","unsupported query",8412,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.8095",,,0,0, -"2018-04-14 00:17:13","198.51.100.87","udp",53,"198-51-100-87.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:16","198.51.100.121","udp",53,"198-51-100-121.example.net","openresolver",,8447,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:20","198.51.100.115","udp",53,,"openresolver",,8447,"AT","TIROL","WAIDRING","1.3810",,,0,0, -"2018-04-14 00:17:22","198.51.100.235","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","GRIESKIRCHEN","1.3810",,,0,0, -"2018-04-14 00:17:33","198.51.100.154","udp",53,,"openresolver",,8447,"AT","STEIERMARK","GRAZ","4.6190",,,0,0, -"2018-04-14 00:17:36","198.51.100.36","udp",53,"198-51-100-36.example.net","openresolver","BIND",12605,"AT","OBEROSTERREICH","LINZ","4.6190",,,0,0, -"2018-04-14 00:17:38","198.51.100.100","udp",53,"198-51-100-100.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:41","198.51.100.181","udp",53,"198-51-100-181.example.net","openresolver","Microsoft DNS 6.0.6002 (17724D35)",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:41","198.51.100.242","udp",53,"198-51-100-242.example.net","openresolver","Microsoft DNS 6.0.6002 (17724D35)",34767,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.2857",,,0,0, -"2018-04-14 00:17:42","198.51.100.38","udp",53,,"openresolver","PowerDNS Recursor tele2/sil",8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:43","198.51.100.132","udp",53,"198-51-100-132.example.net","openresolver","PowerDNS Recursor tele2/sil",8437,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:17:49","198.51.100.166","udp",53,"198-51-100-166.example.net","openresolver","9.8.4-rpz2+rl005.12-P1",13292,"AT","STEIERMARK","KINDBERG","4.6190",,,0,0, -"2018-04-14 00:17:49","198.51.100.212","udp",53,"198-51-100-212.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:51","198.51.100.225","udp",53,,"openresolver",,8220,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:17:53","198.51.100.161","udp",53,"198-51-100-161.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:54","198.51.100.12","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","LANGENLOIS","1.3810",,,0,0, -"2018-04-14 00:17:55","198.51.100.113","udp",53,"198-51-100-113.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:57","198.51.100.175","udp",53,"198-51-100-175.example.net","openresolver","PowerDNS Recursor tele2/sil",1257,"AT","NIEDEROSTERREICH","MODLING","1.3810",,,518111,737401, -"2018-04-14 00:17:59","198.51.100.107","udp",53,"198-51-100-107.example.net","openresolver",,50719,"AT","STEIERMARK","TIESCHEN","3.8095",,,0,0, -"2018-04-14 00:17:59","198.51.100.51","udp",53,"198-51-100-68.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:18:04","198.51.100.131","udp",53,,"openresolver","ZyWALL DNS",8447,"AT","TIROL","OBERPERFUSS","3.4762",,,0,0, -"2018-04-14 00:18:05","198.51.100.138","udp",53,"198-51-100-138.example.net","openresolver","unsupported query",8412,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.8095",,,0,0, -"2018-04-14 00:18:06","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver","viezcnsat13, Customer DNS",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:18:07","198.51.100.109","udp",53,"198-51-100-109.example.net","openresolver",,1901,"AT","OBEROSTERREICH","LINZ","6.9524",,,518210,737415, -"2018-04-14 00:18:10","198.51.100.205","udp",53,"198-51-100-205.example.net","openresolver",,8437,"AT","WIEN","VIENNA","1.3810",,,0,0, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv deleted file mode 100644 index 535dc4ea8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","http","http_code","http_reason","content_type","server","date","experimental","api_version","arch","go_version","os","kernel_version","git_commit","min_api_version","build_time","pkg_version" -"2010-02-10 00:00:00",192.168.0.1,tcp,2375,node01.example.com,docker,18.05.0-ce,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",HTTP/1.1,200,OK,"application/json; charset=UTF-8","Docker/18.05.0-ce (linux)","Fri, 06 May 2022 14:06:30 GMT",false,1.37,amd64,go1.9.5,linux,3.10.0-514.26.2.el7.x86_64,f150324,1.12,2018-05-09T22:18:36.000000000+00:00, -"2010-02-10 00:00:01",192.168.0.2,tcp,2375,node02.example.com,docker,1.13.1,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,"Docker/1.13.1 (linux)","Fri, 06 May 2022 14:08:07 GMT",false,1.26,amd64,go1.10.3,linux,3.10.0-693.2.2.el7.x86_64,7d71120/1.13.1,1.12,2022-03-02T15:25:43.414574467+00:00,docker-1.13.1-209.git7d71120.el7.centos.x86_64 -"2010-02-10 00:00:02",192.168.0.3,tcp,2375,node03.example.com,docker,18.05.0-ce,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,"application/json; charset=UTF-8","Docker/18.05.0-ce (linux)","Fri, 06 May 2022 14:08:06 GMT",false,1.37,amd64,go1.9.5,linux,3.10.0-514.26.2.el7.x86_64,f150324,1.12,2018-05-09T22:18:36.000000000+00:00, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv deleted file mode 100644 index 60c711973..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_id","device_serial","machine_name","manufacturer","method","http_port","internal_port","video_input_channels","alarm_input_channels","video_output_channels","alarm_output_channels","remote_video_input_channels","mac_address","ipv4_address","ipv4_gateway","ipv4_subnet_mask","ipv4_dhcp_enable","ipv6_address","ipv6_link_local","ipv6_gateway","ipv6_dhcp_enable","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,37810,node01.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",General,IPC,BCS-TIP3401IR-E-V,2.800.106F004.0.R,,6J0E022PAG35073,6J0E022PAG35073,General,client.notifyDevInfo,80,37777,1,0,0,0,0,38:c4:e8:03:b3:e2,192.168.0.1,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::1,fe80::3ac4:e8ff:fe03:b3e2/64,fd09:4ab5:dae9:b078::ff,0,794,794.00 -"2010-02-10 00:00:01",192.168.0.2,udp,37810,node02.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,,Private,HCVR,HCVR,3.210.1.4,,2K0488CPAGS0ND6,HCVR,Private,client.notifyDevInfo,80,37777,3,0,0,0,9,3c:ef:8c:18:a5:07,192.168.0.2,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::2,fe80::3eef:8cff:fe18:a507/64,fd09:4ab5:dae9:b078::ff,,761,761.00 -"2010-02-10 00:00:02",192.168.0.3,udp,37810,node03.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",General,HCVR,BCS-XVR0401-IV,4.000.0000002.11,,5L034FAPAZA0E30,XVR,General,client.notifyDevInfo,80,37777,4,0,0,0,0,38:c4:e8:02:74:da,192.168.0.3,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::3,fe80::3ac4:e8ff:fe02:74da/64,fd09:4ab5:dae9:b078::ff,,711,711.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv deleted file mode 100644 index c681a8595..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","ok","name","cluster_name","http_code","build_hash","build_timestamp","build_snapshot","lucene_version","tagline","sector" -"2010-02-10 00:00:00",192.168.0.1,tcp,9200,node01.example.com,elasticsearch,2.3.5,64512,ZZ,Region,City,0,0,,"Red Skull",elasticsearch,,90f439ff60a3c0f497f91663701e64ccd01edbb4,2016-07-27T10:36:52Z,false,5.5.0,"You Know, for Search","Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:01",192.168.0.2,tcp,9200,node02.example.com,elasticsearch,7.17.0,64512,ZZ,Region,City,0,0,,allinonepod,docker-cluster,,bee86328705acaa9a6daede7140defd4d9ec56bd,,false,8.11.1,"You Know, for Search","Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,tcp,9200,node03.example.com,elasticsearch,7.15.0,64512,ZZ,Region,City,0,0,,f547c2952610,docker-cluster,,79d65f6e357953a5b3cbcc5e2c7c21073d89aa29,,false,8.9.0,"You Know, for Search","Communications, Service Provider, and Hosting Service" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv deleted file mode 100644 index 4e375a9b4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv +++ /dev/null @@ -1,8 +0,0 @@ -"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","sector","version","servername","url" -"2021-05-14 00:11:30","12.237.1.2",443,"afs-exch-cas2.xxx.com","exchange;cve-2021-26855",7018,"US","CALIFORNIA","TURLOCK",517311,,"Communications, Service Provider, and Hosting Service","15.2.721","AFS-EXCH2019", -"2021-05-14 00:11:37","98.153.3.4",443,"rrcs-98-153-x-x.west.biz.rr.com","exchange;webshell",20001,"US","CALIFORNIA","LOS ANGELES",517311,,"Communications, Service Provider, and Hosting Service","15.0.847","SSAMAIL", -"2021-05-14 00:11:38","206.210.5.6",443,"webmail.xxx.com","exchange;webshell",17054,"US","PENNSYLVANIA","PITTSBURGH",518210,,,"15.0.1178","OMNYXEXCH02", -"2021-05-14 00:11:38","12.33.7.8",443,"mail.xxx.org","exchange;cve-2021-26855",7018,"US","ARKANSAS","LITTLE ROCK",921120,,"Communications, Service Provider, and Hosting Service","15.1.2176","MHASVR02", -"2021-05-14 00:11:38","41.204.9.10",443,"mail.xxx.mg","exchange;cve-2021-26855",21042,"MG","ANTANANARIVO","ANTANANARIVO",,,,,"SABMHQE0232", -"2021-05-14 00:11:38","62.33.11.12",443,,"exchange;cve-2021-26855",20485,"RU","ALTAYSKIY KRAY","BARNAUL",,,,"15.2.659","PV-SRV04", -"2021-05-14 00:11:43","199.33.13.14",443,"mail.xxx.tv","exchange;cve-2021-26855",26481,"US","CALIFORNIA","LOS ANGELES",,,,"15.1.1779","MAIL", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv deleted file mode 100644 index 912e73d84..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","auth_tls_response","auth_ssl_response","tlsv13_support","tlsv13_cipher","jarm","device_vendor","device_type","device_model","device_version","device_sector" -"2019-03-06 06:37:00","61.126.3.70","tcp",21,"arcus-net.co.jp","ftp",4713,"JP","TOKYO","TOKYO",517311,737401,"220 FTP Server ready.|","TLSv1.2","TLS_RSA_WITH_AES_128_CBC_SHA",2048,"*.bizmw.com","GlobalSign Organization Validation CA - SHA256 - G2","Jan 14 08:04:50 2015 GMT","Jan 14 08:04:50 2020 GMT","D9:98:3F:2E:F9:D1:BE:9A:10:1E:DE:51:2C:C1:DF:01:18:0A:20:65","1121DC7421AB7924C3B1D396AEA3707E9E29",2,"sha256WithRSAEncryption","rsaEncryption","NTT Communications Corporation",,"JP","Tokyo","Minato-ku",,,,,,,,"GlobalSign nv-sa",,"BE",,,,,,,,,,"27:4A:8A:3A:A7:DF:82:D0:43:03:0E:6F:48:30:30:C9:24:77:11:1A:08:EF:F7:B9:74:0C:CE:40:87:03:D2:51","E5:93:8B:72:84:0F:35:52:8E:7A:6C:E3:EF:36:90:4C:F2:86:A7:4D:B2:DD:C0:C6:23:83:18:EF:DD:86:34:92:91:57:22:29:75:45:71:8B:3A:CD:F1:27:A9:CA:5F:70:5E:AC:15:A5:E6:63:FD:6F:BB:C5:E2:45:99:73:E9:E6","D1:A7:BC:96:78:1D:16:D0:24:A8:62:7C:3A:95:5A:4A","N","N","N","OV","234 AUTH TLS successful",,,,,,,,, -"2019-03-06 06:37:00","62.48.156.65","tcp",21,"dial-62-48-156-65.ptprime.net","ftp",15525,"PT","LISBOA","FRIELAS",0,0,"220-================================================================| PT Empresas| Acesso Reservado| Acesso nao autorizado punido por lei: 109/91; 67/98| ----------------------------------------------------------------| HENNES & MAURITZ LDA - 149093| SITE: PT303 - Cascais Shopping| MORADA: | NIR: EWS1822940| ================================================================|220 FTP server ready, 1 active clients of 4 simultaneous clients allowed.|",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"500 Syntax error, command unrecognized.","500 Syntax error, command unrecognized.",,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv deleted file mode 100644 index 26f8ccbcf..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","version","asn","geo","region","city","naics","sic","server_type","clusterid","total_disk","used_disk","free_disk","livenodes","namenodeaddress","volumeinfo" -"2017-09-13 02:06:05","199.116.235.200",50070,,"2.7.3, rbaa91f7c6bc9cb92be5982de4719c1c8af91ccff",15296,"CA","ALBERTA","CALGARY",0,0,"namenode","CID-64471a53-60cb-4302-9832-92f321f111fe",41567956992,53248,25160089600,"edmonton:50010",, -"2017-09-13 02:07:48","104.43.235.92",50075,,"2.7.1.2.4.0.0-169",8075,"US","IOWA","DES MOINES",334111,357101,"datanode","CID-771bae52-9e4f-4ec4-bc1a-c867585751f0",,,,,"sandbox.hortonworks.com","/hadoop/hdfs/data/current" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv deleted file mode 100644 index a7e3eb707..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","http_date" -"2018-04-19 00:02:26","75.74.78.113","tcp",8080,"c-75-74-78-113.hsd1.fl.comcast.net","http",7922,"US","FLORIDA","MIAMI",518111,737401,"HTTP/1.1",200,"OK","text/html",,,,"lighttpd",,"chunked","Thu, 19 Apr 2018 00:02:28 GMT" -"2018-04-19 00:02:26","88.162.174.130","tcp",8080,"sto95-3-88-162-174-130.fbx.proxad.net","http",12322,"FR",,"SAINT-OUEN-LAUMONE",518210,737415,"HTTP/1.1",200,"OK","text/html",,,,,17729,,"Thu, 19 Apr 2018 02:02:28 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv deleted file mode 100644 index b1f2330f1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","proxy_authenticate","via","server","content_length","transfer_encoding","http_date" -"2010-02-10 00:00:00",192.168.0.1,tcp,3128,node01.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,,,,,"Wed, 10 Feb 2010 00:00:00 GMT" -"2010-02-10 00:00:01",192.168.0.2,tcp,3128,node02.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,"HTTP/1.1 s_proxy_den1",,,,"Wed, 10 Feb 2010 00:00:01 GMT" -"2010-02-10 00:00:02",192.168.0.3,tcp,3128,node03.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,"HTTP/1.1 s_proxy_yvr",,,,"Wed, 10 Feb 2010 00:00:02 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv deleted file mode 100644 index 195342533..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","http_date","version","build_date","detail" -"2010-02-10 00:00:00",192.168.0.1,tcp,8080,node01.example.com,"basic-auth,http",64512,ZZ,Region,City,0,0,HTTP/1.1,401,Unauthorized,"text/html; charset=utf-8",,"Basic realm=\"\"OpenWebif\"\"",TWISTED_SESSION=5473ad3faa3de66685fb3a53bffb390b4fcec2039893009a06caf38e1bec8aa8,TwistedWeb/19.7.0,149,,"Wed, 10 Feb 2010 00:00:00 GMT",,, -"2010-02-10 00:00:01",192.168.0.2,tcp,80,node02.example.com,"basic-auth,http",64512,ZZ,Region,City,0,0,HTTP/1.1,401,Unauthorized,"text/html; charset=utf-8",,"Basic realm=\"\"OpenWebif\"\"",TWISTED_SESSION=d2460d37b7fdbdd6c27dd74423ead5704e553d4f2c230672313edc5602059e33,TwistedWeb/19.7.0,149,,"Wed, 10 Feb 2010 00:00:01 GMT",,, -"2010-02-10 00:00:02",192.168.0.3,tcp,443,node03.example.com,git-config-file,64512,ZZ,Region,City,0,0,,,,,,,,,,,"Wed, 10 Feb 2010 00:00:02 GMT",,,"repositoryformatversion = 0;filemode = false;bare = false;logallrefupdates = true;symlinks = false;ignorecase = true" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv deleted file mode 100644 index d327f1f3b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_id","response_size","raw_response" -2022-03-02 00:34:22,192.168.0.1,tcp,502,host1.example.net,modbus,64512,ZZ,REGION,CITY,0,0,Sector,Vendor 1,device_type,device_model,device_version,0,5,dGVzdDE= -2022-03-02 00:34:22,192.168.0.2,tcp,502,host2.example.net,modbus,64513,ZZ,REGION,CITY,0,0,Sector,Vendor 2,device_type,device_model,device_version,0,5,dGVzdDI= -2022-03-02 00:34:22,192.168.0.3,tcp,502,host3.example.net,modbus,64514,ZZ,REGION,CITY,0,0,Sector,Vendor 3,device_type,device_model,device_version,0,5,dGVzdDM= diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv deleted file mode 100644 index 87a98157f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv +++ /dev/null @@ -1,96 +0,0 @@ -"timestamp","ip","port","hostname","tag","ipmi_version","asn","geo","region","city","none_auth","md2_auth","md5_auth","passkey_auth","oem_auth","defaultkg","permessage_auth","userlevel_auth","usernames","nulluser","anon_login","error","deviceid","devicerev","firmwarerev","version","manufacturerid","manufacturername","productid","productname","naics","sic","sector" -"2016-07-24 00:09:42","198.51.100.4",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.182",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:42","198.51.100.4",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.182",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.176",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.221",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:44","198.51.100.176",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:45","198.51.100.174",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:45","198.51.100.167",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:46","198.51.100.60",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:47","198.51.100.7",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:48","198.51.100.24",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.86",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.231",623,,"ipmi","1.5",20686,"DE","BAYERN","HAPPURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.197",623,,"ipmi","2.0",3320,"DE","BERLIN","BERLIN","no","no","yes","yes","yes","default","enabled","enabled","yes","no","yes",,,,,,,,,,541690,874899, -"2016-07-24 00:09:49","198.51.100.87",623,,"ipmi","1.5",3209,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:49","198.51.100.6",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:51","198.51.100.193",623,,"ipmi","2.0",15598,"DE","BAYERN","NUREMBERG","no","yes","yes","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:51","198.51.100.63",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:52","198.51.100.179",623,,"ipmi","2.0",3320,"DE","BAYERN","DENKLINGEN","no","no","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,518210,737415, -"2016-07-24 00:09:53","198.51.100.112",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:53","198.51.100.189",623,,"ipmi","2.0",30134,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0,"Communications" -"2016-07-24 00:09:54","198.51.100.44",623,"198-51-100-44.example.net","ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","yes","yes","no","no","default","disabled","disabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:54","198.51.100.215",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.231",623,"198-51-100-231.example.net","ipmi","2.0",6805,"DE","HAMBURG","HAMBURG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.234",623,,"ipmi","2.0",31103,"DE","THURINGEN","ERFURT","no","no","yes","no","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.165",623,,"ipmi","2.0",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:56","198.51.100.170",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:56","198.51.100.66",623,,"ipmi","2.0",41412,"DE","BAYERN","REGENSBURG","no","yes","yes","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:56","198.51.100.150",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:57","198.51.100.222",623,,"ipmi","2.0",34309,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:57","198.51.100.19",623,,"ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:58","198.51.100.83",623,,"ipmi","1.5",3209,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:00","198.51.100.61",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:00","198.51.100.94",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:01","198.51.100.242",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:03","198.51.100.251",623,,"ipmi","2.0",553,"DE","BADEN-WURTTEMBERG","HEIDELBERG","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:03","198.51.100.41",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:04","198.51.100.160",623,"198-51-100-160.example.net","ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:04","198.51.100.243",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.190",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.29",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.224",623,,"ipmi","2.0",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:06","198.51.100.143",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","HEMER","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.120",623,,"ipmi","2.0",13003,"DE","SACHSEN","LEIPZIG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.196",623,,"ipmi","1.5",20686,"DE","BAYERN","HAPPURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.123",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.122",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.192",623,,"ipmi","2.0",34171,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,518210,737415, -"2016-07-24 00:10:08","198.51.100.146",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.127",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:09","198.51.100.112",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:09","198.51.100.45",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:09","198.51.100.46",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","NEUSS","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:10","198.51.100.202",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:11","198.51.100.6",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:11","198.51.100.34",623,,"ipmi","2.0",3320,"DE","HESSEN","LEUN","no","yes","yes","no","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,518210,737415, -"2016-07-24 00:10:12","198.51.100.210",623,,"ipmi","2.0",3320,"DE","BADEN-WURTTEMBERG","AALEN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,541690,874899, -"2016-07-24 00:10:12","198.51.100.97",623,,"ipmi","2.0",42730,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:12","198.51.100.172",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.20",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.181",623,,"ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","yes","yes","no","no","default","disabled","disabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.244",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.85",623,,"ipmi","2.0",34309,"DE","HESSEN","FRANKFURT AM MAIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.150",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.154",623,,"ipmi","2.0",196763,"DE","SAARLAND","ST. INGBERT","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.83",623,,"ipmi","2.0",31342,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.6",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.228",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.150",623,,"ipmi","2.0",44066,"DE","BAYERN","MUNICH","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:15","198.51.100.71",623,,"ipmi","2.0",44066,"DE","BAYERN","MUNICH","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:15","198.51.100.239",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:17","198.51.100.46",623,"198-51-100-53.example.net","ipmi","2.0",29083,"DE","BRANDENBURG","MAHLOW","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:17","198.51.100.78",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:18","198.51.100.164",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,812990,489999, -"2016-07-24 00:10:18","198.51.100.142",623,,"ipmi","2.0",34568,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:18","198.51.100.85",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.173",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.180",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.119",623,,"ipmi","2.0",12843,"DE","RHEINLAND-PFALZ","SPEYER","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.183",623,,"ipmi","1.5",12348,"DE","BAYERN","NUREMBERG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:20","198.51.100.108",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:20","198.51.100.221",623,"198-51-100-156.example.net","ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:21","198.51.100.200",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.162",623,,"ipmi","1.5",30766,"DE","HESSEN","BENSHEIM","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.140",623,,"ipmi","2.0",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.121",623,,"ipmi","2.0",34549,"DE","HESSEN","FRANKFURT AM MAIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:22","198.51.100.33",623,,"ipmi","2.0",47215,"DE","NORDRHEIN-WESTFALEN","GUTERSLOH","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:22","198.51.100.203",623,,"ipmi","2.0",201011,"DE","BAYERN","NUREMBERG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:23","198.51.100.16",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:24","198.51.100.166",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","no","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.135",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.154",623,"198-51-100-154.example.net","ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.237",623,,"ipmi","2.0",12586,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.45",623,,"ipmi","2.0",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv deleted file mode 100644 index a585db6eb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv +++ /dev/null @@ -1,2 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","ipp_version","cups_version","printer_uris","printer_name","printer_info","printer_more_info","printer_make_and_model","printer_firmware_name","printer_firmware_string_version","printer_firmware_version","printer_organization","printer_organization_unit","printer_uuid","printer_wifi_ssid","device_vendor","device_type","device_model","device_version","device_sector" -"2020-06-08 11:30:14","123.45.67.89","tcp",631,"some.host.com","ipp",12345,"AA","REGION","CITY",517311,0,"IPP/2.1","CUPS/2.0","ipp://123.45.67.89:631/ipp/print","NPI3F0D22","HP Color LaserJet MFP M277dw","http://123.45.67.89:631/hp/device/info_config_AirPrint.html?tab=Networking&menu=AirPrintStatus","HP Color LaserJet MFP M277dw",20191203,20191203,20191203,"org","unit","urn:uuid:456e4238-4a44-4643-4c42-10e1813f0a18","wifissid",,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license deleted file mode 100644 index 476908eeb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv deleted file mode 100644 index cef6b027c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","initiator_spi","responder_spi","next_payload","exchange_type","flags","message_id","next_payload2","domain_of_interpretation","protocol_id","spi_size","notify_message_type" -"2019-09-04 00:17:25","198.123.245.42","udp",500,"example.local","isakmp-vulnerable",5678,"AA","LOCATION","LOCATION",517311,0,"3e35c70729dfedef","253acab7cbfda607",11,05,00,00000000,00,00,,0,14 -"2019-09-04 00:17:28","198.123.245.67","udp",500,"example.local","isakmp-vulnerable",20255,"AA","LOCATION","LOCATION",0,0,"3e35c70729dfedef","b274460e7adc1bf0",11,05,00,00000000,00,00,,0,14 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv deleted file mode 100644 index ab71b9a15..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","http","http_code","http_reason","content_type","server","date","major","minor","git_version","git_commit","git_tree_state","build_date","go_version","compiler","platform","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,6443,node01.example.com,kubernetes,,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:13 GMT",1,20,v1.20.13,2444b3347a2c45eb965b182fb836e1f51dc61b70,clean,2021-11-17T13:00:29Z,go1.15.15,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:01",192.168.0.2,tcp,6443,node02.example.com,kubernetes,,64512,ZZ,Region,City,0,0,"Retail Trade",HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:12 GMT",1,23,v1.23.3+e419edf,6f5a5295923a614a4202a7ad274b38b69f9ca8c0,clean,2022-02-25T06:26:46Z,go1.17.5,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:02",192.168.0.3,tcp,6443,node03.example.com,kubernetes,,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:12 GMT",1,16+,v1.16.9-aliyun.1,4f7ea78,,2020-05-08T07:29:59Z,go1.13.9,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv deleted file mode 100644 index 54121fd3b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","size","configuration_naming_context","current_time","default_naming_context","dns_host_name","domain_controller_functionality","domain_functionality","ds_service_name","forest_functionality","highest_committed_usn","is_global_catalog_ready","is_synchronized","ldap_service_name","naming_contexts","root_domain_naming_context","schema_naming_context","server_name","subschema_subentry","supported_capabilities","supported_control","supported_ldap_policies","supported_ldap_version","supported_sasl_mechanisms","amplification" -"2010-02-10 00:00:00",192.168.0.1,tcp,389,node01.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",,"DC=ad,DC=example,DC=com",node01.example.com,7,,"CN=Configuration,DC=ad,DC=example,DC=com",2,,,,node01.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5, -"2010-02-10 00:00:01",192.168.0.2,tcp,389,node02.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",20220821124435.0Z,"DC=ad,DC=example,DC=com",node02.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,25029662,TRUE,TRUE,node02.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5, -"2010-02-10 00:00:02",192.168.0.3,tcp,389,node03.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",20220821124539.0Z,"DC=ad,DC=example,DC=com",node03.example.com,,,"CN=Configuration,DC=ad,DC=example,DC=com",,,,,node03.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv deleted file mode 100644 index 3cd5021c5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","size","configuration_naming_context","current_time","default_naming_context","dns_host_name","domain_controller_functionality","domain_functionality","ds_service_name","forest_functionality","highest_committed_usn","is_global_catalog_ready","is_synchronized","ldap_service_name","naming_contexts","root_domain_naming_context","schema_naming_context","server_name","subschema_subentry","supported_capabilities","supported_control","supported_ldap_policies","supported_ldap_version","supported_sasl_mechanisms","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,389,node01.example.com,ldap-udp,64512,ZZ,Region,City,0,0,3038,"CN=Configuration,DC=ad,DC=example,DC=com",20220821044533.0Z,"DC=ad,DC=example,DC=com",node01.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,222537,TRUE,TRUE,node01.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5,58.42 -"2010-02-10 00:00:01",192.168.0.2,udp,389,node02.example.com,ldap-udp,64512,ZZ,Region,City,0,0,3062,"CN=Configuration,DC=ad,DC=example,DC=com",20220821044948.0Z,"DC=ad,DC=example,DC=com",node02.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,1478714,TRUE,TRUE,node02.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5,58.88 -"2010-02-10 00:00:02",192.168.0.3,udp,389,node03.example.com,ldap-udp,64512,ZZ,Region,City,0,0,36,"CN=Configuration,DC=ad,DC=example,DC=com",,"DC=ad,DC=example,DC=com",node03.example.com,,,"CN=Configuration,DC=ad,DC=example,DC=com",,,,,node03.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,,,,,0.69 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv deleted file mode 100644 index 4a97121e7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","mdns_name","mdns_ipv4","mdns_ipv6","services","workstation_name","workstation_ipv4","workstation_ipv6","workstation_info","http_name","http_ipv4","http_ipv6","http_ptr","http_info","http_target","http_port","spotify_name","spotify_ipv4","spotify_ipv6","opc_ua_discovery" -"2010-02-10 00:00:00",192.168.0.1,udp,5353,node01.example.com,mdns,64512,ZZ,Region,City,0,0,,,,"_smb._tcp.local.; _device-info._tcp.local.; _http._tcp.local.; _dacp._tcp.local.;",,192.168.0.1,fd09:4ab5:dae9:b078::1,,,192.168.0.1,fd09:4ab5:dae9:b078::1,,,,,,,, -"2010-02-10 00:00:01",192.168.0.2,udp,5353,node02.example.com,mdns,64512,ZZ,Region,City,0,0,,,,_home-assistant._tcp.local.;,,192.168.0.2,fd09:4ab5:dae9:b078::2,,,192.168.0.2,fd09:4ab5:dae9:b078::2,,,,,,,, -"2010-02-10 00:00:02",192.168.0.3,udp,5353,node03.example.com,"mdns,iot",64512,ZZ,Region,City,0,0,,,,"_webdav._tcp.local.; _adisk._tcp.local.; _smb._tcp.local.; _http._tcp.local.; _dacp._tcp.local.; _afpovertcp._tcp.local.; _device-info._tcp.local.;",,192.168.0.3,fd09:4ab5:dae9:b078::3,,snmeijer.local.,192.168.0.3,fd09:4ab5:dae9:b078::3,snmeijer._http._tcp.local.,"\"\"vendor=Synology\"\" \"\"model=DS218+\"\" \"\"serial=17A0PCN482002\"\" \"\"version_major=6\"\" \"\"version_minor=2\"\" \"\"version_build=25556\"\" \"\"admin_port=5000\"\" \"\"secure_admin_port=5001\"\" \"\"mac_address=00:11:32:80:fd:b5\"\"",snmeijer.local.,5000,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv deleted file mode 100644 index 6a1d445e7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","pid","pointer_size","uptime","time","curr_connections","total_connections","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,50260,node01.example.com,memcached,1.4.15,64512,ZZ,Region,City,0,0,1010,64,32908114,"2022-08-21 10:34:06",243,6106,"Communications, Service Provider, and Hosting Service",1144,81.71 -"2010-02-10 00:00:01",192.168.0.2,udp,11211,node02.example.com,memcached,1.4.13,64512,ZZ,Region,City,0,0,5316,64,9618498,"2022-08-21 10:39:21",9,2962,"Communications, Service Provider, and Hosting Service",1053,75.21 -"2010-02-10 00:00:02",192.168.0.3,udp,11211,node03.example.com,memcached,1.2.6,64512,ZZ,Region,City,0,0,1460,32,1375159,"2022-08-21 10:39:39",2,534,,442,31.57 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv deleted file mode 100644 index 1228dcfc6..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv +++ /dev/null @@ -1,11 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","gitversion","sysinfo","opensslversion","allocator","javascriptengine","bits","maxbsonobjectsize","ok","visible_databases","sector" -"2016-07-24 00:40:07","198.51.100.203","tcp",27017,"198-51-100-203.example.net","mongodb","2.4.5",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"a2ddc68ba7c9cee17bfe69ed840383ec3506602b","Linux ip-198-51-100-100 198.51.100.103-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49",,"tcmalloc","V8",64,16777216,1,"local | countly | admin", -"2016-07-24 00:40:07","198.51.100.42","tcp",27017,"198-51-100-208.example.net","mongodb","2.6.12",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"d73c92b1c85703828b55c2916a5dd4ad46535f6a","Linux build5.ny.cbi.10gen.cc 2.6.32-431.3.1.el6.x86_64 #1 SMP Fri Jan 3 21:39:27 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49",,"tcmalloc","V8",64,16777216,1,"none visible","Information Technology" -"2016-07-24 00:40:07","198.51.100.225","tcp",27017,"198-51-100-225.example.net","mongodb","3.0.6",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,"1ef45a23a4c5e3480ac919b28afcba3c615488f2","Linux ip-198-51-100-100 3.4.43-43.43.amzn1.x86_64 #1 SMP Mon May 6 18:04:41 UTC 2013 x86_64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.0-fips 29 Mar 2010","tcmalloc","V8",64,16777216,1,"bluu | local","Communications" -"2016-07-24 00:40:07","198.51.100.144","tcp",27017,"198-51-100-144.example.net","mongodb","2.2.2",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"d1b43b61a5308c4ad0679d34b262c5af9d664267","Linux ip-198-51-100-100 198.51.100.252-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49",,,,64,16777216,1,"errbit_production | DELETED_BECAUSE_YOU_DIDNT_PASSWORD_PROTECT_YOUR_MONGODB | admin | local", -"2016-07-24 00:40:07","198.51.100.68","tcp",27017,,"mongodb","3.2.6",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,"05552b562c7a0b3143a729aaa0838e558dc49b25","deprecated",,"tcmalloc","mozjs",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.101","tcp",27017,,"mongodb","3.0.9",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,"20d60d3491908f1ae252fe452300de3978a040c7","Linux ip-198-51-100-100 3.13.0-24-generic #46-Ubuntu SMP Thu Apr 10 19:11:08 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.1f 6 Jan 2014","tcmalloc","V8",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.53","tcp",27017,"198-51-100-162.example.net","mongodb","3.2.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"05552b562c7a0b3143a729aaa0838e558dc49b25","deprecated",,"tcmalloc","mozjs",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.206","tcp",27017,"198-51-100-206.example.net","mongodb","2.4.10",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"e3d78955d181e475345ebd60053a4738a4c5268a","Linux bs-linux32.10gen.cc 198.51.100.34-2.fc8xen #1 SMP Fri Feb 15 12:39:36 EST 2008 i686 BOOST_LIB_VERSION=1_49",,"system","V8",32,16777216,1,"sharelatex | test1 | local | tmp | lococms_production", -"2016-07-24 00:40:10","198.51.100.157","tcp",27017,"198-51-100-157.example.net","mongodb","2.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"nogitversion","Linux biber 2.6.32-5-amd64 #1 SMP Mon Feb 25 00:26:11 UTC 2013 i686 BOOST_LIB_VERSION=1_49",,,,32,16777216,1,"none visible", -"2016-07-24 00:40:10","198.51.100.173","tcp",27017,"198-51-100-173.example.net","mongodb","2.6.12",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"nogitversion","FreeBSD 101amd64-default-job-24 10.1-RELEASE-p33 FreeBSD 10.1-RELEASE-p33 amd64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.1l-freebsd 15 Jan 2015","system","V8",64,16777216,1,"none visible", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv deleted file mode 100644 index cfe4f0061..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv +++ /dev/null @@ -1,2 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","anonymous_access","raw_response","hex_code","code","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serialNumber" -"2022-02-07 12:56:53","18.220.0.0","tcp",8883,"18-220-0-0.example.com","mqtt",12345,"US","OHIO","COLUMBUS",454110,,"N",20020005,05,"Connection Refused, not authorized","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"*.tracesafe.io","Sectigo RSA Domain Validation Secure Server CA","2020-08-12 00:00:00","2022-11-14 00:00:00","70:84:F1:6D:28:DA:B6:E6:27:60:13:8B:2C:93:52:B6:7B:4B:13:7B","D2:D7:54:52:EB:86:4E:2D:34:4D:FC:CE:CD:CF:39:41:E1:06:5C:8B:B8:54:E6:0C:DF:FD:6E:E3:F1:B5:41:00","17:57:FB:88:9D:BE:A7:F0:29:A5:31:FC:79:DF:F7:8A:1C:D6:4A:DF:1B:4A:DC:BF:05:E7:E8:2F:79:9A:FA:FE:F7:E8:66:22:CB:B9:4C:72:F7:FB:6C:1D:59:8C:54:63:70:05:DE:7F:3C:2F:BA:B8:37:18:CE:29:6F:11:E8:AB","DE:2C:98:30:27:2E:7D:C9:ED:A3:9D:AF:9E:CE:14:CC","085699743A23114C9B6B8DC975A8AF42",2,"sha256WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,"Sectigo Limited",,"GB","Greater Manchester","Salford",,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license deleted file mode 100644 index 476908eeb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv deleted file mode 100644 index e0ab4b929..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","raw_response","hex_code","code","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serialNumber" -"2022-01-10 00:59:34","47.106.0.0","tcp",8883,,"mqtt,mqtt-anon",37963,"CN","GUANGDONG SHENG","SHENZHEN",518210,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"Server","RootCA","2020-05-08 08:07:05","2030-05-06 08:07:05","70:1A:1E:1F:EC:5F:7E:A9:12:32:B2:C9:8A:C9:EE:91:8E:0B:82:45","85:26:A2:F2:A2:50:CD:96:33:19:A6:2D:12:2E:97:6B:D3:06:3C:11:EA:01:B4:B7:25:2A:B7:4F:0A:8F:45:40","72:50:07:30:9A:6F:CB:FD:E2:80:69:02:65:62:77:16:C3:B4:0C:98:44:4E:D4:2C:AC:6B:AF:F8:9E:AB:51:C2:FA:A8:72:A3:45:DF:81:09:50:08:18:EB:03:34:FC:92:33:A7:12:46:FE:90:20:91:86:C5:4D:89:48:86:4C:CD","AB:A8:E0:2C:EF:AE:BF:9D:DD:FA:70:BA:2F:F2:CA:5C",02,2,"sha256WithRSAEncryption","rsaEncryption","EMQ",,"CN","hangzhou",,,,,,,,,"EMQ",,"CN","hangzhou",,,,,,,, -"2022-01-10 00:59:34","144.76.0.0","tcp",8883,,"mqtt,mqtt-anon",24940,"DE","SACHSEN-ANHALT","WERNIGERODE",518210,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"example.com","R3","2021-12-06 13:48:04","2022-03-06 13:48:03","20:0E:AC:E7:AF:07:8D:D3:16:7C:63:D1:B9:12:AD:1D:2C:F0:46:86","DD:7A:4C:8A:1D:66:1D:7C:F5:17:04:5B:A0:B4:C4:E0:80:58:44:B4:DB:A7:5E:61:AE:43:9D:85:4C:9E:DC:83","55:B6:3D:56:A4:39:6E:99:B6:AF:72:AF:4D:3C:7C:C5:A8:C5:4F:A1:79:92:D0:46:8A:A2:9B:2A:48:0D:00:68:39:F0:B8:67:B4:E0:88:51:2A:D7:55:46:83:BD:ED:1E:09:6E:DB:3D:21:E2:AA:DB:42:6A:33:45:1A:2A:DB:4C","23:99:39:C6:77:D8:9F:55:90:FC:A5:FB:BA:72:8B:42","06B25BEAD1F43266ABCFCDDE408D3544D04B",2,"sha256WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,"Lets Encrypt",,"US",,,,,,,,, -"2022-01-10 00:59:34","173.0.0.0","tcp",8883,"example.com","mqtt,mqtt-anon",5555,"US","CALIFORNIA","BURBANK",,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",2048,"foo.example.com","ClearView2Dev","2020-08-07 16:51:57","2030-08-05 16:51:57","32:4B:66:98:FA:5B:D2:D1:F2:53:83:21:19:11:5A:A9:BE:85:56:16","AE:0D:65:34:2F:51:F7:32:1E:DF:B1:DA:12:C7:6A:DE:42:B5:4B:FF:80:2C:E5:EF:99:F6:CC:01:4B:C9:77:68","44:C4:B8:19:FA:39:55:51:EC:E4:6D:C4:6D:0F:A5:46:BB:D5:F9:FD:A6:8D:DF:F3:2D:D2:92:6C:0B:D5:D3:25:CB:19:50:9D:A6:A4:D4:D3:2E:53:10:F5:8D:77:F7:90:F8:65:A7:79:AB:14:62:72:01:F3:EA:38:E2:68:C7:25","43:0D:A7:89:9E:76:8D:6E:D5:AD:95:CC:F2:91:87:56","A71541EFAE529B03",0,"sha256WithRSAEncryption","rsaEncryption","Sohonet",,,,"<",,,,,,,,"Sohonet","ClearView2Dev",,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv deleted file mode 100644 index c12a6063e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","server_name","instance_name","tcp_port","named_pipe","response_size","amplification","sector" -"2010-02-10 00:00:00",192.168.0.1,udp,1434,node01.example.com,mssql,13.2.5026.0,64512,ZZ,Region,City,0,0,ERPOPTIMA,OPTIMA,49729,"\\\\ERPOPTIMA\\pipe\\MSSQL$OPTIMA\\sql\\query",310,310.00, -"2010-02-10 00:00:01",192.168.0.2,udp,1434,node02.example.com,mssql,13.0.1601.5,64512,ZZ,Region,City,0,0,SERWER,MSSQLSERVER,1433,,226,226.00,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,udp,1434,node03.example.com,mssql,10.50.2500.0,64512,ZZ,Region,City,0,0,ILONY,INSERTGT,49358,"\\\\ILONY\\pipe\\MSSQL$INSERTGT\\sql\\query",304,304.00, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv deleted file mode 100644 index 25fed2166..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","mysql_protocol_version","server_version","error_code","error_id","error_message","client_can_handle_expired_passwords","client_compress","client_connect_attrs","client_connect_with_db","client_deprecated_eof","client_found_rows","client_ignore_sigpipe","client_ignore_space","client_interactive","client_local_files","client_long_flag","client_long_password","client_multi_results","client_multi_statements","client_no_schema","client_odbc","client_plugin_auth","client_plugin_auth_len_enc_client_data","client_protocol_41","client_ps_multi_results","client_reserved","client_secure_connection","client_session_track","client_ssl","client_transactions","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,3306,node01.example.com,mysql,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",10,5.7.37-0ubuntu0.18.04.1,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:01",192.168.0.2,tcp,3306,node02.example.com,mysql,,64512,ZZ,Region,City,0,0,,10,5.7.30-0ubuntu0.18.04.1-log,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:02",192.168.0.3,tcp,3306,node03.example.com,mysql,,64512,ZZ,Region,City,0,0,"Retail Trade",10,8.0.23,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv deleted file mode 100644 index e8a1108d5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","opcode","uptime","external_ip","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,5351,node01.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,291278940,192.168.0.1,,12,6.00 -"2010-02-10 00:00:01",192.168.0.2,udp,5351,node02.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,768416,192.168.0.2,,12,6.00 -"2010-02-10 00:00:02",192.168.0.3,udp,5351,node03.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,19629454,192.168.0.3,,12,6.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv deleted file mode 100644 index 932225b0b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","mac_address","asn","geo","region","city","workgroup","machine_name","username","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,137,node01.example.com,netbios,00-00-00-00-00-00,64512,ZZ,Region,City,,NBG6503,NBG6503,0,0,,229,4.58 -"2010-02-10 00:00:01",192.168.0.2,udp,137,node02.example.com,netbios,00-00-00-00-00-00,64512,ZZ,Region,City,PRACOWNIAELN.,NAS-OLD,NAS-OLD,0,0,,193,3.86 -"2010-02-10 00:00:02",192.168.0.3,udp,137,node03.example.com,netbios,00-25-90-F0-64-64,64512,ZZ,Region,City,HRSIGMA,HR-SRV01,,0,0,Government,157,3.14 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv deleted file mode 100644 index 4e9159356..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","response","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,53413,node01.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 -"2010-02-10 00:00:01",192.168.0.2,53413,node02.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 -"2010-02-10 00:00:02",192.168.0.3,53413,node03.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv deleted file mode 100644 index cc3cf6fc2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","version","clk_wander","clock","error","frequency","jitter","leap","mintc","noise","offset","peer","phase","poll","precision","processor","refid","reftime","rootdelay","rootdispersion","stability","state","stratum","system","tai","tc","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,123,node01.example.com,ntpversion,64512,ZZ,Region,City,4,,0xe6ac3809.363028e7,,2.018,0.977,0,,0.984,0.557,18986,,10,-10,unknown,81.15.252.130,0xe6ac35ba.2d2e8f2b,17.685,61.254,0.027,4,4,UNIX,,,0,0,,324,27.00 -"2010-02-10 00:00:01",192.168.0.2,udp,123,node02.example.com,ntpversion,64512,ZZ,Region,City,4,0.007,0xE6AC3806.7DF3B7A0,,-20.407,8.776,0,3,,-14.502,19244,,,-10,unknown,10.48.21.21,0xE6AC3431.B3B64790,32.25,105.778,,,8,UNIX,,10,0,0,"Transportation and Warehousing",328,27.33 -"2010-02-10 00:00:02",192.168.0.3,udp,123,node03.example.com,ntpversion,64512,ZZ,Region,City,4,0.001,0xE6AC380A.5A1CAD00,,-24.01,2.343,0,3,,0.49,51892,,,-10,unknown,172.28.0.1,0xE6AC3020.0C49BA80,7.749,81.612,,,4,UNIX,,10,0,0,,324,27.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv deleted file mode 100644 index dca5386d9..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","packets","size","asn","geo","region","city","naics","sic","sector","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,123,node01.example.com,2,664,64512,ZZ,Region,City,0,0,,55.33 -"2010-02-10 00:00:01",192.168.0.2,udp,123,node02.example.com,100,44000,64512,ZZ,Region,City,0,0,,3666.67 -"2010-02-10 00:00:02",192.168.0.3,udp,123,node03.example.com,100,44000,64512,ZZ,Region,City,0,0,,3666.67 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv deleted file mode 100644 index c32bc3d4d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","programs","mountd_port","exports","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,111,node01.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0","Communications, Service Provider, and Hosting Service",148,3.70 -"2010-02-10 00:00:01",192.168.0.2,udp,111,node02.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0","Communications, Service Provider, and Hosting Service",148,3.70 -"2010-02-10 00:00:02",192.168.0.3,udp,111,node03.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0",Government,148,3.70 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv deleted file mode 100644 index 8c1d6f725..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","supported_protocols","protocol_error_code","protocol_error_file","protocol_error_line","protocol_error_message","protocol_error_routine","protocol_error_severity","protocol_error_severity_v","startup_error_code","startup_error_file","startup_error_line","startup_error_message","startup_error_routine","startup_error_severity","startup_error_severity_v","client_ssl","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,5432,node01.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, -"2010-02-10 00:00:01",192.168.0.2,tcp,5432,node02.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, -"2010-02-10 00:00:02",192.168.0.3,tcp,5432,node03.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv deleted file mode 100644 index 857699376..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","quote","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,17,node01.example.com,qotd,"_The secret of being miserable is to have leisure to bother about whether?? you are happy or not. The cure for it is occupation._?? George Bernard Shaw (1856-1950)?",64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",166,166.00 -"2010-02-10 00:00:01",192.168.0.2,udp,17,node02.example.com,qotd,"_Oh the nerves, the nerves; the mysteries of this machine called man!?? Oh the little that unhinges it, poor creatures that we are!_?? Charles Dickens (1812-70)?",64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",162,162.00 -"2010-02-10 00:00:02",192.168.0.3,udp,17,node03.example.com,qotd,"_Oh the nerves, the nerves; the mysteries of this machine called man!?? Oh the little that unhinges it, poor creatures that we are!_?? Charles Dickens (1812-70)?",64512,ZZ,Region,City,0,0,,162,162.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv deleted file mode 100644 index c9fb18896..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","version_field_1","version_field_2","version_field_3","version_field_4" -"2022-01-10 14:31:17","176.255.0.0","udp",443,"test1.example.com","quic",5607,"UK","LONDON","LONDON",517311,,"Q050",,"Q046","Q043" -"2022-01-10 14:31:17","24.244.0.0","udp",443,,"quic",6327,"CA","SASKATCHEWAN","MEACHAM",517311,,"Q050","Q046",,"Q043" -"2022-01-10 14:31:17","23.60.0.0","udp",443,"test3.example.com","quic",20940,"JP","OSAKA","OSAKA",517919,,,"Q050","Q046","Q043" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv deleted file mode 100644 index 76b388aca..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv +++ /dev/null @@ -1,10 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic" -"2020-07-06 13:55:26","74.101.218.75","tcp",4899,"static-74-101-218-75.nycmny.fios.verizon.net","radmin","Radmin (Details Unknown)",701,"US","NEW YORK","BROOKLYN",517312, -"2020-07-06 13:55:27","192.162.189.171","tcp",4899,"rubin.an.ru","radmin","Radmin v3.X Radmin Authentication",56618,"RU","MURMANSKAYA OBLAST","MURMANSK",0, -"2020-07-06 13:55:27","111.197.143.69","tcp",4899,,"radmin","Radmin (Details Unknown)",4808,"CN","BEIJING SHI","BEIJING",517311, -"2020-07-06 13:55:27","121.147.215.220","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",4766,"KR","GWANGJU-GWANGYEOKSI","DAEIN-DONG",517311, -"2020-07-06 13:55:27","121.147.215.178","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",4766,"KR","GWANGJU-GWANGYEOKSI","DAEIN-DONG",517311, -"2020-07-06 13:55:27","183.230.5.219","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",9808,"CN","CHONGQING SHI","CHONGQING",517312, -"2020-07-06 13:55:27","85.93.154.74","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",34300,"RU","MOSKVA","MOSCOW",0, -"2020-07-06 13:55:27","81.246.135.247","tcp",4899,"247.135-246-81.adsl-dyn.isp.belgacom.be","radmin","Radmin v3.X Radmin Authentication",5432,"BE","ANTWERPEN","BRASSCHAAT",517311, -"2020-07-06 13:55:27","46.27.146.22","tcp",4899,"static-22-146-27-46.ipcom.comunitel.net","radmin","Radmin v3.X Radmin Authentication",12430,"ES","LAS PALMAS","LAS PALMAS DE GRAN CANARIA",517312, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license deleted file mode 100644 index 833024a75..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 sinus-x -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv deleted file mode 100644 index 4bac90f19..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","rdp_protocol","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","naics","sic","sector","tlsv13_support","tlsv13_cipher","cve20190708_vulnerable","bluekeep_vulnerable","jarm" -"2019-09-04 15:45:51","198.123.245.178",5678,,"rdp",,5678,"AA","LOCATION","LOCATION","RDP",5678,"KABESRV.KABE.local","KABESRV.KABE.local","2019-04-29 02:22:06","2019-10-29 02:22:06","EC:BB:4D:DB:9F:0C:D3:FF:5B:49:EA:B1:56:62:B6:A7:5D:60:54:42","1EF2B37AF850C9BF4E88F18177001D6B",2,"sha256WithRSAEncryption","rsaEncryption","B7:C9:F4:07:D5:C0:75:1D:EA:0C:40:E7:26:39:C2:30:C6:13:83:7E:18:46:D8:E9:4C:45:3F:88:1B:0B:70:76","08:AC:75:FA:EB:A3:2B:44:15:DE:6D:A7:0B:C0:AE:17:94:F3:55:D9:EC:70:AC:5B:B7:94:79:F0:D7:84:83:89:CB:A9:11:E0:08:D7:54:4D:33:85:89:D2:A8:DD:9D:15:F4:CC:95:DE:6A:E3:DF:6B:FA:8B:27:E3:DA:16:AF:0A","BC:6E:C3:E2:98:22:EC:BA:5B:30:E2:53:FD:4A:9D:FF",517311,0,,,,"N","N" -"2019-09-04 15:45:51","198.123.245.233",5678,,"rdp",,5678,"AA","LOCATION","LOCATION","RDP",5678,"RAMBLA01.rambla.local","RAMBLA01.rambla.local","2019-04-16 06:15:20","2019-10-16 06:15:20","7A:67:1F:F8:87:C6:B0:AC:A9:84:15:B7:40:EC:CB:19:AA:E3:19:52","3FF3EBC5CF154BA54D128A8548C8AAF5",2,"sha1WithRSAEncryption","rsaEncryption","8F:CD:7D:C4:80:2D:8D:9B:06:A0:40:18:9F:ED:73:7A:BA:83:55:BE:1B:56:83:A2:97:DF:BB:B4:06:57:CB:F1","E8:9B:9A:93:69:B4:58:01:D8:46:C2:DC:01:20:1E:DD:93:E1:EB:E3:9D:6B:65:A0:C5:00:6C:A4:44:08:FE:A4:A6:19:FF:55:79:F2:AA:61:68:C8:1C:B0:CE:78:EB:84:DD:29:9D:64:2F:4E:25:31:3A:6C:B8:02:C9:AF:F5:1F","38:73:6A:B3:AA:41:69:C9:BA:E7:3D:D7:40:16:F8:AA",517311,0,"Information Technology",,,"N","N" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv deleted file mode 100644 index 73d0d55ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sessionid","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,3389,node01.example.com,rdpeudp,64512,ZZ,Region,City,0,0,05b28c0c,1232,77.00 -"2010-02-10 00:00:01",192.168.0.2,udp,3389,node02.example.com,rdpeudp,64512,ZZ,Region,City,0,0,053d355f,1232,77.00 -"2010-02-10 00:00:02",192.168.0.3,udp,3389,node03.example.com,rdpeudp,64512,ZZ,Region,City,0,0,0567a8cb,1232,77.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv deleted file mode 100644 index dc9760cf2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv +++ /dev/null @@ -1,94 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","git_sha1","git_dirty_flag","build_id","mode","os","architecture","multiplexing_api","gcc_version","process_id","run_id","uptime","connected_clients","sector" -"2016-07-24 00:42:33","198.51.100.152","tcp",6379,,"redis","2.8.19",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"26069fb482f6334b","standalone","Linux 3.2.0-4-amd64 x86_64",,"epoll","4.7.2",2127,"d440b0b2fb3d1db655ad607e11e6f38011a0f599",27946314,50, -"2016-07-24 00:42:43","198.51.100.67","tcp",6379,"198-51-100-67.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310556,25376, -"2016-07-24 00:42:43","198.51.100.125","tcp",6379,"198-51-100-125.example.net","redis","2.8.17",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5937320cdd80c1e4","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.9.2",11573,"0d58143df099738a7ce9330ee5ec2367d11b1187",25888041,4, -"2016-07-24 00:42:43","198.51.100.203","tcp",6379,"198-51-100-203.example.net","redis","2.8.4",31103,"DE","THURINGEN","ERFURT",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-83-generic x86_64",,"epoll","4.8.2",3847,"4f7765dee91d8c4b1b24604cc5f0c29fca1a4f32",3068554,38, -"2016-07-24 00:42:43","198.51.100.240","tcp",6379,"198-51-100-30.example.net","redis","3.0.7",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.4",1011,"864c8d7df1e72c662a4edd77b6df6cd30161af6e",2476542,2,"Information Technology" -"2016-07-24 00:42:49","198.51.100.69","tcp",6379,"198-51-100-69.example.net","redis","3.0.6",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"315c8c74805fca88","standalone","Linux 3.2.0-98-generic x86_64",,"epoll","4.6.3",28961,"bc705102c854ea1818213e4740a3c6fd9b9f1716",4633191,1, -"2016-07-24 00:42:53","198.51.100.50","tcp",6379,"198-51-100-50.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"6afb1e1f0d80abd0","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.4.5",1717,"f729595b3642b48f3ac9e098bcccab1d6ef82e3e",6345372,3, -"2016-07-24 00:43:49","198.51.100.113","tcp",6379,,"redis","3.0.6",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310623,24628, -"2016-07-24 00:43:49","198.51.100.228","tcp",6379,"198-51-100-131.example.net","redis","2.8.210",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"83ad777ec89a946b","standalone","Windows",,"winsock_IOCP",,1948,"f5d6ad26e423039636afaf3918ee7e6a7e0b5b68",2214134,4,"Information Technology" -"2016-07-24 00:43:59","198.51.100.155","tcp",6379,,"redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"f09a0843cc9876c3","standalone","Linux 3.13.0-79-generic x86_64",,"epoll","4.9.2",1,"5f4f5b7158f928cc96e3ae6af6092a163ace15eb",2897902,24, -"2016-07-24 00:43:59","198.51.100.171","tcp",6379,,"redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310633,25031, -"2016-07-24 00:44:09","198.51.100.230","tcp",6379,"198-51-100-230.example.net","redis","2.8.11",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"f26bfdf4b8265fc","standalone","Linux 3.8.0-29-generic x86_64",,"epoll","4.6.3",14551,"0d001175cf26cee88486d814b4f0c972a5aa89b9",21038337,9, -"2016-07-24 00:44:09","198.51.100.182","tcp",6379,"198-51-100-182.example.net","redis","3.0.7",197540,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"fd24f54fec00684b","standalone","Linux 3.13.0-85-generic x86_64",,"epoll","4.8.4",949,"b11fdf2b95251b8e6c3e9e782409ef82fc8b89aa",8643389,11, -"2016-07-24 00:44:10","198.51.100.23","tcp",6379,"198-51-100-116.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 4.2.0-27-generic x86_64",,"epoll","4.8.2",335,"90079d58e970a1ae94aa91bc0ea0236a0e55269c",4930922,2,"Information Technology" -"2016-07-24 00:44:19","198.51.100.51","tcp",6379,"198-51-100-51.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310652,26257, -"2016-07-24 00:44:22","198.51.100.88","tcp",6379,,"redis","3.0.6",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310656,26371, -"2016-07-24 00:44:22","198.51.100.107","tcp",6379,"octopus-dev","redis","2.8.14",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"78be6d5e32e34139","standalone","Linux 2.6.32-042stab108.2 x86_64",,"epoll","4.8.2",21205,"b98a41b6ea690c207527587f60bff1f1d24236b4",9364864,4, -"2016-07-24 00:44:22","198.51.100.75","tcp",6379,,"redis","3.0.0",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"2b5201a6bfd5f75e","standalone","Linux 3.11.0-19-generic x86_64",,"epoll","4.8.2",832,"2bdcda8b3b59cef244785b58935d68daf48645be",6745479,5, -"2016-07-24 00:44:25","198.51.100.12","tcp",6379,,"redis","3.0.6",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.13.0-79-generic x86_64",,"epoll","4.8.4",899,"94550e510bf770aa315cc3983ce9958853c77cfe",7816856,9, -"2016-07-24 00:44:27","198.51.100.13","tcp",6379,"198-51-100-13.example.net","redis","3.0.7",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"6f8b503a2787e3a6","standalone","Linux 4.4.5-15.26.amzn1.x86_64 x86_64",,"epoll","4.9.2",1,"e050f40e755a739ffecdb2468e1333f371e2abca",7124048,6,"Communications" -"2016-07-24 00:44:29","198.51.100.12","tcp",6379,"198-51-100-12.example.net","redis","2.8.3",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"992c97be25a6b6d2","standalone","Linux 2.6.32-042stab111.12 x86_64",,"epoll","4.4.5",12340,"d7cda18212cf4bcdfd7c42fff33e506a4e9a2614",16874891,8, -"2016-07-24 00:44:38","198.51.100.66","tcp",6379,"198-51-100-66.example.net","redis","3.2.1",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ed627d97d5dc311e","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"4a6beb721ddbaa411f53e5268e6112127903cae3",2029470,3,"Chemical" -"2016-07-24 00:44:38","198.51.100.170","tcp",6379,,"redis","3.0.6",8881,"DE","SACHSEN","RADEBEUL",0,0,00000000,0,"1b14d17ce6fea422","standalone","Linux 4.2.6-1-pve x86_64",,"epoll","4.9.2",728,"c423ba856285690a2fae350b03514cec80db9d5e",1679635,1, -"2016-07-24 00:44:38","198.51.100.67","tcp",6379,"198-51-100-67.example.net","redis","2.8.23",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"8e819a849ea2d7f8","standalone","Linux 4.2.0-23-generic x86_64",,"epoll","4.9.2",1,"7ee1dc403540ff4d1fc0a80d9f0b2910857b6c1b",9451832,68,"Information Technology" -"2016-07-24 00:44:44","198.51.100.238","tcp",6379,,"redis","2.8.4",44066,"DE","BAYERN","MUNICH",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 2.6.32-19-pve x86_64",,"epoll","4.8.2",2207,"6a079396cc44c1aca745edab13f4014c394da3ab",10338949,3, -"2016-07-24 00:44:44","198.51.100.84","tcp",6379,"198-51-100-84.example.net","redis","3.0.2",51862,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"4795df119e2d77fe","standalone","Linux 3.13.0-91-generic x86_64",,"epoll","4.7.2",1,"c120481a551c232b8e1a9cff20d9e0968a402dd9",1040551,7, -"2016-07-24 00:44:44","198.51.100.23","tcp",6379,"198-51-100-23.example.net","redis","3.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"98c227055d7fa7b6","standalone","Linux 3.10.0-327.10.1.el7.x86_64 x86_64",,"epoll","4.8.5",35198,"424b15e04ce09f26299ff19b252a920916d4e4be",8875355,2, -"2016-07-24 00:44:47","198.51.100.160","tcp",6379,"198-51-100-160.example.net","redis","2.8.210",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"83ad777ec89a946b","standalone","Windows",,"winsock_IOCP",,2284,"9bde76afda6f81acfb241ea5ee3a9e878ad53881",742778,2, -"2016-07-24 00:44:47","198.51.100.111","tcp",6379,"198-51-100-98.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e19bb8c3d1c28291","standalone","Linux 3.10.0-327.22.2.el7.x86_64 x86_64",,"epoll","5.3.0",1,"c951371f430c1d94299bfc93759f6940d8bfce78",208557,2, -"2016-07-24 00:44:48","198.51.100.227","tcp",6379,"198-51-100-227.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310681,26496, -"2016-07-24 00:44:54","198.51.100.18","tcp",6379,"198-51-100-18.example.net","redis","2.8.9",8972,"DE","NORDRHEIN-WESTFALEN","COLOGNE",0,0,00000000,0,"52c7b9284559eb20","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.4.5",31887,"e5b1da35862482c4df8d4fce635ec89a36476a4d",14393072,6, -"2016-07-24 00:44:54","198.51.100.248","tcp",6379,"198-51-100-248.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310687,26112, -"2016-07-24 00:44:57","198.51.100.228","tcp",6379,"198-51-100-228.example.net","redis","3.0.7",8972,"DE","NORDRHEIN-WESTFALEN","COLOGNE",0,0,00000000,0,"5e03212a543f54f8","standalone","Linux 3.13.0-042stab116.1 x86_64",,"epoll","4.8.4",719,"537e3e824a45414c3199ef20201b4362b752eeb5",1263367,2, -"2016-07-24 00:45:04","198.51.100.227","tcp",6379,"198-51-100-227.example.net","redis","2.8.12",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ff040dde4a39b4ff","standalone","Windows",,"winsock_IOCP","0.0.0",1872,"c78751c65793a9a72f6fb0318efa532eb4fc87de",277953,18,"Chemical" -"2016-07-24 00:45:07","198.51.100.132","tcp",6379,,"redis","3.0.5",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"30405cba8f6c2d55","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",2500,"10b4084b930d5a77e5f09e89cf0b21702027bd60",10028956,695, -"2016-07-24 00:46:10","198.51.100.47","tcp",6379,"198-51-100-185.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"6a943c0b5bf37fa1","standalone","Linux 4.4.0-24-generic x86_64",,"epoll","5.3.1",1023,"de9c9c0da3d971f689bd7366c1edc93a00fd1506",2791106,1, -"2016-07-24 01:23:27","198.51.100.246","tcp",6379,"198-51-100-190.example.net","redis","2.8.19",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"665519ce00ddac9b","standalone","Linux 2.6.32-43-pve x86_64",,"epoll","4.9.2",2310,"94595838457eddb30a60184a9db66212268e6f82",9481199,4, -"2016-07-24 01:23:29","198.51.100.187","tcp",6379,"198-51-100-63.example.net","redis","2.8.19",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"c0359e7aa3798aa2","standalone","Linux 3.10.0-229.7.2.el7.x86_64 x86_64",,"epoll","4.8.3",14050,"e67a19de4bd2dc485b98ca353eb6fdc65e8fed4a",14051444,10, -"2016-07-24 01:23:29","198.51.100.228","tcp",6379,"198-51-100-228.example.net","redis","2.8.4",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-32-generic x86_64",,"epoll","4.8.2",22837,"daf5dba760d3db12716c6dc1d0bfe6d5e7b33749",10916038,8, -"2016-07-24 01:23:43","198.51.100.180","tcp",6379,"198-51-100-180.example.net","redis","3.2.1",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ed627d97d5dc311e","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"569881874d8d5e1508d584a3fd9dff0ac3515839",1677711,1,"Chemical" -"2016-07-24 01:23:56","198.51.100.5","tcp",6379,"198-51-100-207.example.net","redis","3.0.7",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.4",1011,"864c8d7df1e72c662a4edd77b6df6cd30161af6e",2479015,2,"Information Technology" -"2016-07-24 01:24:03","198.51.100.226","tcp",6379,"198-51-100-226.example.net","redis","3.0.5",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"b33bc3e2f8ad13f6","standalone","Linux 2.6.32-573.12.1.el6.x86_64 x86_64",,"epoll","4.4.7",1801,"7f4bb7ed008cdbd665672e88d57fc55616b6dbf2",13189200,9, -"2016-07-24 01:24:14","198.51.100.253","tcp",6379,"198-51-100-136.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.19.0-39-generic x86_64",,"epoll","4.8.2",28272,"13a889aa846c6302dc8f5453e35e051a6f359e9a",14046610,185, -"2016-07-24 01:24:28","198.51.100.206","tcp",6379,,"redis","3.0.6",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313061,26695, -"2016-07-24 01:24:35","198.51.100.73","tcp",6379,,"redis","3.0.2",28753,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"6be7fc9e6b88f79","standalone","Linux 2.6.32-71.29.1.el6.x86_64 x86_64",,"epoll","4.4.7",811,"bbd4dce247ab51d029a64243810aeb900e00d1d6",10082205,15, -"2016-07-24 01:24:35","198.51.100.83","tcp",6379,"198-51-100-174.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"7e7b61a55b95e8e7","standalone","Linux 4.2.0-41-generic x86_64",,"epoll","4.8.4",1076,"48f5f780ca53553fc4c0bbdbb32a5cb06a0551cd",814255,88,"Information Technology" -"2016-07-24 01:25:30","198.51.100.182","tcp",6379,,"redis","3.0.7",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM",0,0,00000000,0,"d9ceac045f7983a9","standalone","FreeBSD 10.1-RELEASE-p26 amd64",,"kqueue","4.2.1",957,"48f37d15b3f5169f11aa5d7194fdfccc7f8df20b",6364747,1, -"2016-07-24 01:25:30","198.51.100.211","tcp",6379,"198-51-100-118.example.net","redis","2.8.17",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e4968abcd4b78b2e","standalone","Linux 3.13.0-36-generic x86_64",,"epoll","4.8.2",1643,"665565b1b1fb6e773039707a0f680bbc417186be",20180649,4,"Information Technology" -"2016-07-24 01:25:35","198.51.100.249","tcp",6379,,"redis","3.0.2",28753,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"6be7fc9e6b88f79","standalone","Linux 2.6.32-71.29.1.el6.x86_64 x86_64",,"epoll","4.4.7",811,"bbd4dce247ab51d029a64243810aeb900e00d1d6",10082265,15, -"2016-07-24 01:25:40","198.51.100.55","tcp",6379,,"redis","3.2.1",3320,"DE","NORDRHEIN-WESTFALEN","SOLINGEN",518210,737415,00000000,0,"e19bb8c3d1c28291","standalone","Linux 4.4.0-24-generic x86_64",,"epoll","5.3.0",1,"49687ba2a5be7f7b6cdf0c837e06307442f6a369",494739,1, -"2016-07-24 01:25:42","198.51.100.62","tcp",6379,"198-51-100-62.example.net","redis","3.0.7",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"2b87841ee28adfc3","standalone","Linux 3.13.0-042stab113.11 x86_64",,"epoll","4.8.4",525,"4045d68fd2e59a1135bb303206d7cd0439ba7ffd",6971251,4, -"2016-07-24 01:25:55","198.51.100.127","tcp",6379,"198-51-100-25.example.net","redis","2.8.4",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.2",11492,"3de3e977405eef9392a77db4a50d99a5caa2f2d9",2194103,3,"Information Technology" -"2016-07-24 01:26:08","198.51.100.92","tcp",6379,"198-51-100-92.example.net","redis","2.8.10",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5fce0c4aab65e01","standalone","Linux 2.6.32-042stab113.11 x86_64",,"epoll","4.6.3",490,"15abe68a10b011972f50d0abb3bb18f1735994a5",7505621,4, -"2016-07-24 01:26:17","198.51.100.218","tcp",6379,,"redis","3.0.7",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"dc142e699f115c40","standalone","Linux 3.2.60-grsec-x86_64 x86_64",,"epoll","4.7.3",8006,"53a093bd4d0a7b72b2d084ec3767d23b18b8b947",4024979,7, -"2016-07-24 01:26:29","198.51.100.168","tcp",6379,"198-51-100-168.example.net","redis","3.0.6",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.13.0-37-generic x86_64",,"epoll","4.8.4",1279,"8218bd77a0dcb0e00bd77dbb9478115757c70ba5",2405965,1, -"2016-07-24 01:26:29","198.51.100.155","tcp",6379,"198-51-100-155.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"d9155128f7b25ea0","standalone","Linux 3.19.0-25-generic x86_64",,"epoll","4.8.4",27030,"0ede623cb268643672abc04d0267f684a5ee7a0d",6880190,5,"Information Technology" -"2016-07-24 01:26:34","198.51.100.185","tcp",6379,,"redis","2.8.4",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-43-generic x86_64",,"epoll","4.8.2",1196,"ae80fcbb54017f521212caf257418885cd6836a0",5412584,5, -"2016-07-24 01:26:34","198.51.100.1","tcp",6379,"198-51-100-1.example.net","redis","3.2.0",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"5382f69a4e75566b","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"ff8990f109ff5b2d4e0eee47e5ebc66acc43f9e3",4615889,4,"Chemical" -"2016-07-24 01:26:39","198.51.100.51","tcp",6379,"198-51-100-164.example.net","redis","3.0.0",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"9526f4809583faaa","standalone","Linux 2.6.32-042stab113.21 x86_64",,"epoll","4.4.5",14528,"d7271feff55175f434ace92d199f332ad35776a9",7440370,16, -"2016-07-24 01:26:44","198.51.100.138","tcp",6379,,"redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313197,26452, -"2016-07-24 01:26:47","198.51.100.16","tcp",6379,,"redis","2.8.17",25074,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5937320cdd80c1e4","standalone","Linux 2.6.32-43-pve x86_64",,"epoll","4.9.2",266,"e1d403f2daff849a64b178f74c672db6712f217a",351253,1, -"2016-07-24 01:26:54","198.51.100.171","tcp",6379,"198-51-100-171.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313207,26601, -"2016-07-24 01:27:14","198.51.100.89","tcp",6379,"198-51-100-89.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313227,26358, -"2016-07-24 01:27:24","198.51.100.65","tcp",6379,,"redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-85-generic x86_64",,"epoll","4.8.4",21575,"3ec40168300e14f5776d82a48ba873a3999caec1",1897530,1, -"2016-07-24 01:27:24","198.51.100.248","tcp",6379,"198-51-100-248.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313237,25902, -"2016-07-24 01:27:33","198.51.100.17","tcp",6379,,"redis","2.8.17",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"43dd9e14444e6aea","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",556,"3e8fc2878511cc72f79b765fca86cefe21346912",2607965,72, -"2016-07-24 01:27:33","198.51.100.134","tcp",6379,"198-51-100-134.example.net","redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"6f8b503a2787e3a6","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",1,"b85b2419cf35dd81ff5b9ba6e8bf802cf1d439f6",128621,33, -"2016-07-24 01:27:42","198.51.100.186","tcp",6379,"198-51-100-186.example.net","redis","2.8.13",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"d588bf57ea0dfa69","standalone","Linux 4.4.8-jb1 i686",,"epoll","4.6.3",2460,"97b8d49e62d340d94a38c96c5104abfcacbfa4cb",181557,1, -"2016-07-24 01:27:42","198.51.100.21","tcp",6379,"198-51-100-21.example.net","redis","2.8.19",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"920d7eda78149e99","standalone","Linux 4.4.8-x86_64-jb1 x86_64",,"epoll","4.7.2",3722,"74dfd8a7d87cbb9ecc590ceafd438c85d5073903",183984,1, -"2016-07-24 01:27:43","198.51.100.128","tcp",6379,"198-51-100-203.example.net","redis","3.0.5",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"f3bd5bc2b8b4c486","standalone","Linux 2.6.32-573.8.1.el6.x86_64 x86_64",,"epoll","4.4.7",1968,"0d92b1323fea791ba4b0a43435a156b6ec0aac1c",2967611,2,"Information Technology" -"2016-07-24 01:27:44","198.51.100.216","tcp",6379,"198-51-100-229.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.16.0-30-generic x86_64",,"epoll","4.8.2",1470,"e76cd0cf25eec5d254c880965189ae011a119220",302420,1, -"2016-07-24 01:27:53","198.51.100.242","tcp",6379,"198-51-100-242.example.net","redis","3.0.2",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"6a04b5ede30cd4cd","standalone","Linux 3.13.0-32-generic x86_64",,"epoll","4.8.4",29725,"1b7e8dc53dec8fb29a8a2d76f516fd3dcb8df652",5815739,7, -"2016-07-24 01:27:53","198.51.100.54","tcp",6379,"198-51-100-54.example.net","redis","2.8.4",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-91-generic x86_64",,"epoll","4.8.2",2903,"0e02514dec6031018eb148b13a4a9639cab3e8aa",905886,1, -"2016-07-24 01:27:54","198.51.100.225","tcp",6379,"198-51-100-225.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313267,25281, -"2016-07-24 01:27:57","198.51.100.38","tcp",6379,"198-51-100-38.example.net","redis","3.0.5",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"3b863f97501297e9","standalone","Linux 3.13.0-042stab111.12 x86_64",,"epoll","4.8.4",2088,"31a8cececad2e4a33310a741143d85cdef3479b4",11906868,10, -"2016-07-24 01:27:58","198.51.100.22","tcp",6379,"198-51-100-22.example.net","redis","2.8.9",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"2ac6afaedfd3ea15","standalone","Linux 3.13.0-86-generic x86_64",,"epoll","4.8.4",9082,"8e5d9d74c86a9f148a7012733eb52a21938c3c04",5833880,5, -"2016-07-24 01:28:05","198.51.100.106","tcp",6379,"198-51-100-106.example.net","redis","2.8.19",36351,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"9968db13395be4aa","standalone","Windows",,"winsock_IOCP","0.0.0",4372,"89716352a10cd53b5c10e6d5e6cd1d46f5f53a30",485031,4,"Information Technology" -"2016-07-24 01:28:06","198.51.100.130","tcp",6379,"198-51-100-130.example.net","redis","2.8.3",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"542faa6f897d2236","standalone","Linux 2.6.32-573.3.1.el6.x86_64 x86_64",,"epoll","4.4.7",25531,"9d7606a883f764e744d766b7bf0036ba61f7fb6e",496133,5, -"2016-07-24 01:28:08","198.51.100.37","tcp",6379,"198-51-100-37.example.net","redis","2.8.23",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"50630e46be5feb4f","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.9.2",1,"62d16be721c3c62d6c4d080a9bdbe9502c57ca86",3481683,9,"Communications" -"2016-07-24 01:28:32","198.51.100.148","tcp",6379,"198-51-100-148.example.net","redis","3.0.5",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"83dc15dcf8ee3eb8","standalone","Linux 4.1.7-15.23.amzn1.x86_64 x86_64",,"epoll","4.8.3",2304,"883accf76dc364c60902b4eab7861dd1a7eac71d",10981957,10,"Communications" -"2016-07-24 01:28:49","198.51.100.247","tcp",6379,"198-51-100-247.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"3e971e94fbe2eaa6","standalone","Linux 3.2.0-4-amd64 x86_64",,"epoll","4.7.2",2535,"d223aab0621cdd2e4ab752978ad3009ad3814d8b",7715188,57, -"2016-07-24 02:08:46","198.51.100.220","tcp",6379,"198-51-100-220.example.net","redis","3.0.6",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"1f8e4c92f1ca309","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.8.4",3355,"dd517756bb6ee81e1929fa605972318b2baebb93",5211978,10, -"2016-07-24 02:08:46","198.51.100.239","tcp",6379,"198-51-100-239.example.net","redis","2.8.23",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"83a5616190c5a1aa","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",711,"4117960b13fa313b823c79b0e9f188d8ec6aa3ac",10156283,6, -"2016-07-24 02:08:50","198.51.100.233","tcp",6379,,"redis","2.8.11",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"f26bfdf4b8265fc","standalone","Linux 3.8.0-29-generic x86_64",,"epoll","4.6.3",14551,"0d001175cf26cee88486d814b4f0c972a5aa89b9",21043417,9, -"2016-07-24 02:08:51","198.51.100.208","tcp",6379,"198-51-100-181.example.net","redis","3.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 4.2.0-38-generic x86_64",,"epoll","4.8.4",809,"14c5ec7f9669e42ea45a40ff26a6501d593695c0",2405839,19, -"2016-07-24 02:08:51","198.51.100.60","tcp",6379,"198-51-100-60.example.net","redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"4ed99bd9c45dfc14","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",1144,"9e28c29ff40017e2fbe32fb97755caf801f95793",843538,2, -"2016-07-24 02:08:51","198.51.100.107","tcp",6379,"198-51-100-39.example.net","redis","3.2.0",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"82b2619163aabc80","standalone","Linux 4.2.0-25-generic x86_64",,"epoll","4.9.2",1,"98f6640bbde04b1214730937212e1fd4e58d03a8",2195657,12, -"2016-07-24 02:08:54","198.51.100.31","tcp",6379,,"redis","2.8.4",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-042stab111.12 x86_64",,"epoll","4.8.2",1112,"9c4e55b5ebd06045c5d89d43fa202e219ec8b42c",8839783,7, -"2016-07-24 02:08:56","198.51.100.221","tcp",6379,,"redis","3.0.7",44066,"DE","BAYERN","MUNICH",0,0,00000000,0,"49f951dce0725d71","standalone","FreeBSD 10.0-RELEASE-p7 amd64",,"kqueue","4.2.1",932,"28c6af3c4dedcd9b71cf51a7ebc4e84899196aee",8000949,1, -"2016-07-24 02:09:01","198.51.100.155","tcp",6379,"198-51-100-155.example.net","redis","2.8.22",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"fcdf45e47686c89b","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",7,"946ec6b96fe9925d2b677ce02b6c56097c5e69a8",8449694,6, -"2016-07-24 02:09:02","198.51.100.219","tcp",6379,"198-51-100-219.example.net","redis","2.8.4",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.8.2",1047,"9b83d6a6e7a6ffe50e75dac88cdc5e06f6203c9c",966148,1,"Chemical" -"2016-07-24 02:09:02","198.51.100.193","tcp",6379,"198-51-100-193.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"fd640d8ef55a22dd","standalone","Linux 4.2.0-42-generic x86_64",,"epoll","4.8.4",1397,"ed5ec17d78d089af53afd4abc339f7decf4641d4",651175,2,"Information Technology" -"2016-07-24 02:09:20","198.51.100.120","tcp",6379,"198-51-100-120.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"ed627d97d5dc311e","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",1,"f524508ad29334eee2fcf7bdda5c80b9f99d3dfe",987580,167, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv deleted file mode 100644 index a61e4573e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","module","motd","has_password" -"2010-02-10 00:00:00",192.168.0.1,tcp,873,node01.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N -"2010-02-10 00:00:01",192.168.0.2,tcp,873,node02.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N -"2010-02-10 00:00:02",192.168.0.3,tcp,873,node03.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv deleted file mode 100644 index ee0a625e5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","sip","sip_code","sip_reason","user_agent","sip_via","sip_to","sip_from","content_length","content_type","server","contact","cseq","call_id","allow","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,5060,node01.example.com,sip,64512,ZZ,Region,City,SIP/2.0,489,"Event Package Not Supported",,,,,0,,,,,,"INVITE,ACK,BYE,CANCEL,REGISTER",15.57,109 -"2010-02-10 00:00:01",192.168.0.2,udp,5060,node02.example.com,sip,64512,ZZ,Region,City,SIP/2.0,400,"Bad Request",,,,,364,text/plain,,,,,,62.57,438 -"2010-02-10 00:00:02",192.168.0.3,udp,5060,node03.example.com,sip,64512,ZZ,Region,City,SIP/2.0,400,"Bad Request",,,,,0,,,,,,,6.57,46 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license deleted file mode 100644 index 9f58c89ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv deleted file mode 100644 index 256dd78f6..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","version","function","function_text","flags","next_extension_offset","xid","language_tag_length","language_tag","error_code","error_code_text","response_size","raw_response" -"2010-02-10 00:00:00",192.168.0.1,tcp,427,node01.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== -"2010-02-10 00:00:01",192.168.0.2,tcp,427,node02.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== -"2010-02-10 00:00:02",192.168.0.3,tcp,427,node03.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license deleted file mode 100644 index 9f58c89ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv deleted file mode 100644 index fc7fe2fff..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","smb_implant","arch","key","smbv1_support","smb_major_number","smb_minor_number","smb_revision","smb_version_string" -"2010-02-10 00:00:00",192.168.0.1,445,node01.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" -"2010-02-10 00:00:01",192.168.0.2,445,node02.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" -"2010-02-10 00:00:02",192.168.0.3,445,node03.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv deleted file mode 100644 index 19eb56053..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" -"2021-07-08 11:58:42","1.2.3.4","tcp",25,"smtp-server.invalid","smtp;21nails",12345,"EE","HARJUMAA","TALLINN",,,"220 smtp-server.invalid ESMTP Exim 4.80 Wed, 11 Jun 2021 10:00:00 +0300|" -"2021-07-08 11:58:44","5.6.7.8","tcp",25,"smtp-out.invalid","smtp;21nails",23456,"EE","HARJUMAA","TALLINN",,,"220 smtp-out.invalid, ESMTP EXIM 4.86_2|" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license deleted file mode 100644 index c1900637f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv deleted file mode 100644 index f489261c4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","sysdesc","sysname","asn","geo","region","city","version","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_sector","tag","community","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,161,node01.example.com,"Linux localhost 3.18.20 #1 SMP Mon Jul 9 14:11:21 CST 2018 armv7l",,64512,ZZ,Region,City,2,0,0,,,,,,,snmp,public,165,1.94 -"2010-02-10 00:00:01",192.168.0.2,udp,161,node02.example.com,"RouterOS CCR1009-8G-1S-1S+",,64512,ZZ,Region,City,2,0,0,,MikroTik,router,,,consumer,"snmp,iot",public,115,1.35 -"2010-02-10 00:00:02",192.168.0.3,udp,161,node03.example.com,,,64512,ZZ,Region,City,2,0,0,,,,,,,snmp,public,85,1.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv deleted file mode 100644 index c591a5c09..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector" -"2010-02-10 00:00:00",192.168.0.1,tcp,1080,node01.example.com,socks4,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:01",192.168.0.2,tcp,1080,node02.example.com,socks5,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,tcp,1080,node03.example.com,socks4,64512,ZZ,Region,City,0,0,"Retail Trade" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv deleted file mode 100644 index 460be32c5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","header","asn","geo","region","city","systime","cache_control","location","server","search_target","unique_service_name","host","nts","nt","content_type","naics","sic","sector","server_port","instance","version","updated_at","resource_identifier","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,60194,node01.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,"Sun, 21 Aug 2022 09:51:13 GMT",max-age=100,http://192.168.200.254:49152/description.xml,"Linux/2.6.26, UPnP/1.0, Portable SDK for UPnP devices/1.3.1",upnp:rootdevice,uuid:28802880-2880-1880-a880-001bc502f600::upnp:rootdevice,node01.example.com,,,,0,0,Government,,,,,,3.35,325 -"2010-02-10 00:00:01",192.168.0.2,udp,38732,node02.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,,"max-age = 1800",http://95.160.216.14:52235/dmr/SamsungMRDesc.xml,"Linux/9.0 UPnP/1.0 PROTOTYPE/1.0",upnp:rootdevice,uuid:f144ca92-6816-94b5-b95f-b58180834044::upnp:rootdevice,node02.example.com,,,,0,0,,,,,,,2.71,263 -"2010-02-10 00:00:02",192.168.0.3,udp,57626,node03.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,"Sun, 03 Jan 2016 21:37:50 GMT",max-age=1800,http://192.168.1.3:8008/ssdp/device-desc.xml,"Linux/3.10.79, UPnP/1.0, Portable SDK for UPnP devices/1.6.18",upnp:rootdevice,uuid:62fa0fc8-079d-d00f-2e22-59b49fb488f9::upnp:rootdevice,node03.example.com,,,,0,0,Government,,,,,,4.79,465 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv deleted file mode 100644 index 837adbad1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","serverid_raw","serverid_version","serverid_software","serverid_comment","server_cookie","available_kex","available_ciphers","available_mac","available_compression","selected_kex","algorithm","selected_cipher","selected_mac","selected_compression","server_signature_value","server_signature_raw","server_host_key","server_host_key_sha256","rsa_prime","rsa_prime_length","rsa_generator","rsa_generator_length","rsa_public_key","rsa_public_key_length","rsa_exponent","rsa_modulus","rsa_length","dss_prime","dss_prime_length","dss_generator","dss_generator_length","dss_public_key","dss_public_key_length","dss_dsa_public_g","dss_dsa_public_p","dss_dsa_public_q","dss_dsa_public_y","ecdsa_curve25519","ecdsa_curve","ecdsa_public_key_length","ecdsa_public_key_b","ecdsa_public_key_gx","ecdsa_public_key_gy","ecdsa_public_key_n","ecdsa_public_key_p","ecdsa_public_key_x","ecdsa_public_key_y","ed25519_curve25519","ed25519_cert_public_key_nonce","ed25519_cert_public_key_bytes","ed25519_cert_public_key_raw","ed25519_cert_public_key_sha256","ed25519_cert_public_key_serial","ed25519_cert_public_key_type_id","ed25519_cert_public_key_type_name","ed25519_cert_public_key_keyid","ed25519_cert_public_key_principles","ed25519_cert_public_key_valid_after","ed25519_cert_public_key_valid_before","ed25519_cert_public_key_duration","ed25519_cert_public_key_sigkey_bytes","ed25519_cert_public_key_sigkey_raw","ed25519_cert_public_key_sigkey_sha256","ed25519_cert_public_key_sigkey_value","ed25519_cert_public_key_sig_raw","banner","userauth_methods","device_vendor","device_type","device_model","device_version","device_sector" -"2022-01-10 02:20:37","18.179.0.0","tcp",22,"ec2-18-179-0-0.ap-northeast-1.compute.amazonaws.com","ssh",16509,"JP","TOKYO","TOKYO",454110,,"SSH-2.0-OpenSSH_7.4","2.0","OpenSSH_7.4",,"bGjsifbPIDWT7tAu8BMjyg==","curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha256, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1","chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com, aes128-cbc, aes192-cbc, aes256-cbc, blowfish-cbc, cast128-cbc, 3des-cbc","umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1","none, zlib@openssh.com","curve25519-sha256@libssh.org","ecdsa-sha2-nistp256","aes128-ctr","hmac-sha2-256","none","AAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=","AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAABKAAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=","AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDSEHaLacthwB30rtA4xJgN3G9zXkCmm2WhV/TlNBrD20fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=","a6e4e1c16ba25d51bcddc58a6e16797144575dd18d02d9dedf75093d2b15c557",,,,,,,,,,,,,,,,,,,,"1xx7ASut7BF4ED8b592bebZBMBKTCzOsmbH4cjwx/0U=","P-256",256,"WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=","axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=","T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=","/////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=","/////wAAAAEAAAAAAAAAAAAAAAD///////////////8=","NIQdotpy2HAHfSu0DjEmA3cb3NeQKabZaFX9OU0GsPY=","0fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=",,,,,,,,,,,,,,,,,,,,"publickey",,,,, -"2022-01-10 02:20:37","170.10.0.0","tcp",22,"170-10-0-0.example.com","ssh",11976,"US","TEXAS","MARSHALL",,,"SSH-2.0-ARRIS_0.50","2.0","ARRIS_0.50",,"Y4RQS9sdRgEFwNJKVP6bZg==","diffie-hellman-group1-sha1","aes128-cbc, 3des-cbc, aes256-cbc, twofish256-cbc, twofish-cbc, twofish128-cbc, blowfish-cbc","hmac-sha1-96, hmac-sha1, hmac-md5","none","diffie-hellman-group1-sha1","ssh-rsa","aes128-cbc","hmac-sha1","none","LQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==","AAAAB3NzaC1yc2EAAACCLQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==","AAAAB3NzaC1yc2EAAAADAQABAAAAgwCDq1kGqqwdQVryCNcoyDbBpnL/okvM2d9NmR0OjprcToCZ2TZ5WUZt2BGwPE1QLJYskjhv7GwlfQ4qhEqHDg35wMrkO7j9LTQC7KW3xisOLuUil4FmMxPkol6s39945zBGpjw0l/BmJnUDlutxavkdd84fppFMwXNp2vbjxV1SYVc9","d53fedbfe92e631264629882b2e85bfd213ca4b07b824cd31f8de1fcb8d0ddcb",,,,,,,65537,"g6tZBqqsHUFa8gjXKMg2waZy/6JLzNnfTZkdDo6a3E6Amdk2eVlGbdgRsDxNUCyWLJI4b+xsJX0OKoRKhw4N+cDK5Du4/S00Auylt8YrDi7lIpeBZjMT5KJerN/feOcwRqY8NJfwZiZ1A5brcWr5HXfOH6aRTMFzadr248VdUmFXPQ==",1040,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"publickey, password","Arris",,,, -"2022-01-10 02:20:37","72.17.0.0","tcp",22,"072-017-0-0.example.com","ssh",33363,"US","FLORIDA","ORLANDO",517311,,"SSH-1.99-Cisco-1.25","1.99","Cisco-1.25",,"Z2fOfWsrLlh76Y0bOqa1cw==","diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1","aes128-cbc, 3des-cbc, aes192-cbc, aes256-cbc","hmac-sha1, hmac-sha1-96, hmac-md5, hmac-md5-96","none","diffie-hellman-group14-sha1","ssh-rsa","aes128-cbc","hmac-sha1","none","lrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=","AAAAB3NzaC1yc2EAAAIAlrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=","AAAAB3NzaC1yc2EAAAADAQABAAACAQDIVXBwKGhi35gabwHNZi6Bxls1BGtDVVZFhwvhTpJKTKhV4T2HnDFG7+FBpYejc92wH026Wf+uJHIpnKkVQRnnOV98zKXW68Tz+OnwT8aBQdLI+QYDC7wLwGRf+cOiXEAkpMrp2OJme+GwQ97oBccEwdu2j9vcYAFQ0+eCPNfwPrcZhwVb00kt/moLVSxWRdsDMzQiNDZf2zel+FQIAl5cCfaLSAQa1TIXy8SM13B0brnlpdyIqukQS0zUv/PL/6AsfhgLXeQBgjs1XIf6qL+ZdtQss5AKUDuJgrWDcS3nyNZQg/CAt8XdIsLntu3bCn+VGA1O/gUGLS1a9GoGd/lRArlmODNtbds74m7hxaAf/gzg0LFJx6HhwubmVCzTXEHl95KHYHKoDvCtUOgUm7zUugxWjhsLPfT6UfZCwvCY21SGVYsoEPiTT2DhuAFriM+PT83JresFHgZDosbqW0VCi2bzAKSBu/vphaqTbSdDo0xhkW9JCb3zUkW2ge/e/GrjxV4cNXRC9XQ/XYEIWmtF/gHSi0i9KweX4sN5TEkB/41vDvyDOdyPJ8Jta0I9vBolDwJ6qdMHOPlOW5oW83yCgbmUJNYkZ+MivABlc6iS/006qYiIwknHezbY5foYd8kDON7YAssOwCJcG5viII50Z1N9VsGkUv5sZMr2p9ry8Q==","06ff3cce443ed832927576d982b69d5a526d0e63334c72e87201deda61679406",,,,,,,65537,"yFVwcChoYt+YGm8BzWYugcZbNQRrQ1VWRYcL4U6SSkyoVeE9h5wxRu/hQaWHo3PdsB9Nuln/riRyKZypFUEZ5zlffMyl1uvE8/jp8E/GgUHSyPkGAwu8C8BkX/nDolxAJKTK6djiZnvhsEPe6AXHBMHbto/b3GABUNPngjzX8D63GYcFW9NJLf5qC1UsVkXbAzM0IjQ2X9s3pfhUCAJeXAn2i0gEGtUyF8vEjNdwdG655aXciKrpEEtM1L/zy/+gLH4YC13kAYI7NVyH+qi/mXbULLOQClA7iYK1g3Et58jWUIPwgLfF3SLC57bt2wp/lRgNTv4FBi0tWvRqBnf5UQK5ZjgzbW3bO+Ju4cWgH/4M4NCxSceh4cLm5lQs01xB5feSh2ByqA7wrVDoFJu81LoMVo4bCz30+lH2QsLwmNtUhlWLKBD4k09g4bgBa4jPj0/Nya3rBR4GQ6LG6ltFQotm8wCkgbv76YWqk20nQ6NMYZFvSQm981JFtoHv3vxq48VeHDV0QvV0P12BCFprRf4B0otIvSsHl+LDeUxJAf+Nbw78gzncjyfCbWtCPbwaJQ8CeqnTBzj5TluaFvN8goG5lCTWJGfjIrwAZXOokv9NOqmIiMJJx3s22OX6GHfJAzje2ALLDsAiXBub4iCOdGdTfVbBpFL+bGTK9qfa8vE=",4096,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"publickey, keyboard-interactive, password","Cisco",,,,"enterprise" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv deleted file mode 100644 index 0b125001b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","ssl_poodle","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","freak_vulnerable","freak_cipher_suite","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","tlsv13_support","tlsv13_cipher","jarm" -"2022-01-10 00:01:42","96.60.0.0",10443,"96-60-0-0.example.com","ssl,vpn","TLSv1.2",4181,"US","WISCONSIN","MILWAUKEE","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","N",1024,"FGT60D4614030700","support","2014-06-23 09:56:32","2038-01-19 03:14:07","5A:3D:FF:06:F9:E9:25:37:57:F9:09:52:33:A4:85:15:24:2D:88:7F","168CAE",2,"sha1WithRSAEncryption","rsaEncryption","Fortinet","FortiGate","US","California","Sunnyvale",,,,,"support@fortinet.com",,,"Fortinet","Certificate Authority","US","California","Sunnyvale",,,,,"support@fortinet.com",,,517311,,"N",,,"35:AB:B6:76:2A:3D:17:B2:FB:40:45:1B:FC:0A:99:0A:6E:48:57:F7:30:0A:3B:B1:1A:E6:99:70:5B:7C:32:41","88:7B:16:DB:39:44:0C:47:0E:4A:8F:0B:C5:FB:4D:45:BC:93:5A:00:43:A1:D9:7F:05:1D:86:33:02:F8:FC:57:67:A6:1D:C0:FF:F7:D2:40:D8:9A:21:AE:4E:6D:DC:E7:FF:72:BF:13:CB:EE:A7:5F:CD:83:EA:8A:5E:FB:87:DD","99:45:1F:2E:AE:EB:88:91:27:43:33:79:FA:93:7D:CA","HTTP/1.1",200,"OK","text/html",,,,"xxxxxxxx-xxxxx",131,,"Mon, 10 Jan 2022 00:01:44 GMT","Y","N","N","N","unknown","x509: unknown error",,, -"2022-01-10 00:01:42","113.160.0.0",10443,"","ssl","TLSv1.2",45899,"VN","THAI BINH","THAI BINH","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","N",2048,"1078-btb-tbi-HungHa-61d39c6d5a7e2","1078-btb-tbi-HungHa-61d39c6d5a7e2","2022-01-04 01:01:34","2023-02-06 01:01:34","A9:00:BB:E1:54:4D:56:54:59:F1:B7:EA:F1:1A:D5:36:5C:63:90:8E","36974C4C6B1B3785",2,"sha256WithRSAEncryption","rsaEncryption","pfSense webConfigurator Self-Signed Certificate",,,,,,,,,,,,"pfSense webConfigurator Self-Signed Certificate",,,,,,,,,,,,517311,,"N",,,"38:85:F0:44:1E:AD:84:B8:2F:43:68:BA:AC:EE:17:13:A4:BF:86:1D:48:75:7E:22:FA:08:4C:28:5F:AC:3E:5F","AE:1B:4F:D1:E4:C0:35:9D:2A:4F:7A:37:B8:7B:11:9D:84:25:23:21:AB:EF:B2:0F:DC:C9:F2:A3:72:28:92:E1:74:72:FA:E1:09:6C:E1:F6:B6:E3:A7:61:1C:58:89:34:D7:06:5C:3D:0A:A7:F6:CC:8A:D6:24:D0:04:4C:03:02","16:93:9A:F4:35:7F:9A:85:45:71:91:C7:7C:80:88:00","HTTP/1.1",200,"OK","text/html; charset=UTF-8","keep-alive",,"PHPSESSID=e15bdfa5739c36877608eb4cf46cc388; path=/; secure; HttpO","nginx",,"chunked","Mon, 10 Jan 2022 00:01:44 GMT","N","Y","N","N","unknown","x509: unknown error",,, -"2022-01-10 00:01:42","34.224.0.0",10443,"","ssl,vpn","TLSv1.2",14618,"US","VIRGINIA","ASHBURN","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","N",2048,"","Entrust Certification Authority - L1K","2021-10-07 15:30:28","2022-11-06 15:30:28","AD:19:B2:1C:CB:88:70:9B:DB:8E:7E:F5:65:50:13:D6:43:6C:BE:6E","7B388364A24B88E77E5553B5C6748100",2,"sha256WithRSAEncryption","rsaEncryption","Ciena Corporation",,"US","Maryland","Hanover",,,,,,,,"Entrust, Inc.","(c) 2012 Entrust, Inc. - for authorized use only","US",,,,,,,,,,454110,,"N",,"Retail Trade","9A:64:73:0B:8A:FA:DE:22:D4:6D:5A:C6:C4:6F:D4:A4:2A:28:FA:41:1E:FF:81:DC:D4:D9:00:FD:78:DF:C4:DD","9A:B7:BD:68:7D:F3:E7:C1:B7:D3:F4:2F:01:B6:C4:77:90:A3:2B:1E:C0:89:F5:08:EC:43:87:35:60:36:D4:87:61:AA:B8:A8:B3:8A:E9:F1:04:AA:5B:67:12:FF:63:D5:14:80:77:6E:8F:7D:C3:E2:3A:F3:13:DF:08:43:6C:B0","E7:34:BC:92:84:FA:39:DE:E1:46:6C:27:DA:5A:01:F4","HTTP/1.1",200,"OK","text/html",,,,"xxxxxxxx-xxxxx",131,,"Mon, 10 Jan 2022 00:01:44 GMT","Y","N","N","Y","OV",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv deleted file mode 100644 index ab28456b4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv +++ /dev/null @@ -1,46 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","freak_vulnerable","freak_cipher_suite","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","device_model","device_sector","device_type","device_vendor","device_version","jarm","page_sha256fp","raw_cert","raw_cert_chain","tlsv13_cipher","tlsv13_support" -"2018-04-23 13:25:21","198.51.100.232","443",,"ssl-freak","TLSv1.0","8447","AT","WIEN","VIENNA","TLS_RSA_WITH_RC4_128_SHA","1024","usg50_B0B2DC2FA69D","usg50_B0B2DC2FA69D","2012-05-10 00:01:19","2032-05-05 00:01:19","14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2","4FAB054F","sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,"0","0","Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5",,"57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1","E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87","1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE","HTTP/1.1","200","OK","text/html",,,,,,"chunked","Mon, 23 Apr 2018 13:25:26 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -"2018-04-23 13:25:26","198.51.100.224","443","198-51-100-224.example.net","ssl-freak","TLSv1.0","12577","AT","NIEDEROSTERREICH","BADEN","TLS_RSA_WITH_RC4_128_SHA","1024","usg20w_C86C870287EC","usg20w_C86C870287EC","2010-01-01 00:00:53","2029-12-27 00:00:53","14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2","4B3D3B35","sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,"0","0","Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5",,"57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1","E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87","1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE","HTTP/1.1","200","OK","text/html",,,,,,"chunked","Mon, 23 Apr 2018 13:25:29 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -2018-04-23 13:25:21,198.51.100.232,443,,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC2FA69D,usg50_B0B2DC2FA69D,2012-05-10 00:01:19,2032-05-05 00:01:19,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FAB054F,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:26 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:26,198.51.100.224,443,198-51-100-224.example.net,ssl-freak,TLSv1.0,12577,AT,NIEDEROSTERREICH,BADEN,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_C86C870287EC,usg20w_C86C870287EC,2010-01-01 00:00:53,2029-12-27 00:00:53,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4B3D3B35,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:29 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:33,198.51.100.67,443,,ssl-freak,TLSv1.0,8447,AT,NIEDEROSTERREICH,WAIDHOFEN AN DER THAYA,TLS_RSA_WITH_RC4_128_SHA,1024,Technicolor TG670,Technicolor TG670,2005-01-01 00:00:00,2024-12-31 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-7A2C610E,sha1WithRSAEncryption,rsaEncryption,Technicolor,1112WT0YK,,,,,,,,,,,Technicolor,1112WT0YK,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,,keep-alive,,xAuth_SESSION_ID=bm90aGluZyBoZXJlCg==; path=/;,,0,,"Mon, 23 Apr 2018 14:25:37 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:36,198.51.100.3,443,,ssl-freak,TLSv1.2,8445,AT,SALZBURG,HINTERGLEMM,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,2048,uag2100_04BF6D22A5A9,uag2100_04BF6D22A5A9,2016-03-08 20:27:08,2026-03-06 20:27:08,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B0F07D300BDB4FC4,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:39 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:38,198.51.100.198,443,198-51-100-198.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,2048,198-51-100-198.example.net,Go Daddy Secure Certificate Authority - G2,2016-12-29 08:51:00,2019-12-29 08:51:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,AEA6D3637023B56B,sha256WithRSAEncryption,rsaEncryption,,Domain Control Validated,,,,,,,,,,,198-51-100-198.example.net," Inc.""",http://certs.godaddy.com/repository/,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden ( The server,text/html,close,,,,2024,,,Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-23 13:25:38,198.51.100.98,443,198-51-100-98.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_FCF528743754,usg50_FCF528743754,2013-04-29 00:00:26,2033-04-24 00:00:26,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,517DB81A,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:41,198.51.100.156,443,198-51-100-156.example.net,ssl-freak,TLSv1.0,8339,AT,NIEDEROSTERREICH,SCHWECHAT,TLS_RSA_WITH_AES_128_CBC_SHA,1024,usg200_404A036775FC,usg200_404A036775FC,2010-05-01 00:04:04,2030-04-26 00:04:04,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4BDB6FF4,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:43 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:53,198.51.100.200,443,,ssl-freak,TLSv1.2,8447,AT,NIEDEROSTERREICH,KREMS AN DER DONAU,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB67FC6F,usg20_5CF4AB67FC6F,2015-12-02 00:00:47,2035-11-27 00:00:47,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,565E34AF,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:56 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:02,198.51.100.83,443,198-51-100-83.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_FCF5286F5972,usg20w_FCF5286F5972,2013-03-23 00:00:43,2033-03-18 00:00:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,514CF0AB,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:05 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:03,198.51.100.155,443,198-51-100-155.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-155.example.net,198-51-100-155.example.net,2018-03-19 19:47:07,2023-03-19 19:47:07,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2DF52AA905C7A2B44C2B9F0012FD5745,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden,text/html,,,,Microsoft-IIS/6.0,1939,,"Mon, 23 Apr 2018 13:11:52 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:03,198.51.100.129,443,198-51-100-129.example.net,ssl-freak,TLSv1.0,29654,AT,SALZBURG,SALZBURG,TLS_RSA_WITH_RC4_128_SHA,1024,localhost,localhost,2007-01-31 19:00:29,2008-01-31 19:00:29,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2,sha1WithRSAEncryption,rsaEncryption,Apache HTTP Server,Test Certificate,,,,,,,,,,,Apache HTTP Server,For testing purposes only,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,318,,"Mon, 23 Apr 2018 17:42:37 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:12,198.51.100.7,443,198-51-100-7.example.net,ssl-freak,TLSv1.0,8445,AT,SALZBURG,ALTENMARKT IM PONGAU,TLS_RSA_WITH_RC4_128_SHA,2048,IMM2-5cf3fcaf3abd,IMM2-5cf3fcaf3abd,2013-03-22 14:32:06,2023-03-20 14:32:06,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,D8C631398B585F10,sha1WithRSAEncryption,rsaEncryption,System X,,US,SomeState,SomeCity,,,,,,,,System X,,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,301,Moved Permanently,application/x-appweb-php,keep-alive,,,Mbedthis-Appweb/2.4.2,0,,"Mon, 23 Apr 2018 13:37:08 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:15,198.51.100.93,443,,ssl-freak,TLSv1.2,8447,AT,KARNTEN,SPITTAL AN DER DRAU,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC3308EF,usg50_B0B2DC3308EF,2012-05-25 00:00:39,2032-05-20 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FBECBA7,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:17 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:16,198.51.100.81,443,198-51-100-81.example.net,ssl-freak,TLSv1.0,5385,AT,VORARLBERG,FELDKIRCH,TLS_RSA_WITH_RC4_128_SHA,1024,usg100_5067F03642A5,usg100_5067F03642A5,2010-10-01 00:04:48,2030-09-26 00:04:48,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4CA525A0,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:19 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:17,198.51.100.162,443,198-51-100-162.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,rc1,Peppercon CA,2003-05-08 16:30:05,2008-05-06 16:30:05,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,18,md5WithRSAEncryption,rsaEncryption,,R&D,DE,SomeState,,,,,,198-51-100-162.example.net,,,,Security Department,DE,SomeState,SomeCity,,,,,198-51-100-162.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Redirect,,,,,,,,,N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:22,198.51.100.57,443,,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,GLEISDORF,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB661192,usg20_5CF4AB661192,2015-09-22 00:00:46,2035-09-17 00:00:46,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,56009A2E,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:25 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:28,198.51.100.146,443,198-51-100-146.example.net,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_RC4_128_SHA,1024,zywall_110_A0E4CB7CE5AF,zywall_110_A0E4CB7CE5AF,2015-01-26 17:19:56,2025-01-23 17:19:56,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,54C6773C,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:31 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:34,198.51.100.233,443,,ssl-freak,TLSv1.0,8447,AT,KARNTEN,KLAGENFURT AM WORTHERSEE,TLS_RSA_WITH_RC4_128_SHA,1024,198.51.100.174,198-51-100-174.example.net,2009-04-14 07:26:09,2025-04-15 07:26:09,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,A0571920C03C9EE0DA1168E586E0E8D440E42EA69D898AC829,sha1WithRSAEncryption,rsaEncryption,,General,DE,SomeState,SomeCity,,,,,198-51-100-174.example.net,,,,General,DE,SomeState,SomeCity,,,,,198-51-100-174.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,Close,,,LANCOM 1781A 8.50.0161 / 09.08.2011,,,,Y,N,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:35,198.51.100.106,443,198-51-100-106.example.net,ssl-freak,TLSv1.0,12793,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-106.example.net,SHT-Gruppe CA,2004-07-20 07:28:10,2006-07-20 07:38:10,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,190DBE75000000000007,sha1WithRSAEncryption,rsaEncryption,,,AT,SomeState,SomeCity,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/6.0,1508,,"Mon, 23 Apr 2018 13:26:37 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:37,198.51.100.191,443,,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,LEBRING,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB669448,usg20_5CF4AB669448,2015-10-01 00:00:38,2035-09-26 00:00:38,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,560C77A6,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:42,198.51.100.235,443,198-51-100-235.example.net,ssl-freak,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_107BEF33651A,usg50_107BEF33651A,2014-04-24 00:00:27,2034-04-19 00:00:27,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,5358541B,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:45 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:43,198.51.100.167,443,198-51-100-167.example.net,ssl-freak,TLSv1.0,8412,AT,BURGENLAND,ELTENDORF,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-167.example.net,198-51-100-167.example.net,2008-08-19 06:57:11,2010-08-19 06:57:11,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2,sha1WithRSAEncryption,rsaEncryption,SuSE Linux Web Server,web server,XY,unknown,unknown,,,,,198-51-100-167.example.net,,,SuSE Linux Web Server,CA,XY,SomeState,unknown,,,,,198-51-100-167.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache/2.2.3 (Linux/SUSE),80,,"Mon, 23 Apr 2018 13:26:45 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:47,198.51.100.42,443,198-51-100-42.example.net,ssl-freak,TLSv1.0,8437,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-42.example.net,iLO Default Issuer (Do not trust),2013-11-05 00:00:00,2028-11-04 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,72FD09EF,sha1WithRSAEncryption,rsaEncryption,,,US,SomeState,SomeCity,,,,,,,,,,US,SomeState,Houston,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,,,,,,,,,N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.177,443,198-51-100-177.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB625772,usg20_5CF4AB625772,2015-03-04 00:00:39,2035-02-27 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,54F64B27,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:50 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.66,443,198-51-100-66.example.net,ssl-freak,TLSv1.0,5385,AT,VORARLBERG,DORNBIRN,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-66.example.net,198-51-100-66.example.net,2009-10-06 11:23:48,2015-03-29 11:23:48,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,98B18BCD61B0CD5D,sha1WithRSAEncryption,rsaEncryption,,??,??,??,??,,,,,??,,,,??,??,??,??,,,,,??,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,close,,DSSignInURL=/; path=/; secure,,,,,Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.29,443,198-51-100-29.example.net,ssl-freak,TLSv1.0,6830,AT,NIEDEROSTERREICH,GUNTRAMSDORF,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_FCF52878354B,usg20_FCF52878354B,2013-05-20 00:00:39,2033-05-15 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,519967A7,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:50 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:49,198.51.100.235,443,,ssl-freak,TLSv1.0,8447,AT,TIROL,KITZBUHEL,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC3AEFE7,usg50_B0B2DC3AEFE7,2012-10-30 00:02:36,2032-10-25 00:02:36,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,508F191C,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:52 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:50,198.51.100.159,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-159.example.net,198-51-100-159.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-159.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-159.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:26:52 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:51,198.51.100.138,443,198-51-100-138.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_B0B2DC34A1F6,usg20_B0B2DC34A1F6,2012-06-16 00:00:58,2032-06-11 00:00:58,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FDBCCBA,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:54 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:52,198.51.100.64,443,,ssl-freak,TLSv1.0,1853,AT,OBEROSTERREICH,WILHERING,TLS_RSA_WITH_RC4_128_SHA,1024,198.51.100.171,198.51.100.117,2017-08-10 10:48:40,2020-08-09 10:48:40,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,598C3A08,sha1WithRSAEncryption,rsaEncryption,,,,,SomeCity,,,,,,,,,,,,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,5597,,"Mon, 23 Apr 2018 13:26:54 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:55,198.51.100.189,443,198-51-100-62.example.net,ssl-freak,TLSv1.0,25255,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_107BEF3A4C9E,usg20w_107BEF3A4C9E,2014-07-04 00:00:43,2034-06-29 00:00:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,53B5EEAB,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:57 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.17,443,198-51-100-17.example.net,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,SOEDING,TLS_RSA_WITH_AES_256_CBC_SHA,1024,Vimar By-Web,Vimar By-Web,2011-10-27 09:19:55,2016-10-25 09:19:55,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B82B13ED1FB0FD71,sha1WithRSAEncryption,rsaEncryption,,R&D,IT,SomeState,SomeCity,,,,,,,,,R&D,IT,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,text/html,keep-alive,,,nginx/0.6.32,,chunked,"Mon, 23 Apr 2018 13:26:56 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.111,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-111.example.net,198-51-100-111.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-111.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-111.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:26:58 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.179,443,198-51-100-179.example.net,ssl-freak,TLSv1.0,12605,AT,OBEROSTERREICH,LINZ,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB665FB9,usg20_5CF4AB665FB9,2015-09-25 00:00:42,2035-09-20 00:00:42,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,56048EAA,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:58 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:58,198.51.100.143,443,,ssl-freak,TLSv1.0,8447,AT,KARNTEN,GLAN,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_FCF5285DEDC4,usg20_FCF5285DEDC4,2012-11-09 00:00:44,2032-11-04 00:00:44,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,509C47AC,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:00 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:58,198.51.100.111,443,198-51-100-111.example.net,ssl-freak,TLSv1.0,1901,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,*.*,198-51-100-111.example.net,2009-01-16 12:51:43,2010-01-16 12:51:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6,md5WithRSAEncryption,rsaEncryption,,,IL,SomeState,,,,,,,,,,Visonic CA,IL,SomeState,,,,,,198-51-100-111.example.net,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html,close,,PowerLink=226002836046b4bddcd2d16b809f76d9; path=/,Apache/1.3.31 (Unix) PHP/4.3.9 mod_ssl/2.8.20 Open,,chunked,"Wed, 23 Jan 2002 10:17:09 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:59,198.51.100.79,443,,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB65A17C,usg20_5CF4AB65A17C,2015-09-01 00:00:51,2035-08-27 00:00:51,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,55E4EAB3,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:02 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:59,198.51.100.90,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-90.example.net,198-51-100-90.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-90.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-90.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:27:02 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.186,443,198-51-100-186.example.net,ssl-freak,TLSv1.0,31125,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-186.example.net,198-51-100-186.example.net,2013-07-11 12:20:19,2021-07-09 12:20:19,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,D947ED19BEAB28E6,sha1WithRSAEncryption,rsaEncryption,,IT,PL,SomeState,SomeCity,,,,,198-51-100-186.example.net,,,,IT,PL,SomeState,SomeCity,,,,,198-51-100-186.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,401,Unauthorized,text/plain,close,"Basic realm=""example.com""",,Microsoft-IIS/7.5,0,,"Mon, 23 Apr 2018 14:03:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.150,443,198-51-100-150.example.net,ssl-freak,TLSv1.0,8559,AT,BURGENLAND,NEUSIEDL AM SEE,TLS_ECDHE_RSA_WITH_RC4_128_SHA,2048,198-51-100-150.example.net,COMODO RSA Domain Validation Secure Server CA,2017-02-08 00:00:00,2019-05-09 23:59:59,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B6EF6CF436532F0252627393BD7311FD,sha256WithRSAEncryption,rsaEncryption,,Domain Control Validated,,,,,,,,,,,,,GB,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:06 GMT",N,N,N,N,DV,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.141,443,198-51-100-141.example.net,ssl-freak,TLSv1.0,39372,AT,OBEROSTERREICH,HINTERSTODER,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-141.example.net,iLO Default Issuer (Do not trust),2014-01-14 00:00:00,2029-01-13 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7852761B,sha1WithRSAEncryption,rsaEncryption,,,US,SomeState,SomeCity,,,,,,,,,,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,,,,,,,,,N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:27:04,198.51.100.194,443,198-51-100-194.example.net,ssl-freak,TLSv1.0,8447,AT,KARNTEN,GLAN,TLS_RSA_WITH_RC4_128_SHA,1024,iDRAC6 default certificate,iDRAC6 default certificate,2009-09-17 22:47:28,2019-09-15 22:47:28,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,1,sha1WithRSAEncryption,rsaEncryption,,Remote Access Group,US,SomeState,SomeCity,,,,,,,,,Remote Access Group,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,,keep-alive,,,Mbedthis-Appweb/2.4.2,0,,"Mon, 23 Apr 2018 13:25:57 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -"2022-02-07 00:01:41","2.136.0.0",10443,"2-136-0-0.example.com","ssl,ssl-freak,ssl-poodle,vpn","TLSv1.0",12345,"ES","MADRID","MADRID","TLS_RSA_WITH_RC4_128_SHA",1024,"usg50_107BEF336340","usg50_107BEF336340","2014-04-24 00:00:32","2034-04-19 00:00:32","F5:04:98:CD:D4:67:13:E1:77:B7:38:D4:B9:43:C0:72:50:6C:0D:58",53585420,"sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,517311,,"Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5","Communications, Service Provider, and Hosting Service","AF:3A:71:B7:1B:A2:62:4E:87:22:FF:19:3F:84:1F:7F:CC:DC:06:E0:AF:80:E2:5D:33:A5:68:9A:E3:81:25:45","14:92:CC:6B:C7:B3:09:31:50:8C:1C:8D:5B:FD:D1:BE:41:78:80:97:E0:10:11:48:1F:EE:D6:CB:4F:F0:13:D5:05:56:AC:BA:12:12:02:F7:0F:03:40:95:17:8A:5F:79:98:E1:44:EF:E6:5A:44:E3:AC:3A:F8:49:F7:AC:B6:52","E8:5F:96:16:3F:76:35:F0:07:4F:4C:2C:38:FC:27:6B","HTTP/1.1",200,"OK","text/html",,,,"",,"chunked","Mon, 07 Feb 2022 00:01:43 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,"Zyxel","firewall","ZyWALL USG 50",,"enterprise", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv deleted file mode 100644 index 4bcc6758a..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv +++ /dev/null @@ -1,32 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","ssl_poodle","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","tlsv13_support","tlsv13_cipher","device_model","device_sector","device_type","device_vendor","device_version","jarm","page_sha256fp","raw_cert","raw_cert_chain" -"2018-08-08 00:51:42","203.0.113.85",8443,"example.com","ssl-poodle","TLSv1.0",65540,"AT","WIEN","VIENNA","TLS_RSA_WITH_RC4_128_SHA","Y",1024,"usg20_107BEF394BA5","usg20_107BEF394BA5","2014-06-25 00:00:42","2034-06-20 00:00:42","04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3","53AA112A",2,"sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,0,0,,"16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E","0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE","33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC","HTTP/1.1",200,"OK","text/html",,,,,,"chunked","Wed, 08 Aug 2018 00:51:44 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -2018-04-19 13:32:27,198.51.100.147,443,,ssl-poodle,TLSv1.0,8445,AT,SALZBURG,SALZBURG,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-147.example.net,some_issuer,2017-09-18 08:22:17,2019-09-18 08:22:17,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,746481F100000000000C,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,Tirol,Ehrwald,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:32 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:32,198.51.100.207,443,198-51-100-94.example.net,ssl-poodle,TLSv1.0,25255,AT,SALZBURG,SALZBURG,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2004-06-03 11:11:43,2024-05-29 11:11:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,2,md5WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,"rg_cookie_session_id=1654544029; path=/; expires=Fri, 01 Jan 2038",,,,"Thu, 19 Apr 2018 13:32:34 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:32,198.51.100.200,443,198-51-100-200.example.net,ssl-poodle,TLSv1.2,8445,AT,SALZBURG,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-200.example.net,198-51-100-200.example.net,2016-10-01 14:09:12,2020-10-02 14:09:12,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2E8C9E4A2C7D3EDC,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,,AT,,,,,,,,,,some_org_name,,AT,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=utf-8,close,,,,,,,N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:33,198.51.100.239,443,198-51-100-239.example.net,ssl-poodle,TLSv1.0,8437,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198-51-100-239.example.net,198-51-100-239.example.net,2011-07-27 13:30:18,2012-07-26 13:30:18,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7C91,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,--,SomeState,SomeCity,,,,,198-51-100-239.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-239.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden,text/html; charset=UTF-8,close,,,Apache/2.2.3 (CentOS),4958,,"Thu, 19 Apr 2018 13:32:35 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:35,198.51.100.156,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2010-01-01 00:00:52,2029-12-27 00:00:52,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4B3D3B34,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Thu, 19 Apr 2018 13:32:37 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:36,198.51.100.122,443,198-51-100-122.example.net,ssl-poodle,TLSv1.2,36351,AT,AUSTRIA,?,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,2048,198-51-100-122.example.net,COMODO RSA Organization Validation Secure Server CA,2017-04-06 00:00:00,2019-04-06 23:59:59,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,CAB81F32F3FF4766BC545A2C14DF34B5,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,Wien,Wien,,1130,,,,,,COMODO CA Limited,,GB,Greater Manchester,Salford,,,,,,,,518210,737401,Information Technology,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,0,,"Thu, 19 Apr 2018 13:32:20 GMT",Y,N,N,Y,OV,,,,,,,,,,,, -2018-04-19 13:32:37,198.51.100.58,443,198-51-100-58.example.net,ssl-poodle,TLSv1.2,12605,AT,OBEROSTERREICH,LINZ,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,example,some_issuer,2015-01-17 16:11:24,2020-01-17 16:11:24,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6D9E2D4443F1D69E4A8865CC1C5B6963,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/8.5,701,,"Thu, 19 Apr 2018 13:34:53 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:38,198.51.100.18,443,198-51-100-18.example.net,ssl-poodle,TLSv1.2,6830,AT,OBEROSTERREICH,LINZ,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,2048,198-51-100-18.example.net,TERENA SSL CA 3,2017-07-14 00:00:00,2020-07-22 12:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0386AD387BEC13878473D23C8C786ECE,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,,Linz,,,,,,,,TERENA,,NL,Noord-Holland,Amsterdam,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,,Close,,BNIS_ChallengeState=Bqyd+IQebjQwiiYNKBJkA5Ta0spL1gX5; Path=/; Exp,,61,,,Y,N,N,Y,OV,,,,,,,,,,,, -2018-04-19 13:32:38,198.51.100.246,443,,ssl-poodle,TLSv1.2,8447,AT,SALZBURG,SALZBURG,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,example,some_issuer,2014-09-01 16:18:46,2054-08-24 16:18:46,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,129FA64A4BE039B54E850F1AA65AD835,2,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,,,ASP.NET_SessionId=e3qfk1dfz2mtqwzoym3gul3r; path=/; HttpOnly,Microsoft-IIS/8.5,145,,"Thu, 19 Apr 2018 13:32:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.35,443,198-51-100-35.example.net,ssl-poodle,TLSv1.0,12605,AT,OBEROSTERREICH,LINZ,TLS_RSA_WITH_AES_128_CBC_SHA,Y,2048,198-51-100-35.example.net,Go Daddy Secure Certificate Authority - G2,2017-08-28 13:29:01,2018-09-10 06:28:49,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,90B22B4CEF57C0FC,2,sha256WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,198-51-100-35.example.net,,,,,US,Arizona,Scottsdale,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,266,,"Thu, 19 Apr 2018 13:35:03 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.142,443,,ssl-poodle,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,Y,2048,198.51.100.19,198-51-100-19.example.net,2014-12-11 09:57:33,2024-12-08 09:57:33,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,A0571DCBE5E1A2C062D8FB7001271581B5F69824157E385563FA23527E0B,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,DE,,,,,,,198-51-100-19.example.net,,,some_org_name,Engineering,DE,NRW,Wuerselen,,,,,198-51-100-19.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,Close,,,LANCOM,,,"Thur, 19 Apr 2018 13:32:41 GMT",Y,N,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.178,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2012-05-30 00:00:44,2032-05-25 00:00:44,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FC5632C,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Thu, 19 Apr 2018 13:32:41 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:40,198.51.100.99,443,198-51-100-99.example.net,ssl-poodle,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-99.example.net,RapidSSL RSA CA 2018,2018-03-30 00:00:00,2019-04-29 12:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0BDCB5D6D4C22BD2A1CF55584B6DE09C,2,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,DigiCert Inc,198-51-100-99.example.net,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,404,Not Found,text/html; charset=us-ascii,close,,,Microsoft-HTTPAPI/2.0,315,,"Thu, 19 Apr 2018 13:32:43 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:40,198.51.100.235,443,198-51-100-235.example.net,ssl-poodle,TLSv1.0,25255,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,1024,Nextcloud,Nextcloud,2016-12-13 20:28:39,2017-01-12 20:28:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,CDE5769D28C80B6B,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,,AU,Some-State,,,,,,,,,Internet Widgits Pty Ltd,,AU,Some-State,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,400,Bad Request,text/html; charset=UTF-8,close,,nc_sameSiteCookiestrict=true; path=/; httponly;secure; expires=Fr,Apache/2.4.10 (FreeBSD) OpenSSL/0.9.8zd-freebsd PH,6939,,"Thu, 19 Apr 2018 13:32:42 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:41,198.51.100.187,443,198-51-100-187.example.net,ssl-poodle,TLSv1.2,28760,AT,OBEROSTERREICH,,TLS_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-187.example.net,Go Daddy Secure Certificate Authority - G2,2018-02-12 17:56:01,2020-02-12 17:56:01,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,03BA30FF4972177C,2,sha256WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,198-51-100-187.example.net,,,,,US,Arizona,Scottsdale,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,400,No parameters passed t,text/html,,,,Microsoft-IIS/10.0,11,,"Thu, 19 Apr 2018 13:32:42 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:42,198.51.100.213,443,198-51-100-213.example.net,ssl-poodle,TLSv1.2,8447,AT,OBEROSTERREICH,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-213.example.net,some_issuer,2016-09-22 08:12:17,2018-09-22 08:12:17,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,770000000EBB9429663601BAB700000000000E,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,301,Moved Permanently,,close,,,Microsoft-IIS/8.5,0,,"Thu, 19 Apr 2018 13:32:44 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:42,198.51.100.74,443,198-51-100-74.example.net,ssl-poodle,TLSv1.0,62363,AT,STEIERMARK,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,DC,DC,2016-12-30 17:15:38,2021-12-30 17:15:38,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7753CCEB55990A834E15DAC5707D403A,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:44 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:43,198.51.100.145,443,198-51-100-145.example.net,ssl-poodle,TLSv1.0,8447,AT,KARNTEN,KLAGENFURT AM WORTHERSEE,TLS_RSA_WITH_RC4_128_SHA,Y,1024,localdomain,localdomain,2008-10-07 20:12:54,2018-10-07 20:12:54,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,91B04FFCF174CCFF,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,,CA,,,,,,,198-51-100-145.example.net,,,some_org_name,,CA,Quebec,Gatineau,,,,,198-51-100-145.example.net,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,302,Found,text/html; charset=UTF-8,close,,"HOMEBASEID=658512b32961b9b6f8df7a3d4de7fa01; expires=Tue, 19-Jan-",Apache/2.2.3 (Red Hat),0,,"Thu, 19 Apr 2018 12:52:32 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:44,198.51.100.48,443,198-51-100-48.example.net,ssl-poodle,TLSv1.0,1901,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198-51-100-48.example.net,198-51-100-48.example.net,2013-06-15 20:10:49,2023-06-15 20:10:49,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,013F49762DAE,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,198-51-100-48.example.net,,,Western Digital,Branded Products,US,CS,Mountain View,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,225,,"Thu, 19 Apr 2018 03:08:06 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:45,198.51.100.94,443,198-51-100-94.example.net,ssl-poodle,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-94.example.net,RapidSSL CA,2013-04-03 17:02:33,2014-04-07 03:32:33,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0B697D,2,sha1WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,,,KtAjvog6HgAsml0cyxE4hpc9kv8dhgWZ,"GeoTrust, Inc.",,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,,,ASP.NET_SessionId=z5lph4ufefkvg1xzmd4q2m33; path=/; HttpOnly,Microsoft-IIS/8.0,144,,"Thu, 19 Apr 2018 13:32:48 GMT",Y,N,Y,N,unknown,x509: certificate has expired or is not yet valid,,,,,,,,,,, -2018-04-19 13:32:45,198.51.100.53,443,198-51-100-53.example.net,ssl-poodle,TLSv1.0,8447,AT,TIROL,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2008-11-13 13:47:18,2028-11-08 13:47:18,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,BE2B43544C0AFF2E,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,DE,,,,,,,198-51-100-53.example.net,,,some_org_name,some_org_name,DE,Niedersachsen,38162 Cremlingen (OT Schandelah),,,,,198-51-100-53.example.net,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=iso-8859-1;,,,,GoAhead-Webs,,,,Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:46,198.51.100.56,443,198-51-100-56.example.net,ssl-poodle,TLSv1.0,8445,AT,TIROL,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-56.example.net,some_issuer,2016-11-28 08:05:12,2018-11-28 08:05:12,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,637D34F100010000000E,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:49 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:46,198.51.100.82,443,198-51-100-82.example.net,ssl-poodle,TLSv1.0,6830,AT,OBEROSTERREICH,,TLS_RSA_WITH_AES_128_CBC_SHA,Y,1024,123AFG,7426AC8186F3,2011-01-01 00:00:06,2020-12-29 00:00:06,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,8186F3,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,,,,"Cisco Systems, Inc.",some_org_name,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html,close,,,Embedded HTTP Server.,107,,"Sat, 01 Jan 2011 00:00:45 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:48,198.51.100.29,443,198-51-100-29.example.net,ssl-poodle,TLSv1.0,6830,AT,STEIERMARK,GRAZ,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198.51.100.43,198.51.100.22,2018-04-18 13:32:09,2038-01-15 13:32:09,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,862D98F4B99D0042,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html; charset=utf-8,,,,,,,,Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.114,443,198-51-100-114.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_AES_256_CBC_SHA,Y,1024,198-51-100-114.example.net,198-51-100-114.example.net,2009-08-25 17:47:57,2019-05-25 17:47:57,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,FB09C7848A7F4D77,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,,AT,Vienna,Vienna,,,,,198-51-100-114.example.net,,,Digispectrum,,AT,Vienna,Vienna,,,,,198-51-100-114.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=utf-8,close,,b69223925949d45306d32f1a3d23c011=6a01vehilfpml41pl3pq3oth52; path,Apache/2.2.3 (CentOS),,chunked,"Thu, 19 Apr 2018 13:32:52 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.11,443,198-51-100-11.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,2048,FGT60C3G12019794,FGT60C3G12019794,2012-08-10 07:17:11,2022-08-11 07:17:11,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-6CD83A89,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,,,,,,,,,,,,Fortinet Ltd.,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,79,,"Thu, 19 Apr 2018 13:32:08 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.49,443,198-51-100-49.example.net,ssl-poodle,TLSv1.2,8447,AT,NIEDEROSTERREICH,,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,1024,localhost,localhost,2009-11-10 23:48:47,2019-11-08 23:48:47,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B5C752C98781B503,0,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache/2.4.10 (Win32) OpenSSL/1.0.1i PHP/5.5.15,2190,,"Thu, 19 Apr 2018 13:32:55 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:50,198.51.100.236,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_AES_128_CBC_SHA,Y,1024,example,some_issuer,2013-01-30 12:00:08,2023-01-28 12:00:08,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-462A1420,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,,,,Netgear Inc.,Netgear Prosafe,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html,close,,,Embedded HTTP Server.,107,,"Sat, 01 Jan 2011 00:00:21 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:50,198.51.100.224,443,198-51-100-224.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-224.example.net,some_issuer,2017-08-03 10:21:50,2019-08-03 10:21:50,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6126D181000300000041,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,401,Unauthorized,text/html,,NTLM,,Microsoft-IIS/7.5,1344,,"Thu, 19 Apr 2018 13:32:52 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -"2022-02-07 00:01:41","206.162.0.0",10443,,"ssl,ssl-poodle,vpn","TLSv1.2",12345,"CA","BRITISH COLUMBIA","BURNABY","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","Y",1024,"FWF60D4615000455","support","2015-01-28 18:14:33","2038-01-19 03:14:07","C9:B0:4E:B7:79:94:B4:DD:A7:15:21:86:43:F9:6E:4B:C9:A2:87:D9","1CA40F",2,"sha1WithRSAEncryption","rsaEncryption","Fortinet","FortiGate","US","California","Sunnyvale",,,,,"support@fortinet.com",,,"Fortinet","Certificate Authority","US","California","Sunnyvale",,,,,"support@fortinet.com",,,517311,,"Communications, Service Provider, and Hosting Service","38:F7:E0:92:24:8C:CB:28:43:93:0B:91:17:30:B1:41:8F:4E:2D:E5:A8:93:AE:4D:FE:53:00:D3:0E:53:02:16","0C:F0:37:3F:A8:93:AE:4D:FE:53:00:D3:2A:E6:6D:0B:02:9D:B9:46:58:A6:9E:5A:35:40:FB:62:9C:81:47:0A:4F:15:5D:53:D9:2F:36:4A:0B:3B:10:61:A9:07:EE:94:EC:00:B8:9C:F7:E0:92:24:8C:CB:28:2C:DD:E7:07:C6","8A:B3:08:20:34:79:94:B4:DD:A7:36:D7:14:6E:33:50","HTTP/1.1",200,"OK","text/html",,,,,131,,"Mon, 07 Feb 2022 00:01:43 GMT","Y","N","N","N","unknown","x509: unknown error",,,,,,"Fortinet","firewall","FortiGate",,"enterprise", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv deleted file mode 100644 index fd671ec90..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","transaction_id","magic_cookie","message_length","message_type","mapped_family","mapped_address","mapped_port","xor_mapped_family","xor_mapped_address","xor_mapped_port","software","fingerprint","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,3478,node01.example.com,stun,64512,ZZ,Region,City,0,0,,000000000000000000000000,2112a442,88,0101,01,192.168.0.1,3243,01,192.168.0.1,3243,"Coturn-4.5.1.1 'dan Eider'",0xfaedd06e,5.40,108 -"2010-02-10 00:00:01",192.168.0.2,udp,3478,node02.example.com,stun,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",000000000000000000000000,2112a442,88,0101,01,51.77.39.195,45877,01,192.168.0.2,45877,"Coturn-4.5.1.1 'dan Eider'",0x21128641,5.40,108 -"2010-02-10 00:00:02",192.168.0.3,udp,3478,node03.example.com,stun,64512,ZZ,Region,City,0,0,,000000000000000000000000,2112a442,76,0101,01,192.168.0.3,16321,01,188.68.240.32,16321,"ApolloProxy-1.20.1.28 'sunflower'",,4.80,96 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv deleted file mode 100644 index 8f6355491..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sequence_number","ack_number","window_size","urgent_pointer","tcp_flags","raw_packet","sector" -"2022-01-10 09:18:23","66.9.0.0","tcp",80,,"synfulknock",18885,"US","NEW JERSEY","JERSEY CITY",,,0,791102,8192,0,4608,"3cfdfec601e4700f6a9a2000080045000034c3780000f706789442099555b869f7ee0050b20800000000000c123e8012200002aa0000020405b40101040201030305", -"2022-01-10 09:19:17","213.131.0.0","tcp",80,"host-213-131-55-210-customer.wanex.net","synfulknock",35805,"GE","TBILISI","TBILISI",,,0,791102,8192,0,4608,"90e2baaf0b84700f6a9a200008004500003434100000f2064382d58337d2b8698b720050916200000000000c123e8012200059d50000020405b40101040201030305", -"2022-01-10 09:27:39","213.178.0.0","tcp",80,,"synfulknock",29256,"SY","DIMASHQ","DAMASCUS",,,0,791102,8192,0,4608,"90e2bab9cfd4700f6a9a20000800450000340f1d0000ea068bdad5b2e6914a522f360050eb5200000000000c123e801220001b4a0000020405b40101040201030305" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv deleted file mode 100644 index 3309e9a3d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" -"2019-09-04 12:27:34","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.5|Login:" -"2019-09-04 12:27:40","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.45.3 (stable)|Login:" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv deleted file mode 100644 index 3dde133d4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","opcode","errorcode","error","errormessage","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,35067,node01.example.com,tftp,64512,ZZ,Region,City,0,0,5,0,"Not defined","Get not supported",22,1.57 -"2010-02-10 00:00:01",192.168.0.2,udp,56709,node02.example.com,tftp,64512,ZZ,Region,City,0,0,5,1,"File not found","File not found",19,1.36 -"2010-02-10 00:00:02",192.168.0.3,udp,32785,node03.example.com,tftp,64512,ZZ,Region,City,0,0,5,2,"Access violation","Access violation",21,1.50 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv deleted file mode 100644 index efeab02c4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","mac","radioname","essid","modelshort","modelfull","firmware","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,10001,node01.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,00156db98c3a,kachine.meta.lidia.tereixa,Kachine-Meta-Lidia-Tereixa,NS5,,XS5.ar2313.v3.5.4494.091109.1459,148,37.00 -"2010-02-10 00:00:01",192.168.0.2,udp,10001,node02.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,00156d7c9188,adana.mason.lanikai.ozaner,Adana-Mason-Lanikai-Ozaner,LM5,"NanoStation Loco M5",XM.ar7240.v5.6.3.28591.151130.1749,156,39.00 -"2010-02-10 00:00:02",192.168.0.3,udp,10001,node03.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,0418d6000fd5,tailynn.kadija.noreen.dinkar,Tailynn-Kadija-Noreen-Dinkar,P2B-400,"PowerBeam M2 400",XW.ar934x.v5.6.5.29033.160515.2108,145,36.25 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv deleted file mode 100644 index 000f5ed42..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","asn","geo","region","city","naics","sic","product","banner","sector" -"2019-09-04 14:51:44","198.123.245.53",5678,,5678,"AA","LOCATION","LOCATION",0,0,"Apple remote desktop vnc","RFB 003.889", -"2019-09-04 14:51:44","198.123.245.112",5678,"localhost.localdomain",5678,"AA","LOCATION","LOCATION",517311,0,"RealVNC Enterprise v5.3 or later","RFB 005.000", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv deleted file mode 100644 index 7e279ca3e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","response_size","amplification","error","raw_response" -"2010-02-10 00:00:00",192.168.0.1,udp,3702,node01.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",989,164.83,"Validation constraint violation: SOAP message expected",c2FtcGxlIHJlc3BvbnNlIGRhdGEK -"2010-02-10 00:00:01",192.168.0.2,udp,3702,node02.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",918,183.60,"Validation constraint violation: missing root element",c2FtcGxlIHJlc3BvbnNlIGRhdGEK -"2010-02-10 00:00:02",192.168.0.3,udp,3702,node03.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",989,197.80,"Validation constraint violation: SOAP message expected",c2FtcGxlIHJlc3BvbnNlIGRhdGEK diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license deleted file mode 100644 index 9f58c89ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv deleted file mode 100644 index 7e83bbaf8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","opcode","reported_hostname","status","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,177,node01.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node01.example.com,"Linux 3.0.101-100-default",44,6.29 -"2010-02-10 00:00:01",192.168.0.2,udp,47074,node02.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node02.example.com,"Linux 2.6.9-103.ELsmp",48,6.86 -"2010-02-10 00:00:02",192.168.0.3,udp,177,node03.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node03.example.com,"1 user, load: 6,5, 6,6, 6,6",46,6.57 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv deleted file mode 100644 index 2e7b59158..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","protocol","asn","geo","region","city","hostname","naics","sector","tag","public_source","status","detail","method","device_vendor" -"2010-02-10 00:00:00",192.168.0.1,,,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",cyclops-blink,,"likely compromised",,, -"2010-02-10 00:00:01",192.168.0.2,,,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",cyclops-blink,,"likely compromised",,, -"2010-02-10 00:00:02",192.168.0.3,,,64512,ZZ,Region,City,node03.example.com,0,"Professional, Scientific, and Technical Services",cyclops-blink,,"likely compromised",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later From 2dec6ec3ed4e693a539c52a9ce5bb3424b040465 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:57:12 +0000 Subject: [PATCH 02/67] remove json parser - csv provides better performance --- .../shadowserver/collector_reports_api.py | 7 +- .../bots/parsers/shadowserver/parser_json.py | 171 ------------------ .../test_collector_reports_api.py | 7 +- 3 files changed, 7 insertions(+), 178 deletions(-) delete mode 100644 intelmq/bots/parsers/shadowserver/parser_json.py diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 5e0b045c8..dc8bd6b42 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv' or 'json'). The default is 'json' for compatibility. Using 'csv' is recommended for best performance. + file_format (str): File format to download ('csv'). The 'json' option is not longer supported. """ country = None @@ -67,11 +67,10 @@ def init(self): self._report_list.append(self.country) if self.file_format is not None: - if not (self.file_format == 'csv' or self.file_format == 'json'): + if not (self.file_format == 'csv'): raise ValueError('Invalid file_format') else: - self.file_format = 'json' - self.logger.info("For best performance, set 'file_format' to 'csv' and use intelmq.bots.parsers.shadowserver.parser.") + self.file_format = 'csv' self.preamble = f'{{ "apikey": "{self.api_key}" ' diff --git a/intelmq/bots/parsers/shadowserver/parser_json.py b/intelmq/bots/parsers/shadowserver/parser_json.py deleted file mode 100644 index 893ad877b..000000000 --- a/intelmq/bots/parsers/shadowserver/parser_json.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -Shadowserver JSON Parser - -SPDX-FileCopyrightText: 2020 Intelmq Team -SPDX-License-Identifier: AGPL-3.0-or-later -""" -import re -from typing import Any - -from intelmq.lib.bot import ParserBot -from intelmq.lib.exceptions import InvalidKey, InvalidValue -import intelmq.lib.message as libmessage -import intelmq.bots.parsers.shadowserver._config as config - - -class ShadowserverJSONParserBot(ParserBot): - """Parse all Shadowserver feeds in JSON format (data coming from the reports API) - Shadowserver JSON Parser - - Parameters: - feedname (str): The name of the feed - """ - __is_filename_regex = re.compile(r'^(?:\d{4}-\d{2}-\d{2}-)?(\w+)(-\w+)*\.json$') - feedname = None - _sparser_config = None - recover_line = ParserBot.recover_line_json - overwrite = True - - def init(self): - if self.feedname is not None: - feedname = self.feedname - self._sparser_config = config.get_feed_by_feedname(feedname) - if self._sparser_config: - self.logger.info('Using fixed feed name %r for parsing reports.', feedname) - else: - self.logger.info('Could not determine the feed by the feed name %r given by parameter. ' - 'Will determine the feed from the file names.', feedname) - - def parse(self, report): - report_name = report.get('extra.file_name') - if not report_name: - raise ValueError("No feedname given as parameter and the " - "processed report has no 'extra.file_name'. " - "Ensure that at least one is given. " - "Also have a look at the documentation of the bot.") - - filename_search = self.__is_filename_regex.search(report_name) - - if not filename_search: - raise ValueError(f"Report's 'extra.file_name' {report_name!r} is not valid.") - report_name = filename_search.group(1) - - self.logger.debug("Detected report's file name: %s.", report_name) - retval = config.get_feed_by_filename(report_name) - - if not retval: - raise ValueError('Could not get a config for {!r}, check the documentation.' - ''.format(report_name)) - self.feedname, self._sparser_config = retval - - return self.parse_json(report) - - def parse_line(self, line: Any, report: libmessage.Report): - conf = self._sparser_config - processedkeys = [] - - event = self.new_event(report) - event.add('feed.name', self.feedname, overwrite=self.overwrite) - - extra = {} - - for entry in conf.get('required_fields'): - intelmqkey, shadowserverkey = entry[0], entry[1] - value = self.get_value_from_config(line, entry) - - if value is not None: - event.add(intelmqkey, value) - processedkeys.append(shadowserverkey) - - # Now add optional fields. - # This action may fail, the value is added to - # extra if an add operation failed - for entry in conf.get('optional_fields'): - intelmqkey, shadowserverkey = entry[0], entry[1] - try: - value = self.get_value_from_config(line, entry) - except ValueError: - self.logger.warning('Optional key %s not found in feed %s. Possible change in data' - ' format or misconfiguration.', shadowserverkey, self.feedname) - continue - - intelmqkey, shadowserverkey = entry[0], entry[1] - if value is not None: - if intelmqkey == 'extra.': - extra[shadowserverkey] = value - processedkeys.append(shadowserverkey) - continue - elif intelmqkey and intelmqkey.startswith('extra.'): - extra[intelmqkey.replace('extra.', '', 1)] = value - processedkeys.append(shadowserverkey) - continue - elif intelmqkey is False: - # ignore it explicitly - processedkeys.append(shadowserverkey) - continue - try: - event.add(intelmqkey, value) - processedkeys.append(shadowserverkey) - except InvalidValue: - self.logger.debug('Could not add key %r in feed %r, adding it to extras.', - shadowserverkey, self.feedname) - except InvalidKey: - extra[intelmqkey] = value - processedkeys.append(shadowserverkey) - else: - processedkeys.append(shadowserverkey) - - # Now add additional constant fields. - event.update(conf.get('constant_fields', {})) - - event.add('raw', self.recover_line_json(line)) - - # Add everything which could not be resolved to extra. - for key in line: - if key not in processedkeys: - val = line[key] - if not val == "": - extra[key] = val - - if extra: - event.add('extra', extra) - - yield event - - def get_value_from_config(self, data, entry): - """ - Given a specific config, get the value for that data based on the entry - """ - conv_fun = None - - shadowserverkey = entry[1] - raw_value = data.get(shadowserverkey, None) - value = raw_value - - if raw_value is None: - raise ValueError('Key {!r} not found in feed {!r}. Possible change in data' - ' format or misconfiguration.'.format(shadowserverkey, self.feedname)) - if len(entry) > 2: - conv_fun = entry[2] - - if conv_fun is not None and raw_value is not None: - if len(entry) == 4 and entry[3]: - try: - value = conv_fun(raw_value, data) - except Exception: - self.logger.error('Could not convert shadowserverkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowserverkey, self.feedname, raw_value, conv_fun.__name__) - raise - else: - try: - value = conv_fun(raw_value) - except Exception: - self.logger.error('Could not convert shadowserverkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowserverkey, self.feedname, raw_value, conv_fun.__name__) - raise - return value - - -BOT = ShadowserverJSONParserBot diff --git a/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py b/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py index a625c9d34..2bf6e61e9 100644 --- a/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py +++ b/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py @@ -14,12 +14,13 @@ RANDSTR = secrets.token_urlsafe(50) ASSET_PATH = pathlib.Path(__file__).parent / 'reports-list.json' PARAMETERS = {'reports': 'anarres', 'api_key': RANDSTR, 'secret': RANDSTR, 'logging_level': 'DEBUG', 'types': ['scan_smb', 'cisco_smart_install', 'nonexistent'], 'name': 'shadowservercollector'} -REPORT = {'__type': 'Report', 'extra.file_name': '2020-08-02-scan_smb-anarres-geo.json', 'feed.accuracy': 100.0, 'feed.name': 'shadowservercollector', 'raw': 'e30='} +REPORT = {'__type': 'Report', 'extra.file_name': '2020-08-02-scan_smb-anarres-geo.csv', 'feed.accuracy': 100.0, 'feed.name': 'shadowservercollector', 'raw': 'e30='} def prepare_mocker(mocker): mocker.post('https://transform.shadowserver.org/api2/reports/list', content=ASSET_PATH.read_bytes()) - mocker.post('https://transform.shadowserver.org/api2/reports/download', text='{}') + mocker.get('https://dl.shadowserver.org/xNDSuwXrKnrLrDopU926rR75CAESMWesVCKsuyI8b8ncTv7GCX', text='{}') + mocker.get('https://dl.shadowserver.org/unnzVtn92tS9459rKIEz2J8qb7oJDv0Fa2feGUOiJLCDLqBXnN', text='{}') # Explicit skip_redis is required (although implicitly called by no_cache), otherwise fails in package build environments @@ -80,7 +81,7 @@ def test_report_sent(self, mocker): self.cache.flushdb() prepare_mocker(mocker) self.run_bot(iterations=1, parameters=PARAMETERS) - self.assertAnyLoglineEqual("Sent report: '2020-08-02-cisco_smart_install-anarres-geo.csv' (fixed: '2020-08-02-cisco_smart_install-anarres-geo.json', size: 0.00195 KiB).", 'DEBUG') + self.assertAnyLoglineEqual("Sent report: '2020-08-02-cisco_smart_install-anarres-geo.csv' (fixed: '2020-08-02-cisco_smart_install-anarres-geo.csv', size: 0.00195 KiB).", 'DEBUG') def test_report_content(self, mocker): self.cache.flushdb() From 04549613e9ace976d4f22f1ee3f6d7e20ee0a025 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:59:42 +0000 Subject: [PATCH 03/67] dynamic configuration model --- intelmq/bots/parsers/shadowserver/README.md | 7 + intelmq/bots/parsers/shadowserver/_config.py | 4202 +---------------- intelmq/bots/parsers/shadowserver/parser.py | 46 +- .../parsers/shadowserver/schema.json.test | 180 + .../parsers/shadowserver/update_schema.py | 12 + 5 files changed, 303 insertions(+), 4144 deletions(-) create mode 100644 intelmq/bots/parsers/shadowserver/schema.json.test create mode 100644 intelmq/bots/parsers/shadowserver/update_schema.py diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index eb0ddfb4a..297930861 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,3 +7,10 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/_version_. + +For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. + +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory + +The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index bea3d0c0b..a7b80b7a6 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -77,20 +77,34 @@ feed_idx is not complete. """ +import os import re import base64 import binascii +import json +import urllib.request +import tempfile from typing import Optional, Dict, Tuple, Any import intelmq.lib.harmonization as harmonization +class __Container: + pass + +__config = __Container() +__config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') +__config.schema_mtime = 0.0 +__config.feedname_mapping = {} +__config.filename_mapping = {} def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: - return feedname_mapping.get(given_feedname, None) + reload() + return __config.feedname_mapping.get(given_feedname, None) def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, Any]]]: - return filename_mapping.get(given_filename, None) + reload() + return __config.filename_mapping.get(given_filename, None) def add_UTC_to_timestamp(value: str) -> str: @@ -165,11 +179,6 @@ def invalidate_zero(value: str) -> Optional[int]: return int(value) if value and int(value) != 0 else None -# TODO this function is a wild guess... -def set_tor_node(value: str) -> Optional[bool]: - return True if value else None - - def validate_ip(value: str) -> Optional[str]: """Remove "invalid" IP.""" # FIX: https://github.com/certtools/intelmq/issues/1720 # TODO: Find better fix @@ -240,4126 +249,63 @@ def scan_exchange_identifier(field): return 'exchange-server-webshell' return 'vulnerable-exchange-server' +functions = { + 'add_UTC_to_timestamp': add_UTC_to_timestamp, + 'convert_bool': convert_bool, + 'validate_to_none': validate_to_none, + 'convert_int': convert_int, + 'convert_float': convert_float, + 'convert_http_host_and_url': convert_http_host_and_url, + 'invalidate_zero': invalidate_zero, + 'validate_ip': validate_ip, + 'validate_network': validate_network, + 'validate_fqdn': validate_fqdn, + 'convert_date': convert_date, + 'convert_date_utc': convert_date_utc, + 'force_base64': force_base64, + 'scan_exchange_taxonomy': scan_exchange_taxonomy, + 'scan_exchange_type': scan_exchange_type, + 'scan_exchange_identifier': scan_exchange_identifier, + } + + +def reload (): + """ reload the configuration if it has changed """ + mtime = 0.0 + + if (os.path.isfile(__config.schema_file)): + mtime = os.path.getmtime(__config.schema_file) + if __config.schema_mtime == mtime: + return + schema_file = __config.schema_file + else: + # load a test schema if one has not been downloaded yet + schema_file = __config.schema_file + schema_file += '.test' + + __config.feedname_mapping.clear() + __config.filename_mapping.clear() + with open(schema_file) as fh: + schema = json.load(fh) + for report in schema: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + __config.schema_mtime = mtime + +def update_schema (version): + """ download the latest configuration """ + (th, tmp) = tempfile.mkstemp() + url = 'https://interchange.shadowserver.org/intelmq/'+version + try: + urllib.request.urlretrieve(url, tmp) + except: + raise ValueError("Failed to download %r" % url) -# BEGIN CONFGEN - -# https://www.shadowserver.org/what-we-do/network-reporting/blocklist-report/ -blocklist = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.network', 'ip', validate_network), - ('extra.', 'tag', validate_to_none), - ('source.reverse_dns', 'hostname'), - ('extra.', 'source', validate_to_none), - ('extra.', 'reason', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'blacklisted-ip', - 'classification.taxonomy': 'other', - 'classification.type': 'blacklist', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/compromised-website-report/ -compromised_website = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'application', validate_to_none), - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'http_host', validate_fqdn), - ('source.reverse_dns', 'hostname'), - ('malware.name', 'tag'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('event_description.text', 'category', validate_to_none), - ('extra.', 'system', validate_to_none), - ('extra.', 'detected_since', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'redirect_target', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'cc_url', validate_to_none), - ('extra.', 'family', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'classification.identifier': 'compromised-website', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/device-identification-report/ -device_id = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'undetermined', - 'classification.identifier': 'device-id', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ddos-participant-report/ -event_ddos_participant = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ('extra.', 'http_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'ddos-participant', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-brute-force-events-report/ -event_honeypot_brute_force = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'application'), - ('destination.account', 'username', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'service', validate_to_none), - ('extra.', 'start_time', convert_date_utc), - ('extra.', 'end_time', convert_date_utc), - ('extra.', 'client_version', validate_to_none), - ('extra.', 'password', validate_to_none), - ('extra.', 'payload_url', validate_to_none), - ('extra.', 'payload_md5', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'intrusion-attempts', - 'classification.type': 'brute-force', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-darknet-events-report/ -event_honeypot_darknet = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'tag', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'count', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ddos-events/ -event_honeypot_ddos = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ('extra.', 'http_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'honeypot-ddos', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-amplification-ddos-events-report/ -event_honeypot_ddos_amp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'avg_pps', convert_float), - ('extra.', 'max_pps', convert_float), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'count', convert_int), - ('extra.', 'bytes', convert_int), - ('extra.', 'end_time', convert_date_utc), - ('extra.', 'duration', convert_int), - ], - 'constant_fields': { - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ddos-target-events-report/ -event_honeypot_ddos_target = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'honeypot-ddos-target', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-http-scanner-events/ -event_honeypot_http_scan = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('user_agent', 'http_agent', validate_to_none), - ('extra.method', 'http_request_method', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'pattern', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('extra.', 'url_scheme', validate_to_none), - ('extra.', 'session_tags', validate_to_none), - ('extra.', 'vulnerability_enum', validate_to_none), - ('extra.', 'vulnerability_id', validate_to_none), - ('extra.', 'vulnerability_class', validate_to_none), - ('extra.', 'vulnerability_score', validate_to_none), - ('extra.', 'vulnerability_severity', validate_to_none), - ('extra.', 'vulnerability_version', validate_to_none), - ('extra.', 'threat_framework', validate_to_none), - ('extra.', 'threat_tactic_id', validate_to_none), - ('extra.', 'threat_technique_id', validate_to_none), - ('extra.', 'target_vendor', validate_to_none), - ('extra.', 'target_product', validate_to_none), - ('extra.', 'target_class', validate_to_none), - ('extra.', 'file_md5', validate_to_none), - ('extra.', 'file_sha256', validate_to_none), - ('extra.', 'request_raw', force_base64), - ('extra.', 'body_raw', force_base64), - ], - 'constant_fields': { - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'protocol.application': 'http', - 'classification.identifier': 'honeypot-http-scan', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ics-scanner-events-report/ -event_honeypot_ics_scan = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'state', validate_to_none), - ('extra.', 'sensor_id', validate_to_none), - ('extra.', 'slave_id', validate_to_none), - ('extra.', 'function_code', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'response', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'ics', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ip-spoofer-events-report/ -event_ip_spoofer = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'infection', validate_to_none), - ('source.network', 'network', validate_network), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'routedspoof', validate_to_none), - ('extra.', 'session', validate_to_none), - ('extra.', 'nat', convert_bool), - ], - 'constant_fields': { - 'classification.taxonomy': 'fraud', - 'classification.type': 'masquerade', - 'classification.identifier': 'ip-spoofer', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-events-report/ -event_sinkhole = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'infection', validate_to_none), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-dns-events-report/ -event_sinkhole_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.naics', 'src_naics', invalidate_zero), - ('extra.sector', 'src_sector', validate_to_none), - ('extra.dns_query_type', 'query_type'), - ('extra.dns_query', 'query'), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'count', convert_int), - ], - 'constant_fields': { - 'classification.identifier': 'sinkholedns', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-events-report/ -event_sinkhole_http = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'tag'), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_agent', validate_to_none), - ('extra.', 'forwarded_by', validate_to_none), - ('extra.', 'ssl_cipher', validate_to_none), - ('extra.', 'http_referer', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-referer-events-report/ -event_sinkhole_http_referer = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ], - 'optional_fields': [ - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('extra.', 'http_referer_ip', validate_ip), - ('extra.', 'http_referer_port', convert_int), - ('extra.', 'http_referer_asn', invalidate_zero), - ('extra.', 'http_referer_geo', validate_to_none), - ('extra.', 'http_referer_region', validate_to_none), - ('extra.', 'http_referer_city', validate_to_none), - ('extra.', 'http_referer_hostname', validate_to_none), - ('extra.', 'http_referer_naics', invalidate_zero), - ('extra.', 'http_referer_sector', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_referer', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'sinkhole-http-referer', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/malware-url-report/ -malware_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'source', validate_to_none), - ('malware.hash.sha256', 'sha256', validate_to_none), - ('extra.', 'application', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'malware-url', - }, -} - -phish_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'source', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'fraud', - 'classification.type': 'phishing', - 'classification.identifier': 'phish-url', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-http-proxy-report/ -population_http_proxy = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('malware.name', 'tag'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'proxy_authenticate', validate_to_none), - ('extra.', 'via', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'http', - }, -} - -# http://www.shadowserver.org/wiki/pmwiki.php/Services/Sandbox-Connection -sandbox_conn = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('destination.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('malware.hash.md5', 'md5', validate_to_none), - ('protocol.transport', 'protocol'), - ('extra.', 'bytes_in', validate_to_none), - ('extra.', 'bytes_out', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'sandbox-conn', - }, -} - -sandbox_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ], - 'optional_fields': [ - ('extra.dns_query_type', 'type', validate_to_none), - ('malware.hash.md5', 'md5hash', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'response', validate_to_none), - ('extra.', 'family', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'source', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'dns', - 'classification.identifier': 'sandbox-dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sandbox-url-report/ -sandbox_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('destination.fqdn', 'host', validate_fqdn), - ('extra.http_request_method', 'method', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('malware.hash.md5', 'md5', validate_to_none), - ('destination.url', 'url', convert_http_host_and_url, True), - ('user_agent', 'user_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'sandbox-url', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-adb-report/ -scan_adb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'name', validate_to_none), - ('extra.', 'model', validate_to_none), - ('extra.', 'device', validate_to_none), - ('extra.', 'features', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-adb', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'adb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-afp-report/ -scan_afp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'machine_type', validate_to_none), - ('extra.', 'afp_versions', validate_to_none), - ('extra.', 'uams', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'signature', validate_to_none), - ('extra.', 'directory_service', validate_to_none), - ('extra.', 'utf8_servername', validate_to_none), - ('extra.', 'network_address', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-afp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'afp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-amqp-report/ -scan_amqp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'channel', validate_to_none), - ('extra.', 'message_length', convert_int), - ('extra.', 'class', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'version_major', validate_to_none), - ('extra.', 'version_minor', validate_to_none), - ('extra.', 'capabilities', validate_to_none), - ('extra.', 'cluster_name', validate_to_none), - ('extra.', 'platform', validate_to_none), - ('extra.', 'product', validate_to_none), - ('extra.', 'product_version', validate_to_none), - ('extra.', 'mechanisms', validate_to_none), - ('extra.', 'locales', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-amqp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'amqp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-apple-remote-desktop-ard-report/ -scan_ard = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-chargen-report/ -scan_chargen = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.response_size', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'chargen', - 'classification.identifier': 'open-chargen', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-cisco-smart-install-report/ -scan_cisco_smart_install = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'cisco-smart-install', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-coap-report/ -scan_coap = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'response', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'coap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-couchdb-report/ -scan_couchdb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'server_version', validate_to_none), - ('extra.', 'couchdb_message', validate_to_none), - ('extra.', 'couchdb_version', validate_to_none), - ('extra.', 'git_sha', validate_to_none), - ('extra.', 'features', validate_to_none), - ('extra.', 'vendor', validate_to_none), - ('extra.', 'visible_databases', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'error_reason', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'CouchDB', - 'classification.identifier': 'open-couchdb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-cwmp-report/ -scan_cwmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'cwmp', - 'classification.identifier': 'open-cwmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-db2-discovery-service-report/ -scan_db2 = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'db2_hostname', validate_to_none), - ('extra.', 'servername', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'db2', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-ddos-middlebox-report/ -scan_ddos_middlebox = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'source_port', validate_to_none), - ('extra.', 'bytes', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'method', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ddos-middlebox', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/dns-open-resolvers-report/ -scan_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'min_amplification', convert_float), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'dns_version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'dns-open-resolver', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-docker-service-report/ -scan_docker = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'experimental', validate_to_none), - ('extra.', 'api_version', validate_to_none), - ('extra.', 'arch', validate_to_none), - ('extra.', 'go_version', validate_to_none), - ('extra.os.name', 'os', validate_to_none), - ('extra.', 'kernel_version', validate_to_none), - ('extra.', 'git_commit', validate_to_none), - ('extra.', 'min_api_version', validate_to_none), - ('extra.', 'build_time', validate_to_none), - ('extra.', 'pkg_version', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'docker', - 'classification.identifier': 'open-docker', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-dvr-dhcpdiscover-report/ -scan_dvr_dhcpdiscover = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('extra.', 'video_input_channels', convert_int), - ('extra.', 'alarm_input_channels', convert_int), - ('extra.', 'video_output_channels', convert_int), - ('extra.', 'alarm_output_channels', convert_int), - ('extra.', 'remote_video_input_channels', convert_int), - ('extra.', 'ipv4_dhcp_enable', convert_bool), - ('extra.', 'ipv6_dhcp_enable', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_id', validate_to_none), - ('extra.', 'device_serial', validate_to_none), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'manufacturer', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'http_port', convert_int), - ('extra.', 'internal_port', convert_int), - ('extra.', 'mac_address', validate_to_none), - ('extra.', 'ipv4_address', validate_to_none), - ('extra.', 'ipv4_gateway', validate_to_none), - ('extra.', 'ipv4_subnet_mask', validate_to_none), - ('extra.', 'ipv6_address', validate_to_none), - ('extra.', 'ipv6_link_local', validate_to_none), - ('extra.', 'ipv6_gateway', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-dvr-dhcpdiscover', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-elasticsearch-report/ -scan_elasticsearch = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'build_snapshot', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'ok', convert_bool), - ('extra.', 'name', validate_to_none), - ('extra.', 'cluster_name', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'build_hash', validate_to_none), - ('extra.', 'build_timestamp', validate_to_none), - ('extra.', 'lucene_version', validate_to_none), - ('extra.', 'tagline', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'elasticsearch', - 'classification.identifier': 'open-elasticsearch', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-erlang-port-mapper-report-daemon/ -scan_epmd = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'nodes', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'Erlang Port Mapper Daemon', - 'classification.identifier': 'open-epmd', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-exchange-server-report/ -scan_exchange = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('classification.taxonomy', 'tag', scan_exchange_taxonomy), - ('classification.type', 'tag', scan_exchange_type), - ('classification.identifier', 'tag', scan_exchange_identifier), - ('extra.', 'tag', validate_to_none), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'servername', validate_to_none), - ('destination.url', 'url', convert_http_host_and_url, True), - ], - 'constant_fields': { - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ftp-report/ -scan_ftp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'auth_tls_response', validate_to_none), - ('extra.', 'auth_ssl_response', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ftp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-hadoop-report/ -scan_hadoop = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'total_disk', convert_int), - ('extra.', 'used_disk', convert_int), - ('extra.', 'free_disk', convert_int), - ('source.reverse_dns', 'hostname'), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'clusterid', validate_to_none), - ('extra.', 'livenodes', validate_to_none), - ('extra.', 'namenodeaddress', validate_to_none), - ('extra.', 'volumeinfo', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-http-report/ -scan_http = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-http-proxy-report/ -scan_http_proxy = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'proxy_authenticate', validate_to_none), - ('extra.', 'via', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-http-report/ -scan_http_vulnerable = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'version', validate_to_none), - ('extra.', 'build_date', validate_to_none), - ('extra.', 'detail', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ics-report/ -scan_ics = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_id', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ics', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ipmi-report/ -scan_ipmi = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'none_auth', convert_bool), - ('extra.', 'md2_auth', convert_bool), - ('extra.', 'md5_auth', convert_bool), - ('extra.', 'passkey_auth', convert_bool), - ('extra.', 'oem_auth', convert_bool), - ('extra.', 'permessage_auth', convert_bool), - ('extra.', 'userlevel_auth', convert_bool), - ('extra.', 'usernames', convert_bool), - ('extra.', 'nulluser', convert_bool), - ('extra.', 'anon_login', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'ipmi_version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'defaultkg', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'deviceid', validate_to_none), - ('extra.', 'devicerev', validate_to_none), - ('extra.', 'firmwarerev', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'manufacturerid', validate_to_none), - ('extra.', 'manufacturername', validate_to_none), - ('extra.', 'productid', validate_to_none), - ('extra.', 'productname', validate_to_none), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipmi', - 'protocol.transport': 'udp', - 'classification.identifier': 'open-ipmi', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ipp-report/ -scan_ipp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'ipp_version', validate_to_none), - ('extra.', 'cups_version', validate_to_none), - ('extra.', 'printer_uris', validate_to_none), - ('extra.', 'printer_name', validate_to_none), - ('extra.', 'printer_info', validate_to_none), - ('extra.', 'printer_more_info', validate_to_none), - ('extra.', 'printer_make_and_model', validate_to_none), - ('extra.', 'printer_firmware_name', validate_to_none), - ('extra.', 'printer_firmware_string_version', validate_to_none), - ('extra.', 'printer_firmware_version', validate_to_none), - ('extra.', 'printer_organization', validate_to_none), - ('extra.', 'printer_organization_unit', validate_to_none), - ('extra.', 'printer_uuid', validate_to_none), - ('extra.', 'printer_wifi_ssid', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipp', - 'classification.identifier': 'open-ipp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-isakmp-report/ -scan_isakmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'spi_size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'initiator_spi', validate_to_none), - ('extra.', 'responder_spi', validate_to_none), - ('extra.', 'next_payload', validate_to_none), - ('extra.', 'exchange_type', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'message_id', validate_to_none), - ('extra.', 'next_payload2', validate_to_none), - ('extra.', 'domain_of_interpretation', validate_to_none), - ('extra.', 'protocol_id', validate_to_none), - ('extra.', 'notify_message_type', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'open-ike', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipsec', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-kubernetes-api-server-report/ -scan_kubernetes = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'major', validate_to_none), - ('extra.', 'minor', validate_to_none), - ('extra.', 'git_version', validate_to_none), - ('extra.', 'git_commit', validate_to_none), - ('extra.', 'git_tree_state', validate_to_none), - ('extra.', 'build_date', validate_to_none), - ('extra.', 'go_version', validate_to_none), - ('extra.', 'compiler', validate_to_none), - ('extra.', 'platform', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'kubernetes', - 'classification.identifier': 'open-kubernetes', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ldap-tcp-report/ -scan_ldap_tcp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'dns_host_name', validate_to_none), - ('extra.', 'domain_controller_functionality', convert_int), - ('extra.', 'domain_functionality', convert_int), - ('extra.', 'forest_functionality', convert_int), - ('extra.', 'highest_committed_usn', convert_int), - ('extra.', 'is_global_catalog_ready', convert_bool), - ('extra.', 'is_synchronized', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'size', convert_int), - ('extra.', 'configuration_naming_context', validate_to_none), - ('extra.', 'current_time', validate_to_none), - ('extra.', 'default_naming_context', validate_to_none), - ('extra.', 'ds_service_name', validate_to_none), - ('extra.', 'ldap_service_name', validate_to_none), - ('extra.', 'naming_contexts', validate_to_none), - ('extra.', 'root_domain_naming_context', validate_to_none), - ('extra.', 'schema_naming_context', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'subschema_subentry', validate_to_none), - ('extra.', 'supported_capabilities', validate_to_none), - ('extra.', 'supported_control', validate_to_none), - ('extra.', 'supported_ldap_policies', validate_to_none), - ('extra.', 'supported_ldap_version', validate_to_none), - ('extra.', 'supported_sasl_mechanisms', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ldap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ldap-report/ -scan_ldap_udp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'dns_host_name', validate_to_none), - ('extra.', 'domain_controller_functionality', convert_int), - ('extra.', 'domain_functionality', convert_int), - ('extra.', 'forest_functionality', convert_int), - ('extra.', 'highest_committed_usn', convert_int), - ('extra.', 'is_global_catalog_ready', convert_bool), - ('extra.', 'is_synchronized', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'size', convert_int), - ('extra.', 'configuration_naming_context', validate_to_none), - ('extra.', 'current_time', validate_to_none), - ('extra.', 'default_naming_context', validate_to_none), - ('extra.', 'ds_service_name', validate_to_none), - ('extra.', 'ldap_service_name', validate_to_none), - ('extra.', 'naming_contexts', validate_to_none), - ('extra.', 'root_domain_naming_context', validate_to_none), - ('extra.', 'schema_naming_context', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'subschema_subentry', validate_to_none), - ('extra.', 'supported_capabilities', validate_to_none), - ('extra.', 'supported_control', validate_to_none), - ('extra.', 'supported_ldap_policies', validate_to_none), - ('extra.', 'supported_ldap_version', validate_to_none), - ('extra.', 'supported_sasl_mechanisms', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ldap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mdns-report/ -scan_mdns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'mdns_name', validate_to_none), - ('extra.', 'mdns_ipv4', validate_to_none), - ('extra.', 'mdns_ipv6', validate_to_none), - ('extra.', 'services', validate_to_none), - ('extra.', 'workstation_name', validate_to_none), - ('extra.', 'workstation_ipv4', validate_to_none), - ('extra.', 'workstation_ipv6', validate_to_none), - ('extra.', 'workstation_info', validate_to_none), - ('extra.', 'http_name', validate_to_none), - ('extra.', 'http_ipv4', validate_to_none), - ('extra.', 'http_ipv6', validate_to_none), - ('extra.', 'http_ptr', validate_to_none), - ('extra.', 'http_info', validate_to_none), - ('extra.', 'http_target', validate_to_none), - ('extra.', 'http_port', convert_int), - ('extra.', 'spotify_name', validate_to_none), - ('extra.', 'spotify_ipv4', validate_to_none), - ('extra.', 'spotify_ipv6', validate_to_none), - ('extra.', 'opc_ua_discovery', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mdns', - 'classification.identifier': 'open-mdns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-memcached-report/ -scan_memcached = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'pid', convert_int), - ('extra.', 'pointer_size', convert_int), - ('extra.', 'uptime', convert_int), - ('extra.', 'curr_connections', convert_int), - ('extra.', 'total_connections', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'time', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'memcached', - 'classification.identifier': 'open-memcached', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mongodb-report/ -scan_mongodb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'gitversion', validate_to_none), - ('extra.', 'sysinfo', validate_to_none), - ('extra.', 'opensslversion', validate_to_none), - ('extra.', 'allocator', validate_to_none), - ('extra.', 'javascriptengine', validate_to_none), - ('extra.', 'bits', validate_to_none), - ('extra.', 'maxbsonobjectsize', validate_to_none), - ('extra.', 'ok', convert_bool), - ('extra.', 'visible_databases', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mongodb', - 'classification.identifier': 'open-mongodb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mqtt-report/ -scan_mqtt = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'anonymous_access', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'raw_response', validate_to_none), - ('extra.', 'hex_code', validate_to_none), - ('extra.', 'code', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serialNumber', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mqtt', - 'classification.identifier': 'open-mqtt', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mqtt-report/ -scan_mqtt_anon = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'raw_response', validate_to_none), - ('extra.', 'hex_code', validate_to_none), - ('extra.', 'code', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serialNumber', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mqtt', - 'classification.identifier': 'open-mqtt-anon', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-MSSQL -scan_mssql = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'server_name', validate_to_none), - ('extra.', 'tcp_port', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'instance_name', validate_to_none), - ('extra.', 'named_pipe', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mssql', - 'classification.identifier': 'open-mssql', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-mysql-server-report/ -scan_mysql = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'client_can_handle_expired_passwords', convert_bool), - ('extra.', 'client_compress', convert_bool), - ('extra.', 'client_connect_attrs', convert_bool), - ('extra.', 'client_connect_with_db', convert_bool), - ('extra.', 'client_deprecated_eof', convert_bool), - ('extra.', 'client_found_rows', convert_bool), - ('extra.', 'client_ignore_sigpipe', convert_bool), - ('extra.', 'client_ignore_space', convert_bool), - ('extra.', 'client_interactive', convert_bool), - ('extra.', 'client_local_files', convert_bool), - ('extra.', 'client_long_flag', convert_bool), - ('extra.', 'client_long_password', convert_bool), - ('extra.', 'client_multi_results', convert_bool), - ('extra.', 'client_multi_statements', convert_bool), - ('extra.', 'client_no_schema', convert_bool), - ('extra.', 'client_odbc', convert_bool), - ('extra.', 'client_plugin_auth', convert_bool), - ('extra.', 'client_plugin_auth_len_enc_client_data', convert_bool), - ('extra.', 'client_protocol_41', convert_bool), - ('extra.', 'client_ps_multi_results', convert_bool), - ('extra.', 'client_reserved', convert_bool), - ('extra.', 'client_secure_connection', convert_bool), - ('extra.', 'client_session_track', convert_bool), - ('extra.', 'client_ssl', convert_bool), - ('extra.', 'client_transactions', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'mysql_protocol_version', validate_to_none), - ('extra.', 'server_version', validate_to_none), - ('extra.', 'error_code', validate_to_none), - ('extra.', 'error_id', validate_to_none), - ('extra.', 'error_message', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'mysql', - 'classification.identifier': 'open-mysql', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-NATPMP -scan_nat_pmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'uptime', convert_int), - ('extra.', 'external_ip', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'natpmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-netbios-report/ -scan_netbios = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.account', 'username'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'mac_address', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'workgroup', validate_to_none), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'netbios-nameservice', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/netcore-netis-router-vulnerability-scan-report/ -scan_netis_router = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'response', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.transport': 'udp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ntp-version-report/ -scan_ntp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'clk_wander', convert_float), - ('extra.', 'frequency', convert_float), - ('extra.', 'jitter', convert_float), - ('extra.', 'leap', convert_float), - ('extra.', 'offset', convert_float), - ('extra.', 'peer', convert_int), - ('extra.', 'poll', convert_int), - ('extra.', 'precision', convert_int), - ('extra.', 'rootdelay', convert_float), - ('extra.', 'rootdispersion', convert_float), - ('extra.', 'stratum', convert_int), - ('extra.', 'tc', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'version', validate_to_none), - ('extra.', 'clock', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'mintc', validate_to_none), - ('extra.', 'noise', validate_to_none), - ('extra.', 'phase', validate_to_none), - ('extra.', 'processor', validate_to_none), - ('extra.', 'refid', validate_to_none), - ('extra.', 'reftime', validate_to_none), - ('extra.', 'stability', validate_to_none), - ('extra.', 'state', validate_to_none), - ('extra.', 'system', validate_to_none), - ('extra.', 'tai', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ntp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ntp-monitor-report/ -scan_ntpmonitor = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'packets', convert_int), - ('extra.', 'size', convert_int), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ntp', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Portmapper -scan_portmapper = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'programs', validate_to_none), - ('extra.', 'mountd_port', validate_to_none), - ('extra.', 'exports', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'portmapper', - 'classification.identifier': 'open-portmapper', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-postgresql-server-report/ -scan_postgres = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'startup_error_line', convert_int), - ('extra.', 'client_ssl', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'supported_protocols', validate_to_none), - ('extra.', 'protocol_error_code', validate_to_none), - ('extra.', 'protocol_error_file', validate_to_none), - ('extra.', 'protocol_error_line', validate_to_none), - ('extra.', 'protocol_error_message', validate_to_none), - ('extra.', 'protocol_error_routine', validate_to_none), - ('extra.', 'protocol_error_severity', validate_to_none), - ('extra.', 'protocol_error_severity_v', validate_to_none), - ('extra.', 'startup_error_code', validate_to_none), - ('extra.', 'startup_error_file', validate_to_none), - ('extra.', 'startup_error_message', validate_to_none), - ('extra.', 'startup_error_routine', validate_to_none), - ('extra.', 'startup_error_severity', validate_to_none), - ('extra.', 'startup_error_severity_v', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'postgres', - 'classification.identifier': 'open-postgres', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-QOTD -scan_qotd = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'quote', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'qotd', - 'classification.identifier': 'open-qotd', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-quic-report/ -scan_quic = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'version_field_1', validate_to_none), - ('extra.', 'version_field_2', validate_to_none), - ('extra.', 'version_field_3', validate_to_none), - ('extra.', 'version_field_4', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'open-quic', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-radmin-report/ -scan_radmin = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-radmin', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-rdp-report/ -scan_rdp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'cve20190708_vulnerable', convert_bool), - ('extra.', 'bluekeep_vulnerable', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'rdp_protocol', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'rdp', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-rdp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ms-rdpeudp/ -scan_rdpeudp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sessionid', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Redis -scan_redis = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'git_sha1', validate_to_none), - ('extra.', 'git_dirty_flag', validate_to_none), - ('extra.', 'build_id', validate_to_none), - ('extra.', 'mode', validate_to_none), - ('extra.os.name', 'os', validate_to_none), - ('extra.', 'architecture', validate_to_none), - ('extra.', 'multiplexing_api', validate_to_none), - ('extra.', 'gcc_version', validate_to_none), - ('extra.', 'process_id', validate_to_none), - ('extra.', 'run_id', validate_to_none), - ('extra.', 'uptime', convert_int), - ('extra.', 'connected_clients', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'redis', - 'classification.identifier': 'open-redis', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-rsync-report/ -scan_rsync = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'module', validate_to_none), - ('extra.', 'motd', validate_to_none), - ('extra.', 'has_password', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'rsync', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-sip-report/ -scan_sip = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'sip', validate_to_none), - ('extra.', 'sip_code', validate_to_none), - ('extra.', 'sip_reason', validate_to_none), - ('user_agent', 'user_agent', validate_to_none), - ('extra.', 'sip_via', validate_to_none), - ('extra.', 'sip_to', validate_to_none), - ('extra.', 'sip_from', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'content_type', validate_to_none), - ('extra.sip_server', 'server', validate_to_none), - ('extra.sip_contact', 'contact', validate_to_none), - ('extra.sip_cseq', 'cseq', validate_to_none), - ('extra.sip_call_id', 'call_id', validate_to_none), - ('extra.sip_allow', 'allow', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'sip', - 'classification.identifier': 'open-sip', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-slp-service-report/ -scan_slp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'function', validate_to_none), - ('extra.', 'function_text', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'next_extension_offset', validate_to_none), - ('extra.', 'xid', validate_to_none), - ('extra.', 'language_tag_length', validate_to_none), - ('extra.', 'language_tag', validate_to_none), - ('extra.', 'error_code', validate_to_none), - ('extra.', 'error_code_text', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'slp', - 'classification.identifier': 'open-slp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-smb-report/ -scan_smb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'smb_implant', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'arch', validate_to_none), - ('extra.', 'key', validate_to_none), - ('extra.', 'smbv1_support', validate_to_none), - ('extra.', 'smb_major_number', validate_to_none), - ('extra.', 'smb_minor_number', validate_to_none), - ('extra.', 'smb_revision', validate_to_none), - ('extra.', 'smb_version_string', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'smb', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-smb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-smtp-report/ -scan_smtp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'smtp', - 'classification.identifier': 'open-smtp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-smtp-report/ -scan_smtp_vulnerable = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'smtp', - 'classification.identifier': 'vulnerable-smtp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-snmp-report/ -scan_snmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'sysdesc', validate_to_none), - ('extra.', 'sysname', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'version', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'community', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'snmp', - 'classification.identifier': 'open-snmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-socks4-5-proxy-report/ -scan_socks = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-socks', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-SSDP -scan_ssdp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'header', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'systime', validate_to_none), - ('extra.', 'cache_control', validate_to_none), - ('extra.', 'location', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'search_target', validate_to_none), - ('extra.', 'unique_service_name', validate_to_none), - ('extra.', 'host', validate_to_none), - ('extra.', 'nts', validate_to_none), - ('extra.', 'nt', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'server_port', validate_to_none), - ('extra.', 'instance', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'updated_at', validate_to_none), - ('extra.', 'resource_identifier', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ssdp', - 'classification.identifier': 'open-ssdp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ssh-report/ -scan_ssh = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'serverid_raw', validate_to_none), - ('extra.', 'serverid_version', validate_to_none), - ('extra.', 'serverid_software', validate_to_none), - ('extra.', 'serverid_comment', validate_to_none), - ('extra.', 'server_cookie', validate_to_none), - ('extra.', 'available_kex', validate_to_none), - ('extra.', 'available_ciphers', validate_to_none), - ('extra.', 'available_mac', validate_to_none), - ('extra.', 'available_compression', validate_to_none), - ('extra.', 'selected_kex', validate_to_none), - ('extra.', 'algorithm', validate_to_none), - ('extra.', 'selected_cipher', validate_to_none), - ('extra.', 'selected_mac', validate_to_none), - ('extra.', 'selected_compression', validate_to_none), - ('extra.', 'server_signature_value', validate_to_none), - ('extra.', 'server_signature_raw', validate_to_none), - ('extra.', 'server_host_key', validate_to_none), - ('extra.', 'server_host_key_sha256', validate_to_none), - ('extra.', 'rsa_prime', validate_to_none), - ('extra.', 'rsa_prime_length', validate_to_none), - ('extra.', 'rsa_generator', validate_to_none), - ('extra.', 'rsa_generator_length', validate_to_none), - ('extra.', 'rsa_public_key', validate_to_none), - ('extra.', 'rsa_public_key_length', validate_to_none), - ('extra.', 'rsa_exponent', validate_to_none), - ('extra.', 'rsa_modulus', validate_to_none), - ('extra.', 'rsa_length', validate_to_none), - ('extra.', 'dss_prime', validate_to_none), - ('extra.', 'dss_prime_length', validate_to_none), - ('extra.', 'dss_generator', validate_to_none), - ('extra.', 'dss_generator_length', validate_to_none), - ('extra.', 'dss_public_key', validate_to_none), - ('extra.', 'dss_public_key_length', validate_to_none), - ('extra.', 'dss_dsa_public_g', validate_to_none), - ('extra.', 'dss_dsa_public_p', validate_to_none), - ('extra.', 'dss_dsa_public_q', validate_to_none), - ('extra.', 'dss_dsa_public_y', validate_to_none), - ('extra.', 'ecdsa_curve25519', validate_to_none), - ('extra.', 'ecdsa_curve', validate_to_none), - ('extra.', 'ecdsa_public_key_length', validate_to_none), - ('extra.', 'ecdsa_public_key_b', validate_to_none), - ('extra.', 'ecdsa_public_key_gx', validate_to_none), - ('extra.', 'ecdsa_public_key_gy', validate_to_none), - ('extra.', 'ecdsa_public_key_n', validate_to_none), - ('extra.', 'ecdsa_public_key_p', validate_to_none), - ('extra.', 'ecdsa_public_key_x', validate_to_none), - ('extra.', 'ecdsa_public_key_y', validate_to_none), - ('extra.', 'ed25519_curve25519', validate_to_none), - ('extra.', 'ed25519_cert_public_key_nonce', validate_to_none), - ('extra.', 'ed25519_cert_public_key_bytes', validate_to_none), - ('extra.', 'ed25519_cert_public_key_raw', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sha256', validate_to_none), - ('extra.', 'ed25519_cert_public_key_serial', validate_to_none), - ('extra.', 'ed25519_cert_public_key_type_id', validate_to_none), - ('extra.', 'ed25519_cert_public_key_type_name', validate_to_none), - ('extra.', 'ed25519_cert_public_key_keyid', validate_to_none), - ('extra.', 'ed25519_cert_public_key_principles', validate_to_none), - ('extra.', 'ed25519_cert_public_key_valid_after', validate_to_none), - ('extra.', 'ed25519_cert_public_key_valid_before', validate_to_none), - ('extra.', 'ed25519_cert_public_key_duration', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_bytes', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_raw', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_sha256', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_value', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sig_raw', validate_to_none), - ('extra.', 'banner', validate_to_none), - ('extra.', 'userauth_methods', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'open-ssh', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ssl-report/ -scan_ssl = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'ssl_poodle', convert_bool), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'freak_vulnerable', convert_bool), - ('extra.', 'freak_cipher_suite', validate_to_none), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'https', - 'classification.identifier': 'open-ssl', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Freak-Scan -scan_ssl_freak = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'freak_vulnerable', convert_bool), - ('extra.', 'freak_cipher_suite', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'page_sha256fp', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'https', - 'classification.identifier': 'ssl-freak', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Scan -scan_ssl_poodle = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'ssl_poodle', convert_bool), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'page_sha256fp', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'https', - 'classification.identifier': 'ssl-poodle', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-stun-service-report/ -scan_stun = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'mapped_port', convert_int), - ('extra.', 'xor_mapped_port', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'transaction_id', validate_to_none), - ('extra.', 'magic_cookie', validate_to_none), - ('extra.', 'message_length', convert_int), - ('extra.', 'message_type', validate_to_none), - ('extra.', 'mapped_family', validate_to_none), - ('extra.', 'mapped_address', validate_to_none), - ('extra.', 'xor_mapped_family', validate_to_none), - ('extra.', 'xor_mapped_address', validate_to_none), - ('extra.', 'software', validate_to_none), - ('extra.', 'fingerprint', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'Session Traversal Utilities for NAT', - 'classification.identifier': 'open-stun', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/synful-scan-report/ -scan_synfulknock = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'ack_number', convert_int), - ('extra.', 'window_size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'sequence_number', validate_to_none), - ('extra.', 'urgent_pointer', validate_to_none), - ('extra.', 'tcp_flags', validate_to_none), - ('extra.', 'raw_packet', validate_to_none), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-synfulknock', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-telnet-report/ -scan_telnet = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'telnet', - 'classification.identifier': 'open-telnet', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-TFTP -scan_tftp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'errorcode', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'errormessage', validate_to_none), - ('extra.', 'size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'tftp', - 'classification.identifier': 'open-tftp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ubiquiti-report/ -scan_ubiquiti = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.mac_address', 'mac', validate_to_none), - ('extra.radio_name', 'radioname', validate_to_none), - ('extra.model', 'modelshort', validate_to_none), - ('extra.model_full', 'modelfull', validate_to_none), - ('extra.firmwarerev', 'firmware', validate_to_none), - ('extra.response_size', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'essid', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-vnc-report/ -scan_vnc = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'product', validate_to_none), - ('extra.', 'banner', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'vnc', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-vnc', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ws-discovery-service-report/ -scan_ws_discovery = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'error', validate_to_none), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ws-discovery', - 'classification.identifier': 'open-ws-discovery', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-xdmcp-service-report/ -scan_xdmcp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'reported_hostname', validate_to_none), - ('extra.', 'status', validate_to_none), - ('extra.', 'size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'xdmcp', - 'classification.identifier': 'open-xdmcp', - }, -} - -# http://www.shadowserver.org/wiki/pmwiki.php/Services/Spam-URL -spam_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'source', validate_to_none), - ('extra.', 'sender', validate_to_none), - ('extra.', 'subject', validate_to_none), - ('malware.hash.md5', 'md5', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'abusive-content', - 'classification.type': 'spam', - 'classification.identifier': 'spam-url', - }, -} - -special = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('source.reverse_dns', 'hostname'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'status', validate_to_none), - ('extra.', 'detail', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'special', - }, -} - -mapping = ( - # feed name, file name, function - ('Blocklist', 'blocklist', blocklist), - ('Compromised-Website', 'compromised_website', compromised_website), - ('Device-Identification IPv4', 'device_id', device_id), - ('Device-Identification IPv6', 'device_id6', device_id), - ('DDoS-Participant', 'event4_ddos_participant', event_ddos_participant), - ('Honeypot-Brute-Force-Events', 'event4_honeypot_brute_force', event_honeypot_brute_force), - ('Honeypot-Darknet', 'event4_honeypot_darknet', event_honeypot_darknet), - ('Honeypot-DDoS', 'event4_honeypot_ddos', event_honeypot_ddos), - ('Honeypot-Amplification-DDoS-Events', 'event4_honeypot_ddos_amp', event_honeypot_ddos_amp), - ('Honeypot-DDoS-Target', 'event4_honeypot_ddos_target', event_honeypot_ddos_target), - ('Honeypot-HTTP-Scan', 'event4_honeypot_http_scan', event_honeypot_http_scan), - ('Honeypot-ICS-Scanner', 'event4_honeypot_ics_scan', event_honeypot_ics_scan), - ('IP-Spoofer-Events', 'event4_ip_spoofer', event_ip_spoofer), - ('Microsoft-Sinkhole-Events IPv4', 'event4_microsoft_sinkhole', event_sinkhole), - ('Microsoft-Sinkhole-Events-HTTP IPv4', 'event4_microsoft_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events IPv4', 'event4_sinkhole', event_sinkhole), - ('Sinkhole-DNS', 'event4_sinkhole_dns', event_sinkhole_dns), - ('Sinkhole-Events-HTTP IPv4', 'event4_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer IPv4', 'event4_sinkhole_http_referer', event_sinkhole_http_referer), - ('Sinkhole-Events IPv6', 'event6_sinkhole', event_sinkhole), - ('Sinkhole-Events-HTTP IPv6', 'event6_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer IPv6', 'event6_sinkhole_http_referer', event_sinkhole_http_referer), - ('Malware-URL', 'malware_url', malware_url), - ('Phish-URL', 'phish_url', phish_url), - ('IPv6-Accessible-HTTP-Proxy', 'population6_http_proxy', population_http_proxy), - ('Accessible-HTTP-Proxy', 'population_http_proxy', population_http_proxy), - ('Sandbox-Connections', 'sandbox_conn', sandbox_conn), - ('Sandbox-DNS', 'sandbox_dns', sandbox_dns), - ('Sandbox-URL', 'sandbox_url', sandbox_url), - ('IPv6-Accessible-CWMP', 'scan6_cwmp', scan_cwmp), - ('IPv6-DNS-Open-Resolvers', 'scan6_dns', scan_dns), - ('IPv6-Vulnerable-Exchange', 'scan6_exchange', scan_exchange), - ('IPv6-Accessible-FTP', 'scan6_ftp', scan_ftp), - ('IPv6-Accessible-HTTP', 'scan6_http', scan_http), - ('IPv6-Open-HTTP-Proxy', 'scan6_http_proxy', scan_http_proxy), - ('IPv6-Vulnerable-HTTP', 'scan6_http_vulnerable', scan_http_vulnerable), - ('IPv6-Open-IPP', 'scan6_ipp', scan_ipp), - ('IPv6-Open-LDAP-TCP', 'scan6_ldap_tcp', scan_ldap_tcp), - ('IPv6-Open-MQTT', 'scan6_mqtt', scan_mqtt), - ('IPv6-Open-Anonymous-MQTT', 'scan6_mqtt_anon', scan_mqtt_anon), - ('IPv6-Accessible-MySQL', 'scan6_mysql', scan_mysql), - ('IPv6-NTP-Version', 'scan6_ntp', scan_ntp), - ('IPv6-NTP-Monitor', 'scan6_ntpmonitor', scan_ntpmonitor), - ('IPv6-Accessible-PostgreSQL', 'scan6_postgres', scan_postgres), - ('IPv6-Accessible-RDP', 'scan6_rdp', scan_rdp), - ('IPv6-Accessible-SLP', 'scan6_slp', scan_slp), - ('IPv6-Accessible-SMB', 'scan6_smb', scan_smb), - ('IPv6-Accessible-SMTP', 'scan6_smtp', scan_smtp), - ('IPv6-Vulnerable-SMTP', 'scan6_smtp_vulnerable', scan_smtp_vulnerable), - ('IPv6-Open-SNMP', 'scan6_snmp', scan_snmp), - ('IPv6-Accessible-SSH', 'scan6_ssh', scan_ssh), - ('IPv6-Accessible-SSL', 'scan6_ssl', scan_ssl), - ('SSL-FREAK-Vulnerable-Servers IPv6', 'scan6_ssl_freak', scan_ssl_freak), - ('SSL-POODLE-Vulnerable-Servers IPv6', 'scan6_ssl_poodle', scan_ssl_poodle), - ('IPv6-Accessible-Session-Traversal-Utilities-for-NAT', 'scan6_stun', scan_stun), - ('IPv6-Accessible-Telnet', 'scan6_telnet', scan_telnet), - ('IPv6-Accessible-VNC', 'scan6_vnc', scan_vnc), - ('Accessible-ADB', 'scan_adb', scan_adb), - ('Accessible-AFP', 'scan_afp', scan_afp), - ('Accessible-AMQP', 'scan_amqp', scan_amqp), - ('Accessible-ARD', 'scan_ard', scan_ard), - ('Open-Chargen', 'scan_chargen', scan_chargen), - ('Accessible-Cisco-Smart-Install', 'scan_cisco_smart_install', scan_cisco_smart_install), - ('Accessible-CoAP', 'scan_coap', scan_coap), - ('Accessible-CouchDB', 'scan_couchdb', scan_couchdb), - ('Accessible-CWMP', 'scan_cwmp', scan_cwmp), - ('Open-DB2-Discovery-Service', 'scan_db2', scan_db2), - ('Vulnerable-DDoS-Middlebox', 'scan_ddos_middlebox', scan_ddos_middlebox), - ('DNS-Open-Resolvers', 'scan_dns', scan_dns), - ('Accessible-Docker', 'scan_docker', scan_docker), - ('Accessible-DVR-DHCPDiscover', 'scan_dvr_dhcpdiscover', scan_dvr_dhcpdiscover), - ('Open-Elasticsearch', 'scan_elasticsearch', scan_elasticsearch), - ('Accessible-Erlang-Port-Mapper-Daemon', 'scan_epmd', scan_epmd), - ('Vulnerable-Exchange-Server', 'scan_exchange', scan_exchange), - ('Accessible-FTP', 'scan_ftp', scan_ftp), - ('Accessible-Hadoop', 'scan_hadoop', scan_hadoop), - ('Accessible-HTTP', 'scan_http', scan_http), - ('Open-HTTP-Proxy', 'scan_http_proxy', scan_http_proxy), - ('Vulnerable-HTTP', 'scan_http_vulnerable', scan_http_vulnerable), - ('Accessible-ICS', 'scan_ics', scan_ics), - ('Open-IPMI', 'scan_ipmi', scan_ipmi), - ('Open-IPP', 'scan_ipp', scan_ipp), - ('Vulnerable-ISAKMP', 'scan_isakmp', scan_isakmp), - ('Accessible-Kubernetes-API', 'scan_kubernetes', scan_kubernetes), - ('Open-LDAP-TCP', 'scan_ldap_tcp', scan_ldap_tcp), - ('Open-LDAP', 'scan_ldap_udp', scan_ldap_udp), - ('Open-mDNS', 'scan_mdns', scan_mdns), - ('Open-Memcached', 'scan_memcached', scan_memcached), - ('Open-MongoDB', 'scan_mongodb', scan_mongodb), - ('Open-MQTT', 'scan_mqtt', scan_mqtt), - ('Open-Anonymous-MQTT', 'scan_mqtt_anon', scan_mqtt_anon), - ('Open-MSSQL', 'scan_mssql', scan_mssql), - ('Accessible-MySQL', 'scan_mysql', scan_mysql), - ('Open-NATPMP', 'scan_nat_pmp', scan_nat_pmp), - ('Open-NetBIOS-Nameservice', 'scan_netbios', scan_netbios), - ('Open-Netis', 'scan_netis_router', scan_netis_router), - ('NTP-Version', 'scan_ntp', scan_ntp), - ('NTP-Monitor', 'scan_ntpmonitor', scan_ntpmonitor), - ('Open-Portmapper', 'scan_portmapper', scan_portmapper), - ('Accessible-PostgreSQL', 'scan_postgres', scan_postgres), - ('Open-QOTD', 'scan_qotd', scan_qotd), - ('Accessible-QUIC', 'scan_quic', scan_quic), - ('Accessible-Radmin', 'scan_radmin', scan_radmin), - ('Accessible-RDP', 'scan_rdp', scan_rdp), - ('Accessible-MS-RDPEUDP', 'scan_rdpeudp', scan_rdpeudp), - ('Open-Redis', 'scan_redis', scan_redis), - ('Accessible-Rsync', 'scan_rsync', scan_rsync), - ('Accessible-SIP', 'scan_sip', scan_sip), - ('Accessible-SLP', 'scan_slp', scan_slp), - ('Accessible-SMB', 'scan_smb', scan_smb), - ('Accessible-SMTP', 'scan_smtp', scan_smtp), - ('Vulnerable-SMTP', 'scan_smtp_vulnerable', scan_smtp_vulnerable), - ('Open-SNMP', 'scan_snmp', scan_snmp), - ('Accessible-SOCKS4/5-Proxy', 'scan_socks', scan_socks), - ('Open-SSDP', 'scan_ssdp', scan_ssdp), - ('Accessible-SSH', 'scan_ssh', scan_ssh), - ('Accessible-SSL', 'scan_ssl', scan_ssl), - ('SSL-FREAK-Vulnerable-Servers', 'scan_ssl_freak', scan_ssl_freak), - ('SSL-POODLE-Vulnerable-Servers IPv4', 'scan_ssl_poodle', scan_ssl_poodle), - ('Accessible-Session-Traversal-Utilities-for-NAT', 'scan_stun', scan_stun), - ('SYNful-Knock', 'scan_synfulknock', scan_synfulknock), - ('Accessible-Telnet', 'scan_telnet', scan_telnet), - ('Open-TFTP', 'scan_tftp', scan_tftp), - ('Accessible-Ubiquiti-Discovery-Service', 'scan_ubiquiti', scan_ubiquiti), - ('Accessible-VNC', 'scan_vnc', scan_vnc), - ('Accessible-WS-Discovery-Service', 'scan_ws_discovery', scan_ws_discovery), - ('Open-XDMCP', 'scan_xdmcp', scan_xdmcp), - ('Spam-URL', 'spam_url', spam_url), - ('Special', 'special', special), - ('Accessible-RDPEUDP', 'scan_rdpeudp', scan_rdpeudp), - ('Sinkhole-Events', 'event4_sinkhole', event_sinkhole), - ('Sinkhole-Events-HTTP', 'event4_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer', 'event4_sinkhole_http_referer', event_sinkhole_http_referer), -) -# END CONFGEN + try: + with open(tmp) as fh: + schema = json.load(fh) + except: + # leave tempfile behind for diagnosis + raise ValueError("Failed to validate %r" % tmp) -feedname_mapping = {feedname: function for feedname, filename, function in mapping} -filename_mapping = {filename: (feedname, function) for feedname, filename, function in mapping} + os.replace(tmp, __config.schema_file) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 70ba3b4bb..f14549141 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -22,6 +22,7 @@ """ import copy import re +import os from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue @@ -29,7 +30,13 @@ class ShadowserverParserBot(ParserBot): - """Parse all ShadowServer feeds""" + """ + Parse all ShadowServer feeds + + Parameters: + schema_file (str): Path to the report schema file + + """ recover_line = ParserBot.recover_line_csv_dict _csv_params = {'dialect': 'unix'} @@ -124,10 +131,17 @@ def parse_line(self, row, report): value = raw_value if conv_func is not None and raw_value is not None: - if len(item) == 4 and item[3]: - value = conv_func(raw_value, row) - else: - value = conv_func(raw_value) + try: + if len(item) == 4 and item[3]: + value = config.functions[conv_func](raw_value, row) + else: + value = config.functions[conv_func](raw_value) + except Exception: + """ fail early and often in this case. We want to be able to convert everything """ + self.logger.error('Could not convert shadowkey: %r in feed %r, ' + 'value: %r via conversion function %r.', + shadowkey, self.feedname, raw_value, conv_func) + raise if value is not None: event.add(intelmqkey, value) @@ -153,17 +167,17 @@ def parse_line(self, row, report): value = raw_value if conv_func is not None and raw_value is not None: - if len(item) == 4 and item[3]: - value = conv_func(raw_value, row) - else: - try: - value = conv_func(raw_value) - except Exception: - """ fail early and often in this case. We want to be able to convert everything """ - self.logger.error('Could not convert shadowkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowkey, self.feedname, raw_value, conv_func.__name__) - raise + try: + if len(item) == 4 and item[3]: + value = config.functions[conv_func](raw_value, row) + else: + value = config.functions[conv_func](raw_value) + except Exception: + """ fail early and often in this case. We want to be able to convert everything """ + self.logger.error('Could not convert shadowkey: %r in feed %r, ' + 'value: %r via conversion function %r.', + shadowkey, self.feedname, raw_value, conv_func) + raise if value is not None: if intelmqkey == 'extra.': diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test b/intelmq/bots/parsers/shadowserver/schema.json.test new file mode 100644 index 000000000..2cfb8bb1d --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/schema.json.test @@ -0,0 +1,180 @@ +{ + "test_smb" : { + "constant_fields" : { + "classification.identifier" : "test-smb", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "smb", + "protocol.transport" : "tcp" + }, + "feed_name" : "Test-Accessible-SMB", + "file_name" : "test_smb", + "optional_fields" : [ + [ + "extra.", + "smb_implant", + "convert_bool" + ], + [ + "source.reverse_dns", + "hostname" + ], + [ + "extra.", + "tag" + ], + [ + "source.asn", + "asn", + "invalidate_zero" + ], + [ + "source.geolocation.cc", + "geo" + ], + [ + "source.geolocation.region", + "region" + ], + [ + "source.geolocation.city", + "city" + ], + [ + "extra.source.naics", + "naics", + "invalidate_zero" + ], + [ + "extra.source.sic", + "sic", + "invalidate_zero" + ], + [ + "extra.", + "arch", + "validate_to_none" + ], + [ + "extra.", + "key", + "validate_to_none" + ], + [ + "extra.", + "smbv1_support", + "validate_to_none" + ], + [ + "extra.", + "smb_major_number", + "validate_to_none" + ], + [ + "extra.", + "smb_minor_number", + "validate_to_none" + ], + [ + "extra.", + "smb_revision", + "validate_to_none" + ], + [ + "extra.", + "smb_version_string", + "validate_to_none" + ] + ], + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ] + ] + }, + "test_telnet" : { + "constant_fields" : { + "classification.identifier" : "test-telnet", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "telnet" + }, + "feed_name" : "Test-Accessible-Telnet", + "file_name" : "test_telnet", + "optional_fields" : [ + [ + "protocol.transport", + "protocol" + ], + [ + "source.reverse_dns", + "hostname" + ], + [ + "extra.", + "tag", + "validate_to_none" + ], + [ + "source.asn", + "asn", + "invalidate_zero" + ], + [ + "source.geolocation.cc", + "geo" + ], + [ + "source.geolocation.region", + "region" + ], + [ + "source.geolocation.city", + "city" + ], + [ + "extra.", + "naics", + "invalidate_zero" + ], + [ "extra.", + "sic", + "invalidate_zero" + ], + [ + "extra.", + "banner", + "validate_to_none" + ] + ], + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ] + ] + } +} diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py new file mode 100644 index 000000000..040f67259 --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/update_schema.py @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import intelmq.bots.parsers.shadowserver._config as config + +if __name__ == '__main__': # pragma: no cover + exec(open(os.path.join(os.path.dirname(__file__), '../../../version.py')).read()) # defines __version__ + config.update_schema(__version__) From 0a39e0de01db827faa9a603c97b0846effdb0cbb Mon Sep 17 00:00:00 2001 From: elsif2 Date: Wed, 12 Apr 2023 00:01:32 +0000 Subject: [PATCH 04/67] revised tests --- .../bots/parsers/shadowserver/test_broken.py | 12 +- .../bots/parsers/shadowserver/test_mapping.py | 8 +- .../parsers/shadowserver/test_parameters.py | 37 +++--- .../parsers/shadowserver/test_report_smb.py | 124 ++++++++++++++++++ .../shadowserver/test_report_switch.py | 16 +-- .../shadowserver/test_report_telnet.py | 87 ++++++++++++ .../shadowserver/testdata/test_smb.csv | 4 + .../testdata/test_smb.csv.license | 2 + .../shadowserver/testdata/test_telnet.csv | 3 + .../testdata/test_telnet.csv.license | 2 + 10 files changed, 260 insertions(+), 35 deletions(-) create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_report_smb.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 472dd0b90..2b803142e 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -13,12 +13,12 @@ REPORT1 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_http-test-test.csv", + "extra.file_name": "2019-01-01-test_smb-test-test.csv", } REPORT2 = {"raw": utils.base64_encode('timestamp,ip,port\n2018-08-01T00:00:00+00,127.0.0.1,80'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ftp-test-test.csv", + "extra.file_name": "2019-01-01-test_telnet-test-test.csv", } REPORT3 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", @@ -48,10 +48,10 @@ def test_broken(self): """ self.input_message = REPORT1 self.run_bot(allowed_error_count=1) - self.assertLogMatches(pattern="Detected report's file name: 'scan_http'.", + self.assertLogMatches(pattern="Detected report's file name: 'test_smb'.", levelname="DEBUG") self.assertLogMatches(pattern="Failed to parse line.") - self.assertLogMatches(pattern="ValueError: Required column 'timestamp' not found in feed 'Accessible-HTTP'. Possible change in data format or misconfiguration.") + self.assertLogMatches(pattern="ValueError: Required column 'timestamp' not found in feed 'Test-Accessible-SMB'. Possible change in data format or misconfiguration.") self.assertLogMatches(pattern=r"Sent 0 events and found 1 problem\(s\)\.", levelname="INFO") @@ -61,9 +61,9 @@ def test_half_broken(self): """ self.input_message = REPORT2 self.run_bot(allowed_warning_count=63) - self.assertLogMatches(pattern="Detected report's file name: 'scan_ftp'.", + self.assertLogMatches(pattern="Detected report's file name: 'test_telnet'.", levelname="DEBUG") - self.assertLogMatches(pattern="Optional key 'jarm' not found in feed 'Accessible-FTP'.", + self.assertLogMatches(pattern="Optional key 'banner' not found in feed 'Test-Accessible-Telnet'.", levelname="WARNING") self.assertLogMatches(pattern=r"Sent 1 events and found 0 problem\(s\)\.", levelname="INFO") diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index f58aed66e..6a2af9447 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -11,22 +11,22 @@ with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_telnet.csv')) as handle: + 'testdata/test_telnet.csv')) as handle: TELNET_FILE = handle.read() EXAMPLE_TELNET = { "raw": utils.base64_encode(TELNET_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_telnet.csv", + "extra.file_name": "2019-01-01-test_telnet.csv", } with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_vnc.csv')) as handle: + 'testdata/test_smb.csv')) as handle: TELNET_FILE = handle.read() EXAMPLE_VNC = { "raw": utils.base64_encode(TELNET_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_vnc.csv", + "extra.file_name": "2019-01-01-test_smb.csv", } diff --git a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py index a5ea81f19..677cd0319 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py @@ -12,38 +12,41 @@ from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dns.csv')) as handle: + 'testdata/test_smb.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() EXAMPLE_REPORT = {"raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_dns-test-test.csv", + "extra.file_name": "2019-01-01-test_smb-test-test.csv", 'feed.name': 'report feedname', } EVENTS = [{ '__type': 'Event', 'feed.name': 'report feedname', - "classification.identifier": "dns-open-resolver", + "classification.identifier": 'test-smb', "classification.taxonomy": "vulnerable", "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.66", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", + "extra.smb_implant": False, + "extra.smb_major_number": '2', + "extra.smb_minor_number": '1', + "extra.smb_version_string": 'SMB 2.1', + "extra.smbv1_support": 'N', + "extra.tag": "smb", + "protocol.application": "smb", + "protocol.transport": "tcp", 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.179", - "source.port": 53, - "source.reverse_dns": "198-51-100-189.example.net", + "source.asn": 64512, + "source.geolocation.cc": "ZZ", + "source.geolocation.city": "City", + "source.geolocation.region": "Region", + "source.ip": "192.168.0.1", + "source.port": 445, + "source.reverse_dns": "node01.example.com", "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:34+00:00" + "time.source": "2010-02-10T00:00:00+00:00" }, ] @@ -70,7 +73,7 @@ def test_overwrite_feed_name(self): self.run_bot(prepare=False) for i, EVENT in enumerate(EVENTS): event = EVENT.copy() - event['feed.name'] = 'DNS-Open-Resolvers' + event['feed.name'] = 'Test-Accessible-SMB' self.assertMessageEqual(i, event) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py new file mode 100644 index 000000000..c7eefdf0a --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import unittest + +import intelmq.lib.test as test +import intelmq.lib.utils as utils +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot + +with open(os.path.join(os.path.dirname(__file__), + 'testdata/test_smb.csv')) as handle: + EXAMPLE_FILE = handle.read() +EXAMPLE_LINES = EXAMPLE_FILE.splitlines() + +EXAMPLE_REPORT = {'feed.name': 'Test-Accessible-SMB', + "raw": utils.base64_encode(EXAMPLE_FILE), + "__type": "Report", + "time.observation": "2015-01-01T00:00:00+00:00", + "extra.file_name": "2019-01-01-test_smb-test-geo.csv", + } +EVENTS = [ +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.1', + 'source.port' : 445, + 'source.reverse_dns' : 'node01.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:00+00:00' +}, + +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.2', + 'source.port' : 445, + 'source.reverse_dns' : 'node02.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:01+00:00' +}, + +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.3', + 'source.port' : 445, + 'source.reverse_dns' : 'node03.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:02+00:00' +} + ] + + +class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for a ShadowserverParserBot. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.default_input_message = EXAMPLE_REPORT + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() + for i, EVENT in enumerate(EVENTS): + self.assertMessageEqual(i, EVENT) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index 0a34a69f0..570d612fb 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -12,24 +12,24 @@ from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ftp.csv')) as handle: + 'testdata/test_smb.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] -FIRST_REPORT = {'feed.name': 'Accessible FTP', +FIRST_REPORT = {'feed.name': 'Test-Accessible-SMB', "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), "__type": "Report", "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ftp-test-test.csv", + "extra.file_name": "2019-03-25-test_smb-test-test.csv", } -with open(os.path.join(os.path.dirname(__file__), 'testdata/blocklist.csv')) as handle: +with open(os.path.join(os.path.dirname(__file__), 'testdata/test_telnet.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] SECOND_REPORT = { - 'feed.name': 'Blocklist', + 'feed.name': 'Test-Accessible-Telnet', "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-blocklist-test-geo.csv", + "extra.file_name": "2019-01-01-test_telnet-test-geo.csv", } @@ -48,9 +48,9 @@ def test_event(self): """ Test if the parser correctly detects and handles different report types. """ self.input_message = [FIRST_REPORT, SECOND_REPORT] self.run_bot(iterations=2) - self.assertLogMatches("Detected report's file name: 'scan_ftp'", + self.assertLogMatches("Detected report's file name: 'test_smb'", levelname='DEBUG') - self.assertLogMatches("Detected report's file name: 'blocklist'", + self.assertLogMatches("Detected report's file name: 'test_telnet'", levelname='DEBUG') diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py new file mode 100644 index 000000000..6d539ac4a --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import unittest + +import intelmq.lib.test as test +import intelmq.lib.utils as utils +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot + +with open(os.path.join(os.path.dirname(__file__), + 'testdata/test_telnet.csv')) as handle: + EXAMPLE_FILE = handle.read() +EXAMPLE_LINES = EXAMPLE_FILE.splitlines() + +EXAMPLE_REPORT = {'feed.name': 'Test-Accessible-Telnet', + "raw": utils.base64_encode(EXAMPLE_FILE), + "__type": "Report", + "time.observation": "2015-01-01T00:00:00+00:00", + "extra.file_name": "2019-01-01-test_telnet-test-geo.csv", + } +EVENTS = [{'__type': 'Event', + 'feed.name': 'Test-Accessible-Telnet', + "classification.identifier": "test-telnet", + "classification.taxonomy": "vulnerable", + "classification.type": "vulnerable-system", + "extra.banner": "|MikroTik v6.5|Login:", + "extra.tag": "telnet-alt", + "protocol.application": "telnet", + "protocol.transport": "tcp", + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], + EXAMPLE_LINES[1]])), + "source.asn": 20255, + "source.geolocation.cc": "AA", + "source.geolocation.city": "LOCATION", + "source.geolocation.region": "LOCATION", + "source.ip": "198.123.245.145", + "source.port": 5678, + "source.reverse_dns": "example.local", + "time.observation": "2015-01-01T00:00:00+00:00", + "time.source": "2019-09-04T12:27:34+00:00" + }, + {'__type': 'Event', + 'feed.name': 'Test-Accessible-Telnet', + "classification.identifier": "test-telnet", + "classification.taxonomy": "vulnerable", + "classification.type": "vulnerable-system", + "extra.banner": "|MikroTik v6.45.3 (stable)|Login:", + "extra.tag": "telnet-alt", + "protocol.application": "telnet", + "protocol.transport": "tcp", + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], + EXAMPLE_LINES[2]])), + "source.asn": 20255, + "source.geolocation.cc": "AA", + "source.geolocation.city": "LOCATION", + "source.geolocation.region": "LOCATION", + "source.ip": "198.123.245.145", + "source.port": 5678, + "source.reverse_dns": "example.local", + "time.observation": "2015-01-01T00:00:00+00:00", + "time.source": "2019-09-04T12:27:40+00:00" + }] + + +class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for a ShadowserverParserBot. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.default_input_message = EXAMPLE_REPORT + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() + for i, EVENT in enumerate(EVENTS): + self.assertMessageEqual(i, EVENT) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv new file mode 100644 index 000000000..fc7fe2fff --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv @@ -0,0 +1,4 @@ +"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","smb_implant","arch","key","smbv1_support","smb_major_number","smb_minor_number","smb_revision","smb_version_string" +"2010-02-10 00:00:00",192.168.0.1,445,node01.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" +"2010-02-10 00:00:01",192.168.0.2,445,node02.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" +"2010-02-10 00:00:02",192.168.0.3,445,node03.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license new file mode 100644 index 000000000..f512a890e --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2022 The Shadowserver Foundation +SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv new file mode 100644 index 000000000..3309e9a3d --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv @@ -0,0 +1,3 @@ +"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" +"2019-09-04 12:27:34","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.5|Login:" +"2019-09-04 12:27:40","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.45.3 (stable)|Login:" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license new file mode 100644 index 000000000..942a94035 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +SPDX-License-Identifier: AGPL-3.0-or-later From 94b22fb677173f968bd57f02ed6a056f6c2cb5c7 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 8 May 2023 15:05:12 +0000 Subject: [PATCH 05/67] Updated to reset report type on reload #2361 --- intelmq/bots/parsers/shadowserver/README.md | 2 +- intelmq/bots/parsers/shadowserver/_config.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index 297930861..bb6216b9a 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -11,6 +11,6 @@ The report configuration is now stored in a _schema.json_ file downloaded from h For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index a7b80b7a6..29382d278 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -272,15 +272,14 @@ def scan_exchange_identifier(field): def reload (): """ reload the configuration if it has changed """ mtime = 0.0 + schema_file = __config.schema_file - if (os.path.isfile(__config.schema_file)): + if os.path.isfile(__config.schema_file): mtime = os.path.getmtime(__config.schema_file) if __config.schema_mtime == mtime: return - schema_file = __config.schema_file else: # load a test schema if one has not been downloaded yet - schema_file = __config.schema_file schema_file += '.test' __config.feedname_mapping.clear() From b2f9bc371ed35a88c3179e0d6ed002c43362368d Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 23 May 2023 01:12:47 +0000 Subject: [PATCH 06/67] Added schema download on startup and additional logging --- intelmq/bots/parsers/shadowserver/_config.py | 33 +++++++++++++------ intelmq/bots/parsers/shadowserver/parser.py | 1 + .../parsers/shadowserver/update_schema.py | 3 +- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 29382d278..f766be322 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -106,6 +106,8 @@ def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, A reload() return __config.filename_mapping.get(given_filename, None) +def set_logger(logger): + __config.logger = logger def add_UTC_to_timestamp(value: str) -> str: return value + ' UTC' @@ -272,29 +274,38 @@ def scan_exchange_identifier(field): def reload (): """ reload the configuration if it has changed """ mtime = 0.0 - schema_file = __config.schema_file if os.path.isfile(__config.schema_file): mtime = os.path.getmtime(__config.schema_file) if __config.schema_mtime == mtime: return else: - # load a test schema if one has not been downloaded yet - schema_file += '.test' + __config.logger.info("The schema file does not exist.") + + if __config.schema_mtime == 0.0 and mtime == 0.0 and not os.environ.get('INTELMQ_SKIP_INTERNET'): + __config.logger.info("Attempting to download schema.") + update_schema() __config.feedname_mapping.clear() __config.filename_mapping.clear() - with open(schema_file) as fh: - schema = json.load(fh) - for report in schema: - __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) - __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + for schema_file in [ __config.schema_file, ".".join([__config.schema_file, 'test']) ]: + if os.path.isfile(schema_file): + with open(schema_file) as fh: + schema = json.load(fh) + for report in schema: + if report == "_meta": + __config.logger.info("Loading schema %s." % schema[report]['date_created']) + for msg in schema[report]['change_log']: + __config.logger.info(msg) + else: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime -def update_schema (version): +def update_schema (): """ download the latest configuration """ (th, tmp) = tempfile.mkstemp() - url = 'https://interchange.shadowserver.org/intelmq/'+version + url = 'https://interchange.shadowserver.org/intelmq/v1' try: urllib.request.urlretrieve(url, tmp) except: @@ -307,4 +318,6 @@ def update_schema (version): # leave tempfile behind for diagnosis raise ValueError("Failed to validate %r" % tmp) + if os.path.exists(__config.schema_file): + os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) os.replace(tmp, __config.schema_file) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index f14549141..2f20262bf 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -47,6 +47,7 @@ class ShadowserverParserBot(ParserBot): overwrite = False def init(self): + config.set_logger(self.logger) if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py index 040f67259..a7975147e 100644 --- a/intelmq/bots/parsers/shadowserver/update_schema.py +++ b/intelmq/bots/parsers/shadowserver/update_schema.py @@ -8,5 +8,4 @@ import intelmq.bots.parsers.shadowserver._config as config if __name__ == '__main__': # pragma: no cover - exec(open(os.path.join(os.path.dirname(__file__), '../../../version.py')).read()) # defines __version__ - config.update_schema(__version__) + config.update_schema() From d5cf063756d0a087d17531cde546d920d7768703 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 23 May 2023 23:32:53 +0000 Subject: [PATCH 07/67] Added version support to the schema update function. --- intelmq/bots/parsers/shadowserver/README.md | 6 ++-- intelmq/bots/parsers/shadowserver/_config.py | 32 +++++++++++++++++--- intelmq/bots/parsers/shadowserver/parser.py | 4 +++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index bb6216b9a..c757020e9 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,10 +7,10 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/_version_. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1. -For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. +For environments that have internet connectivity the `update_schema.py` script can be called from a cron job to obtain the latest revision. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index f766be322..bb67db525 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -97,6 +97,11 @@ class __Container: __config.feedname_mapping = {} __config.filename_mapping = {} +def set_logger(logger): + """ Sets the logger instance. """ + __config.logger = logger + + def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: reload() return __config.feedname_mapping.get(given_feedname, None) @@ -106,8 +111,6 @@ def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, A reload() return __config.filename_mapping.get(given_filename, None) -def set_logger(logger): - __config.logger = logger def add_UTC_to_timestamp(value: str) -> str: return value + ' UTC' @@ -304,20 +307,39 @@ def reload (): def update_schema (): """ download the latest configuration """ - (th, tmp) = tempfile.mkstemp() + if os.environ.get('INTELMQ_SKIP_INTERNET'): + return None + + (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) url = 'https://interchange.shadowserver.org/intelmq/v1' try: urllib.request.urlretrieve(url, tmp) except: raise ValueError("Failed to download %r" % url) + new_version = '' + old_version = '' + try: with open(tmp) as fh: schema = json.load(fh) + new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis raise ValueError("Failed to validate %r" % tmp) if os.path.exists(__config.schema_file): - os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) - os.replace(tmp, __config.schema_file) + old_version = '' + try: + with open(__config.schema_file) as fh: + schema = json.load(fh) + old_version = schema['_meta']['date_created'] + if new_version != old_version: + os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) + except: + pass + + if new_version != old_version: + os.replace(tmp, __config.schema_file) + else: + os.unlink(tmp) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 2f20262bf..71489e2ec 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -48,6 +48,10 @@ class ShadowserverParserBot(ParserBot): def init(self): config.set_logger(self.logger) + try: + config.update_schema() + except Exception as e: + logger.warning(f"Schema update failed: {e}.") if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: From 1e6ea8982b085b8e0e4d2224c4f574803097be63 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Sun, 28 May 2023 23:13:54 +0000 Subject: [PATCH 08/67] Documentation and style updates. --- CHANGELOG.md | 6 + .../shadowserver/collector_reports_api.py | 2 +- intelmq/bots/parsers/shadowserver/README.md | 39 ++++- intelmq/bots/parsers/shadowserver/_config.py | 52 +++--- intelmq/bots/parsers/shadowserver/parser.py | 2 +- .../bots/parsers/shadowserver/test_broken.py | 4 +- .../bots/parsers/shadowserver/test_mapping.py | 1 - .../parsers/shadowserver/test_report_smb.py | 151 +++++++++--------- .../shadowserver/test_report_switch.py | 10 +- .../shadowserver/test_report_telnet.py | 4 +- 10 files changed, 154 insertions(+), 117 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea160a7c5..6b54eb84d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,15 +62,21 @@ CHANGELOG ### Bots #### Collectors +<<<<<<< HEAD - `intelmq.bots.collector.rt`: - restrict `python-rt` to be below version 3.0 due to introduced breaking changes, - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). +======= +- `intelmq.bots.collectors.shadowserver.collector_reports_api`: + - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) +>>>>>>> Documentation and style updates. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: - Reset detected `feedname` at shutdown to re-detect the feedname on reloads (PR#2361 by @elsif2, fixes #2360). + - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. Please see intelmq/bots/parsers/shadowserver/README.md for details. (PR#2372) - `intelmq.bots.parsers.shadowserver._config`: - Added 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Service' report. (PR#2338) - Removed unused `p0f_genre` and `p0f_detail` from the 'DNS-Open-Resolvers' report. (PR#2338) diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index dc8bd6b42..5e7117bd2 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv'). The 'json' option is not longer supported. + file_format (str): File format to download ('csv'). The 'json' option is no longer supported. """ country = None diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index c757020e9..ae38dcb8c 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,10 +7,45 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. -For environments that have internet connectivity the `update_schema.py` script can be called from a cron job to obtain the latest revision. +For environments that have internet connectivity the `update_schema.py` script should be called from a cron job to obtain the latest revision. +The parser will attempt to download a schema update on startup unless INTELMQ_SKIP_INTERNET is set. For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. + + +## Sample configuration: + +``` +shadowserver-collector: + description: Our bot responsible for getting reports from Shadowserver + enabled: true + group: Collector + module: intelmq.bots.collectors.shadowserver.collector_reports_api + name: Shadowserver_Collector + parameters: + destination_queues: + _default: [shadowserver-parser-queue] + file_format: csv + api_key: "$API_KEY_received_from_the_shadowserver_foundation" + secret: "$SECRET_received_from_the_shadowserver_foundation" + run_mode: continuous +``` + +``` +shadowserver-parser: + bot_id: shadowserver-parser + name: Shadowserver Parser + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.shadowserver.parser + parameters: + destination_queues: + _default: [file-output-queue] + run_mode: continuous +``` + diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index bb67db525..5219fdb34 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -88,15 +88,18 @@ import intelmq.lib.harmonization as harmonization + class __Container: pass + __config = __Container() __config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') __config.schema_mtime = 0.0 __config.feedname_mapping = {} __config.filename_mapping = {} + def set_logger(logger): """ Sets the logger instance. """ __config.logger = logger @@ -254,27 +257,28 @@ def scan_exchange_identifier(field): return 'exchange-server-webshell' return 'vulnerable-exchange-server' + functions = { - 'add_UTC_to_timestamp': add_UTC_to_timestamp, - 'convert_bool': convert_bool, - 'validate_to_none': validate_to_none, - 'convert_int': convert_int, - 'convert_float': convert_float, - 'convert_http_host_and_url': convert_http_host_and_url, - 'invalidate_zero': invalidate_zero, - 'validate_ip': validate_ip, - 'validate_network': validate_network, - 'validate_fqdn': validate_fqdn, - 'convert_date': convert_date, - 'convert_date_utc': convert_date_utc, - 'force_base64': force_base64, - 'scan_exchange_taxonomy': scan_exchange_taxonomy, - 'scan_exchange_type': scan_exchange_type, - 'scan_exchange_identifier': scan_exchange_identifier, - } - - -def reload (): + 'add_UTC_to_timestamp': add_UTC_to_timestamp, + 'convert_bool': convert_bool, + 'validate_to_none': validate_to_none, + 'convert_int': convert_int, + 'convert_float': convert_float, + 'convert_http_host_and_url': convert_http_host_and_url, + 'invalidate_zero': invalidate_zero, + 'validate_ip': validate_ip, + 'validate_network': validate_network, + 'validate_fqdn': validate_fqdn, + 'convert_date': convert_date, + 'convert_date_utc': convert_date_utc, + 'force_base64': force_base64, + 'scan_exchange_taxonomy': scan_exchange_taxonomy, + 'scan_exchange_type': scan_exchange_type, + 'scan_exchange_identifier': scan_exchange_identifier, +} + + +def reload(): """ reload the configuration if it has changed """ mtime = 0.0 @@ -291,7 +295,7 @@ def reload (): __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [ __config.schema_file, ".".join([__config.schema_file, 'test']) ]: + for schema_file in [__config.schema_file, ".".join([__config.schema_file, 'test'])]: if os.path.isfile(schema_file): with open(schema_file) as fh: schema = json.load(fh) @@ -305,13 +309,14 @@ def reload (): __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime -def update_schema (): + +def update_schema(): """ download the latest configuration """ if os.environ.get('INTELMQ_SKIP_INTERNET'): return None (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) - url = 'https://interchange.shadowserver.org/intelmq/v1' + url = 'https://interchange.shadowserver.org/intelmq/v1/schema' try: urllib.request.urlretrieve(url, tmp) except: @@ -329,7 +334,6 @@ def update_schema (): raise ValueError("Failed to validate %r" % tmp) if os.path.exists(__config.schema_file): - old_version = '' try: with open(__config.schema_file) as fh: schema = json.load(fh) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 71489e2ec..668a81534 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -51,7 +51,7 @@ def init(self): try: config.update_schema() except Exception as e: - logger.warning(f"Schema update failed: {e}.") + self.logger.warning("Schema update failed: %s." % e) if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 2b803142e..3797f03cd 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -24,12 +24,12 @@ "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2019-01-01-some_string-test-test.csv", -} + } REPORT4 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2020.wrong-filename.csv", -} + } class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index 6a2af9447..d296dfdc2 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -52,6 +52,5 @@ def test_changed_feed(self): self.run_bot(iterations=2) - if __name__ == '__main__': # pragma: no cover unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py index c7eefdf0a..93d592d15 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -22,85 +22,78 @@ "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2019-01-01-test_smb-test-geo.csv", } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] +EVENTS = [{'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.1', + 'source.port': 445, + 'source.reverse_dns': 'node01.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:00+00:00' + }, + {'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.2', + 'source.port': 445, + 'source.reverse_dns': 'node02.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:01+00:00' + }, + {'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.3', + 'source.port': 445, + 'source.reverse_dns': 'node03.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:02+00:00' + }] class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index 570d612fb..a9be8a0a1 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -16,11 +16,11 @@ EXAMPLE_LINES = handle.read().splitlines()[:2] FIRST_REPORT = {'feed.name': 'Test-Accessible-SMB', - "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-test_smb-test-test.csv", - } + "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), + "__type": "Report", + "time.observation": "2019-03-25T00:00:00+00:00", + "extra.file_name": "2019-03-25-test_smb-test-test.csv", + } with open(os.path.join(os.path.dirname(__file__), 'testdata/test_telnet.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py index 6d539ac4a..df9cf25dc 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -42,7 +42,7 @@ "source.reverse_dns": "example.local", "time.observation": "2015-01-01T00:00:00+00:00", "time.source": "2019-09-04T12:27:34+00:00" - }, + }, {'__type': 'Event', 'feed.name': 'Test-Accessible-Telnet', "classification.identifier": "test-telnet", @@ -63,7 +63,7 @@ "source.reverse_dns": "example.local", "time.observation": "2015-01-01T00:00:00+00:00", "time.source": "2019-09-04T12:27:40+00:00" - }] + }] class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): From fc3f5b0aa1685109f62a7addc69699a92676e935 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 30 May 2023 16:05:26 +0000 Subject: [PATCH 09/67] Added schema.json.test.license. --- intelmq/bots/parsers/shadowserver/schema.json.test.license | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 intelmq/bots/parsers/shadowserver/schema.json.test.license diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test.license b/intelmq/bots/parsers/shadowserver/schema.json.test.license new file mode 100644 index 000000000..9f58c89ef --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/schema.json.test.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +SPDX-License-Identifier: AGPL-3.0-or-later From b996e0e5ef93f9acd283609d0b5fd9f196d44438 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 27 Jul 2023 20:19:25 +0000 Subject: [PATCH 10/67] Updates in response to feedback. --- .../shadowserver/collector_reports_api.py | 9 +++- intelmq/bots/parsers/shadowserver/README.md | 21 ++++++-- intelmq/bots/parsers/shadowserver/_config.py | 53 +++++++++++++------ intelmq/bots/parsers/shadowserver/parser.py | 45 +++++++++++++--- .../parsers/shadowserver/update_schema.py | 11 ---- .../shadowserver/test_download_schema.py | 28 ++++++++++ 6 files changed, 130 insertions(+), 37 deletions(-) delete mode 100644 intelmq/bots/parsers/shadowserver/update_schema.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_download_schema.py diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 5e7117bd2..05bffa898 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -68,12 +68,19 @@ def init(self): if self.file_format is not None: if not (self.file_format == 'csv'): - raise ValueError('Invalid file_format') + raise ValueError("Invalid file_format '%s'. Must be 'csv'." % self.file_format) else: self.file_format = 'csv' self.preamble = f'{{ "apikey": "{self.api_key}" ' + def check(parameters: dict): + for key in parameters: + if key == 'file_format' and parameters[key] != 'csv': + return [["error", "Invalid file_format '%s'. Must be 'csv'." % parameters[key]]] + elif key == 'country': + return [["warning", "Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0."]] + def _headers(self, data): return {'HMAC2': hmac.new(self.secret.encode(), data.encode('utf-8'), digestmod=hashlib.sha256).hexdigest()} diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index ae38dcb8c..cd750d00b 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,16 +7,28 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. +The report configuration is now stored in a _shadowserver-schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. -For environments that have internet connectivity the `update_schema.py` script should be called from a cron job to obtain the latest revision. -The parser will attempt to download a schema update on startup unless INTELMQ_SKIP_INTERNET is set. +The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. +Schema downloads can also be scheduled as a cron job: + +``` +02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema +``` + +For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. The parser will automatically reload the configuration when the file changes. +## Schema contract + +Once set the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static. + +Once set report fields will not be deleted. + + ## Sample configuration: ``` @@ -46,6 +58,7 @@ shadowserver-parser: parameters: destination_queues: _default: [file-output-queue] + auto_update: true run_mode: continuous ``` diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 5219fdb34..afe3a6b11 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -82,11 +82,12 @@ import base64 import binascii import json -import urllib.request import tempfile from typing import Optional, Dict, Tuple, Any import intelmq.lib.harmonization as harmonization +from intelmq.lib.utils import create_request_session +from intelmq import VAR_STATE_PATH class __Container: @@ -94,8 +95,10 @@ class __Container: __config = __Container() -__config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') +__config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') +__config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') __config.schema_mtime = 0.0 +__config.auto_update = False __config.feedname_mapping = {} __config.filename_mapping = {} @@ -105,13 +108,16 @@ def set_logger(logger): __config.logger = logger +def enable_auto_update(enable): + """ Enable automatic schema update. """ + __config.auto_update = enable + + def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: - reload() return __config.feedname_mapping.get(given_feedname, None) def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, Any]]]: - reload() return __config.filename_mapping.get(given_filename, None) @@ -289,19 +295,18 @@ def reload(): else: __config.logger.info("The schema file does not exist.") - if __config.schema_mtime == 0.0 and mtime == 0.0 and not os.environ.get('INTELMQ_SKIP_INTERNET'): - __config.logger.info("Attempting to download schema.") + if __config.schema_mtime == 0.0 and mtime == 0.0 and __config.auto_update: update_schema() __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [__config.schema_file, ".".join([__config.schema_file, 'test'])]: + for schema_file in [__config.schema_file, __config.schema_base]: if os.path.isfile(schema_file): with open(schema_file) as fh: schema = json.load(fh) for report in schema: if report == "_meta": - __config.logger.info("Loading schema %s." % schema[report]['date_created']) + __config.logger.info("Loading schema %r." % schema[report]['date_created']) for msg in schema[report]['change_log']: __config.logger.info(msg) else: @@ -313,37 +318,55 @@ def reload(): def update_schema(): """ download the latest configuration """ if os.environ.get('INTELMQ_SKIP_INTERNET'): - return None + return False - (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) + # download the schema to a temp file + (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) url = 'https://interchange.shadowserver.org/intelmq/v1/schema' + __config.logger.info("Attempting to download schema from %r" % url) + __config.logger.debug("Using temp file %r for the download." % tmp) try: - urllib.request.urlretrieve(url, tmp) + with create_request_session() as session: + with session.get(url, stream=True) as r: + r.raise_for_status() + with open(tmp, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) except: - raise ValueError("Failed to download %r" % url) + __config.logger.error("Failed to download %r" % url) + return False + __config.logger.info("Download successful.") new_version = '' old_version = '' try: + # validate the downloaded file with open(tmp) as fh: schema = json.load(fh) new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis - raise ValueError("Failed to validate %r" % tmp) + __config.logger.error("Failed to validate %r" % tmp) + return False if os.path.exists(__config.schema_file): + # compare the new version against the old; rename the existing file try: with open(__config.schema_file) as fh: schema = json.load(fh) old_version = schema['_meta']['date_created'] if new_version != old_version: os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) - except: - pass + except Exception as e: + __config.logger.error("Unable to replace schema file: %s" % str(e)) + return False if new_version != old_version: os.replace(tmp, __config.schema_file) + __config.logger.info("New schema version is %r." % new_version) + return True else: os.unlink(tmp) + + return False diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 668a81534..2e383a004 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -26,6 +26,8 @@ from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue +from intelmq.bin.intelmqctl import IntelMQController +import intelmq.lib.utils as utils import intelmq.bots.parsers.shadowserver._config as config @@ -34,8 +36,7 @@ class ShadowserverParserBot(ParserBot): Parse all ShadowServer feeds Parameters: - schema_file (str): Path to the report schema file - + auto_update (boolean): Enable automatic schema download """ recover_line = ParserBot.recover_line_csv_dict @@ -45,13 +46,15 @@ class ShadowserverParserBot(ParserBot): feedname = None _mode = None overwrite = False + auto_update = False def init(self): config.set_logger(self.logger) - try: - config.update_schema() - except Exception as e: - self.logger.warning("Schema update failed: %s." % e) + if self.auto_update: + config.enable_auto_update(True) + self.logger.debug("Feature 'auto_update' is enabled.") + config.reload() + if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: @@ -228,5 +231,35 @@ def parse_line(self, row, report): def shutdown(self): self.feedname = None + @classmethod + def _create_argparser(cls): + argparser = super()._create_argparser() + argparser.add_argument("--update-schema", action='store_true', help='downloads latest report schema') + argparser.add_argument("--verbose", action='store_true', help='be verbose') + return argparser + + @classmethod + def run(cls, parsed_args=None): + if not parsed_args: + parsed_args = cls._create_argparser().parse_args() + if parsed_args.update_schema: + logger = utils.log(__name__, log_path=None) + if parsed_args.verbose: + logger.setLevel('INFO') + else: + logger.setLevel('ERROR') + config.set_logger(logger) + if config.update_schema(): + runtime_conf = utils.get_bots_settings() + try: + ctl = IntelMQController() + for bot in runtime_conf: + if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('auto_update', True): + ctl.bot_reload(bot) + except Exception as e: + logger.error("Failed to signal bot: %r" % str(e)) + else: + super().run(parsed_args=parsed_args) + BOT = ShadowserverParserBot diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py deleted file mode 100644 index a7975147e..000000000 --- a/intelmq/bots/parsers/shadowserver/update_schema.py +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import intelmq.bots.parsers.shadowserver._config as config - -if __name__ == '__main__': # pragma: no cover - config.update_schema() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py new file mode 100644 index 000000000..e68587682 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- +""" +Created on Thu Jul 27 19:44:44 2023 + +""" + +import unittest +import os +import logging +from intelmq import VAR_STATE_PATH +import intelmq.bots.parsers.shadowserver._config as config +import intelmq.lib.utils as utils +import intelmq.lib.test as test + +@test.skip_internet() +class TestShadowserverSchemaDownload(unittest.TestCase): + + def test_download(self): + schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + config.set_logger(utils.log('test-bot', log_path=None)) + if os.path.exists(schema_file): + os.unlink(schema_file) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From 661a96471a991cc3dac8faf4a553874f1164d4d6 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 28 Jul 2023 14:17:41 +0000 Subject: [PATCH 11/67] Removed file_format parameter --- .../shadowserver/collector_reports_api.py | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 05bffa898..66169d96f 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,6 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv'). The 'json' option is no longer supported. """ country = None @@ -42,7 +41,6 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): secret = None types = None reports = None - file_format = None rate_limit: int = 86400 redis_cache_db: int = 12 redis_cache_host: str = "127.0.0.1" # TODO: type could be ipadress @@ -66,18 +64,12 @@ def init(self): self.logger.warn("Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0.") self._report_list.append(self.country) - if self.file_format is not None: - if not (self.file_format == 'csv'): - raise ValueError("Invalid file_format '%s'. Must be 'csv'." % self.file_format) - else: - self.file_format = 'csv' - self.preamble = f'{{ "apikey": "{self.api_key}" ' def check(parameters: dict): for key in parameters: - if key == 'file_format' and parameters[key] != 'csv': - return [["error", "Invalid file_format '%s'. Must be 'csv'." % parameters[key]]] + if key == 'file_format': + return [["error", "The file_format parameter is no longer supported. All reports are CSV."]] elif key == 'country': return [["warning", "Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0."]] @@ -129,11 +121,7 @@ def _report_download(self, reportid: str): data = self.preamble data += f',"id": "{reportid}"}}' self.logger.debug('Downloading report with data: %s.', data) - - if (self.file_format == 'json'): - response = self.http_session().post(APIROOT + 'reports/download', data=data, headers=self._headers(data)) - else: - response = self.http_session().get(DLROOT + reportid) + response = self.http_session().get(DLROOT + reportid) response.raise_for_status() return response.text @@ -150,7 +138,7 @@ def process(self): for item in reportslist: filename = item['file'] - filename_fixed = FILENAME_PATTERN.sub('.' + self.file_format, filename, count=1) + filename_fixed = FILENAME_PATTERN.sub('.csv', filename, count=1) if self.cache_get(filename): self.logger.debug('Processed file %r (fixed: %r) already.', filename, filename_fixed) continue From a045bee263ee0c2b447b8191503d88e3829e5a9f Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 24 Aug 2023 16:04:21 +0000 Subject: [PATCH 12/67] Minor changes based on feedback 2023-08-24 --- CHANGELOG.md | 2 - intelmq/bots/parsers/shadowserver/README.md | 2 + intelmq/bots/parsers/shadowserver/_config.py | 49 ++++++++++--------- intelmq/bots/parsers/shadowserver/parser.py | 6 ++- .../bots/parsers/shadowserver/test_broken.py | 5 ++ .../bots/parsers/shadowserver/test_mapping.py | 1 + .../parsers/shadowserver/test_parameters.py | 3 +- .../parsers/shadowserver/test_report_smb.py | 1 + .../shadowserver/test_report_switch.py | 1 + .../shadowserver/test_report_telnet.py | 1 + 10 files changed, 45 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b54eb84d..b7daa0be0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,10 +68,8 @@ CHANGELOG - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). -======= - `intelmq.bots.collectors.shadowserver.collector_reports_api`: - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) ->>>>>>> Documentation and style updates. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index cd750d00b..4969acb6d 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -28,6 +28,8 @@ Once set the `classification.identifier`, `classification.taxonomy`, and `classi Once set report fields will not be deleted. +The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. + ## Sample configuration: diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index afe3a6b11..4bfadb9d9 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -95,8 +95,10 @@ class __Container: __config = __Container() +__config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema' __config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') __config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') +__config.schema_active = __config.schema_file __config.schema_mtime = 0.0 __config.auto_update = False __config.feedname_mapping = {} @@ -108,6 +110,13 @@ def set_logger(logger): __config.logger = logger +def enable_test_mode(enable): + """ Set which schema to load. """ + if enable: + __config.schema_active = __config.schema_base + else: + __config.schema_active = __config.schema_file + def enable_auto_update(enable): """ Enable automatic schema update. """ __config.auto_update = enable @@ -300,40 +309,36 @@ def reload(): __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [__config.schema_file, __config.schema_base]: - if os.path.isfile(schema_file): - with open(schema_file) as fh: - schema = json.load(fh) - for report in schema: - if report == "_meta": - __config.logger.info("Loading schema %r." % schema[report]['date_created']) - for msg in schema[report]['change_log']: - __config.logger.info(msg) - else: - __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) - __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + if os.path.isfile(__config.schema_active): + with open(__config.schema_active) as fh: + schema = json.load(fh) + for report in schema: + if report == "_meta": + __config.logger.info("Loading schema %r.", schema[report]['date_created']) + for msg in schema[report]['change_log']: + __config.logger.info(msg) + else: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime def update_schema(): """ download the latest configuration """ - if os.environ.get('INTELMQ_SKIP_INTERNET'): - return False # download the schema to a temp file (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) - url = 'https://interchange.shadowserver.org/intelmq/v1/schema' - __config.logger.info("Attempting to download schema from %r" % url) - __config.logger.debug("Using temp file %r for the download." % tmp) + __config.logger.info("Attempting to download schema from %r", __config.schema_url) + __config.logger.debug("Using temp file %r for the download.", tmp) try: with create_request_session() as session: - with session.get(url, stream=True) as r: + with session.get(__config.schema_url, stream=True) as r: r.raise_for_status() with open(tmp, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) except: - __config.logger.error("Failed to download %r" % url) + __config.logger.error("Failed to download %r", __config.schema_url) return False __config.logger.info("Download successful.") @@ -347,7 +352,7 @@ def update_schema(): new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis - __config.logger.error("Failed to validate %r" % tmp) + __config.logger.error("Failed to validate %r", tmp) return False if os.path.exists(__config.schema_file): @@ -359,12 +364,12 @@ def update_schema(): if new_version != old_version: os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) except Exception as e: - __config.logger.error("Unable to replace schema file: %s" % str(e)) + __config.logger.error("Unable to replace schema file: %s", str(e)) return False if new_version != old_version: os.replace(tmp, __config.schema_file) - __config.logger.info("New schema version is %r." % new_version) + __config.logger.info("New schema version is %r.", new_version) return True else: os.unlink(tmp) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 2e383a004..fd9fa6b2c 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -37,6 +37,7 @@ class ShadowserverParserBot(ParserBot): Parameters: auto_update (boolean): Enable automatic schema download + test_mode (boolean): Use test schema """ recover_line = ParserBot.recover_line_csv_dict @@ -47,9 +48,12 @@ class ShadowserverParserBot(ParserBot): _mode = None overwrite = False auto_update = False + test_mode = False def init(self): config.set_logger(self.logger) + if self.test_mode: + config.enable_test_mode(True) if self.auto_update: config.enable_auto_update(True) self.logger.debug("Feature 'auto_update' is enabled.") @@ -254,7 +258,7 @@ def run(cls, parsed_args=None): try: ctl = IntelMQController() for bot in runtime_conf: - if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('auto_update', True): + if runtime_conf[bot]["module"] == __name__: ctl.bot_reload(bot) except Exception as e: logger.error("Failed to signal bot: %r" % str(e)) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 3797f03cd..54a85e780 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -46,6 +46,7 @@ def test_broken(self): """ Test a report which does not have valid fields """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT1 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="Detected report's file name: 'test_smb'.", @@ -59,6 +60,7 @@ def test_half_broken(self): """ Test a report which does not have an optional field. """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT2 self.run_bot(allowed_warning_count=63) self.assertLogMatches(pattern="Detected report's file name: 'test_telnet'.", @@ -72,6 +74,7 @@ def test_no_config(self): """ Test a report which does not have a valid extra.file_name """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT3 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: Could not get a config for 'some_string', check the documentation.") @@ -80,6 +83,7 @@ def test_invalid_filename(self): """ Test a report which does not have a valid extra.file_name """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT4 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: Report's 'extra.file_name' '2020.wrong-filename.csv' is not valid.") @@ -89,6 +93,7 @@ def test_no_report_name(self): Test a report without file_name and no given feedname as parameter. Error message should be verbose. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: No feedname given as parameter and the " "processed report has no 'extra.file_name'. " diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index d296dfdc2..b764de827 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -48,6 +48,7 @@ def test_changed_feed(self): Tests if the parser correctly re-detects the feed for the second received report #1493 """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = (EXAMPLE_TELNET, EXAMPLE_VNC) self.run_bot(iterations=2) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py index 677cd0319..45a4a8735 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py @@ -63,13 +63,14 @@ def set_bot(cls): def test_default(self): """ Test if feed name is not overwritten has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) def test_overwrite_feed_name(self): """ Test if feed name is overwritten if asked to do so. """ - self.prepare_bot(parameters={'overwrite': True}) + self.prepare_bot(parameters={'test_mode': True, 'overwrite': True}) self.run_bot(prepare=False) for i, EVENT in enumerate(EVENTS): event = EVENT.copy() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py index 93d592d15..aa6940061 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -108,6 +108,7 @@ def set_bot(cls): def test_event(self): """ Test if correct Event has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index a9be8a0a1..488f5a51a 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -46,6 +46,7 @@ def set_bot(cls): def test_event(self): """ Test if the parser correctly detects and handles different report types. """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = [FIRST_REPORT, SECOND_REPORT] self.run_bot(iterations=2) self.assertLogMatches("Detected report's file name: 'test_smb'", diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py index df9cf25dc..b2499c589 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -78,6 +78,7 @@ def set_bot(cls): def test_event(self): """ Test if correct Event has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) From 0660a893e4a520973b8dfd6e27b93240574bb007 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 24 Aug 2023 16:26:59 +0000 Subject: [PATCH 13/67] Added VAR_STATE_PATH check. --- intelmq/bots/parsers/shadowserver/_config.py | 1 + .../parsers/shadowserver/test_download_schema.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 4bfadb9d9..6ffffdae8 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -117,6 +117,7 @@ def enable_test_mode(enable): else: __config.schema_active = __config.schema_file + def enable_auto_update(enable): """ Enable automatic schema update. """ __config.auto_update = enable diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index e68587682..f9512ca98 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -20,9 +20,10 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') - config.set_logger(utils.log('test-bot', log_path=None)) - if os.path.exists(schema_file): - os.unlink(schema_file) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + if os.path.isdir(VAR_STATE_PATH): + schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + config.set_logger(utils.log('test-bot', log_path=None)) + if os.path.exists(schema_file): + os.unlink(schema_file) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From 33370cf1eeb4fefef24ca8f7c2ea34dc02c97b42 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 15:37:51 +0000 Subject: [PATCH 14/67] Changes based on feedback 2023-08-25. --- CHANGELOG.md | 6 +- docs/user/bots.rst | 171 ++++++------------ intelmq/bots/parsers/shadowserver/README.md | 57 ------ intelmq/bots/parsers/shadowserver/_config.py | 10 +- .../shadowserver/test_download_schema.py | 8 +- 5 files changed, 72 insertions(+), 180 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7daa0be0..0e9ede890 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,20 +62,18 @@ CHANGELOG ### Bots #### Collectors -<<<<<<< HEAD - `intelmq.bots.collector.rt`: - restrict `python-rt` to be below version 3.0 due to introduced breaking changes, - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). - `intelmq.bots.collectors.shadowserver.collector_reports_api`: - - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) + - The 'json' option is no longer supported as the 'csv' option provides better performance. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: - Reset detected `feedname` at shutdown to re-detect the feedname on reloads (PR#2361 by @elsif2, fixes #2360). - - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. Please see intelmq/bots/parsers/shadowserver/README.md for details. (PR#2372) -- `intelmq.bots.parsers.shadowserver._config`: + - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. - Added 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Service' report. (PR#2338) - Removed unused `p0f_genre` and `p0f_detail` from the 'DNS-Open-Resolvers' report. (PR#2338) - Added 'Accessible-SIP' report. (PR#2348) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index 8e8f36396..3da99af1e 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -673,6 +673,23 @@ The resulting reports contain the following special field: * `extra.file_name`: The name of the downloaded file, with fixed filename extension. The API returns file names with the extension `.csv`, although the files are JSON, not CSV. Therefore, for clarity and better error detection in the parser, the file name in `extra.file_name` uses `.json` as extension. +**Sample configuration** + +.. code-block:: yaml + + shadowserver-collector: + description: Our bot responsible for getting reports from Shadowserver + enabled: true + group: Collector + module: intelmq.bots.collectors.shadowserver.collector_reports_api + name: Shadowserver_Collector + parameters: + destination_queues: + _default: [shadowserver-parser-queue] + file_format: csv + api_key: "$API_KEY_received_from_the_shadowserver_foundation" + secret: "$SECRET_received_from_the_shadowserver_foundation" + run_mode: continuous .. _intelmq.bots.collectors.shodan.collector_stream: @@ -1554,17 +1571,15 @@ This does not affect URLs which already include the scheme. .. _intelmq.bots.parsers.shadowserver.parser: -.. _intelmq.bots.parsers.shadowserver.parser_json: Shadowserver ^^^^^^^^^^^^ -There are two Shadowserver parsers, one for data in ``CSV`` format (``intelmq.bots.parsers.shadowserver.parser``) and one for data in ``JSON`` format (``intelmq.bots.parsers.shadowserver.parser_json``). -The latter was added in IntelMQ 2.3 and is meant to be used together with the Shadowserver API collector. +The Shadowserver parser operates on ``CSV`` formatted data. **Information** -* `name:` `intelmq.bots.parsers.shadowserver.parser` (for CSV data) or `intelmq.bots.parsers.shadowserver.parser_json` (for JSON data) +* `name:` `intelmq.bots.parsers.shadowserver.parser` * `public:` yes * `description:` Parses different reports from Shadowserver. @@ -1600,107 +1615,45 @@ A list of possible feeds can be found in the table below in the column "feed nam **Supported reports** -These are the supported feed name and their corresponding file name for automatic detection: - - ======================================= ========================= - feed name file name - ======================================= ========================= - Accessible-ADB `scan_adb` - Accessible-AFP `scan_afp` - Accessible-AMQP `scan_amqp` - Accessible-ARD `scan_ard` - Accessible-Cisco-Smart-Install `cisco_smart_install` - Accessible-CoAP `scan_coap` - Accessible-CWMP `scan_cwmp` - Accessible-MS-RDPEUDP `scan_msrdpeudp` - Accessible-FTP `scan_ftp` - Accessible-Hadoop `scan_hadoop` - Accessible-HTTP `scan_http` - Accessible-Radmin `scan_radmin` - Accessible-RDP `scan_rdp` - Accessible-Rsync `scan_rsync` - Accessible-SMB `scan_smb` - Accessible-Telnet `scan_telnet` - Accessible-Ubiquiti-Discovery-Service `scan_ubiquiti` - Accessible-VNC `scan_vnc` - Blacklisted-IP (deprecated) `blacklist` - Blocklist `blocklist` - Compromised-Website `compromised_website` - Device-Identification IPv4 / IPv6 `device_id`/`device_id6` - DNS-Open-Resolvers `scan_dns` - Honeypot-Amplification-DDoS-Events `event4_honeypot_ddos_amp` - Honeypot-Brute-Force-Events `event4_honeypot_brute_force` - Honeypot-Darknet `event4_honeypot_darknet` - Honeypot-HTTP-Scan `event4_honeypot_http_scan` - HTTP-Scanners `hp_http_scan` - ICS-Scanners `hp_ics_scan` - IP-Spoofer-Events `event4_ip_spoofer` - Microsoft-Sinkhole-Events IPv4 `event4_microsoft_sinkhole` - Microsoft-Sinkhole-Events-HTTP IPv4 `event4_microsoft_sinkhole_http` - NTP-Monitor `scan_ntpmonitor` - NTP-Version `scan_ntp` - Open-Chargen `scan_chargen` - Open-DB2-Discovery-Service `scan_db2` - Open-Elasticsearch `scan_elasticsearch` - Open-IPMI `scan_ipmi` - Open-IPP `scan_ipp` - Open-LDAP `scan_ldap` - Open-LDAP-TCP `scan_ldap_tcp` - Open-mDNS `scan_mdns` - Open-Memcached `scan_memcached` - Open-MongoDB `scan_mongodb` - Open-MQTT `scan_mqtt` - Open-MSSQL `scan_mssql` - Open-NATPMP `scan_nat_pmp` - Open-NetBIOS-Nameservice `scan_netbios` - Open-Netis `netis_router` - Open-Portmapper `scan_portmapper` - Open-QOTD `scan_qotd` - Open-Redis `scan_redis` - Open-SNMP `scan_snmp` - Open-SSDP `scan_ssdp` - Open-TFTP `scan_tftp` - Open-XDMCP `scan_xdmcp` - Outdated-DNSSEC-Key `outdated_dnssec_key` - Outdated-DNSSEC-Key-IPv6 `outdated_dnssec_key_v6` - Sandbox-URL `cwsandbox_url` - Sinkhole-DNS `sinkhole_dns` - Sinkhole-Events `event4_sinkhole`/`event6_sinkhole` - Sinkhole-Events IPv4 `event4_sinkhole` - Sinkhole-Events IPv6 `event6_sinkhole` - Sinkhole-HTTP-Events `event4_sinkhole_http`/`event6_sinkhole_http` - Sinkhole-HTTP-Events IPv4 `event4_sinkhole_http` - Sinkhole-HTTP-Events IPv6 `event6_sinkhole_http` - Sinkhole-Events-HTTP-Referer `event4_sinkhole_http_referer`/`event6_sinkhole_http_referer` - Sinkhole-Events-HTTP-Referer IPv4 `event4_sinkhole_http_referer` - Sinkhole-Events-HTTP-Referer IPv6 `event6_sinkhole_http_referer` - Spam-URL `spam_url` - SSL-FREAK-Vulnerable-Servers `scan_ssl_freak` - SSL-POODLE-Vulnerable-Servers `scan_ssl_poodle`/`scan6_ssl_poodle` - Vulnerable-Exchange-Server `*` `scan_exchange` - Vulnerable-ISAKMP `scan_isakmp` - Vulnerable-HTTP `scan_http` - Vulnerable-SMTP `scan_smtp_vulnerable` - ======================================= ========================= - -`*` This report can also contain data on active webshells (column `tag` is `exchange;webshell`), and are therefore not only vulnerable but also actively infected. - -In addition, the following legacy reports are supported: - - =========================== =================================================== ======================== - feed name successor feed name file name - =========================== =================================================== ======================== - Amplification-DDoS-Victim Honeypot-Amplification-DDoS-Events ``ddos_amplification`` - CAIDA-IP-Spoofer IP-Spoofer-Events ``caida_ip_spoofer`` - Darknet Honeypot-Darknet ``darknet`` - Drone Sinkhole-Events ``botnet_drone`` - Drone-Brute-Force Honeypot-Brute-Force-Events, Sinkhole-HTTP-Events ``drone_brute_force`` - Microsoft-Sinkhole Sinkhole-HTTP-Events ``microsoft_sinkhole`` - Sinkhole-HTTP-Drone Sinkhole-HTTP-Events ``sinkhole_http_drone`` - IPv6-Sinkhole-HTTP-Drone Sinkhole-HTTP-Events ``sinkhole6_http`` - =========================== =================================================== ======================== - -More information on these legacy reports can be found in `Changes in Sinkhole and Honeypot Report Types and Formats `_. +The report configuration is stored in a `shadowserver-schema.json` file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. + +The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. + +Schema downloads can also be scheduled as a cron job: + +.. code-block:: bash + + 02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema + + +For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. + +The parser will automatically reload the configuration when the file changes. + +**Schema contract** + +Once set in the schema, the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static for a specific report. + +Report fields will not be removed from a report. + +The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. + +**Sample configuration** + +.. code-block:: yaml + + shadowserver-parser: + bot_id: shadowserver-parser + name: Shadowserver Parser + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.shadowserver.parser + parameters: + destination_queues: + _default: [file-output-queue] + auto_update: true + run_mode: continuous **Development** @@ -1712,14 +1665,6 @@ The parser consists of two files: Both files are required for the parser to work properly. -**Add new Feedformats** - -Add a new feed format and conversions if required to the file -``_config.py``. Don't forget to update the ``mapping`` dict. -It is required to look up the correct configuration. - -Look at the documentation in the bot's ``_config.py`` file for more information. - .. _intelmq.bots.parsers.shodan.parser: diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index 4969acb6d..eb0ddfb4a 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,60 +7,3 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _shadowserver-schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. - -The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. - -Schema downloads can also be scheduled as a cron job: - -``` -02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema -``` - -For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. - -The parser will automatically reload the configuration when the file changes. - - -## Schema contract - -Once set the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static. - -Once set report fields will not be deleted. - -The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. - - -## Sample configuration: - -``` -shadowserver-collector: - description: Our bot responsible for getting reports from Shadowserver - enabled: true - group: Collector - module: intelmq.bots.collectors.shadowserver.collector_reports_api - name: Shadowserver_Collector - parameters: - destination_queues: - _default: [shadowserver-parser-queue] - file_format: csv - api_key: "$API_KEY_received_from_the_shadowserver_foundation" - secret: "$SECRET_received_from_the_shadowserver_foundation" - run_mode: continuous -``` - -``` -shadowserver-parser: - bot_id: shadowserver-parser - name: Shadowserver Parser - enabled: true - group: Parser - groupname: parsers - module: intelmq.bots.parsers.shadowserver.parser - parameters: - destination_queues: - _default: [file-output-queue] - auto_update: true - run_mode: continuous -``` - diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 6ffffdae8..279093dfe 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -95,6 +95,7 @@ class __Container: __config = __Container() +__config.var_state_path = VAR_STATE_PATH __config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema' __config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') __config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') @@ -328,7 +329,7 @@ def update_schema(): """ download the latest configuration """ # download the schema to a temp file - (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) + (th, tmp) = tempfile.mkstemp(dir=__config.var_state_path) __config.logger.info("Attempting to download schema from %r", __config.schema_url) __config.logger.debug("Using temp file %r for the download.", tmp) try: @@ -376,3 +377,10 @@ def update_schema(): os.unlink(tmp) return False + + +def prepare_update_schema_test(path): + """ Reconfigure internal settings to perform a schema update test. """ + __config.var_state_path = path + __config.schema_file = os.path.join(path, 'shadowserver-schema.json') + return __config.schema_file diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index f9512ca98..5246e6bb6 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -10,8 +10,8 @@ import unittest import os +import tempfile import logging -from intelmq import VAR_STATE_PATH import intelmq.bots.parsers.shadowserver._config as config import intelmq.lib.utils as utils import intelmq.lib.test as test @@ -20,10 +20,8 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - if os.path.isdir(VAR_STATE_PATH): - schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) config.set_logger(utils.log('test-bot', log_path=None)) - if os.path.exists(schema_file): - os.unlink(schema_file) self.assertEqual(True, config.update_schema()) self.assertEqual(True, os.path.exists(schema_file)) From bd76ab71368485bfcba545a8005f442bc90e6ce2 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 15:51:38 +0000 Subject: [PATCH 15/67] Added INTELMQ_SKIP_INTERNET check --- .../bots/parsers/shadowserver/test_download_schema.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index 5246e6bb6..203a3c0b1 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -20,8 +20,9 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - with tempfile.TemporaryDirectory() as tmp_dir: - schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None)) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + if not os.environ.get('INTELMQ_SKIP_INTERNET'): + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) + config.set_logger(utils.log('test-bot', log_path=None)) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From 6e5e110f3baae88c9782e7c214a8cb1a6cdcbf51 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 16:11:21 +0000 Subject: [PATCH 16/67] Added debug logging for CI test. --- intelmq/bots/parsers/shadowserver/_config.py | 3 ++- .../tests/bots/parsers/shadowserver/test_download_schema.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 279093dfe..d573d12c6 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -339,8 +339,9 @@ def update_schema(): with open(tmp, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) - except: + except Exception as e: __config.logger.error("Failed to download %r", __config.schema_url) + __config.logger.debug(str(e)) return False __config.logger.info("Download successful.") diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index 203a3c0b1..abcd0ca2a 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -23,6 +23,6 @@ def test_download(self): if not os.environ.get('INTELMQ_SKIP_INTERNET'): with tempfile.TemporaryDirectory() as tmp_dir: schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None)) + config.set_logger(utils.log('test-bot', log_path=None, log_level=logging.DEBUG)) self.assertEqual(True, config.update_schema()) self.assertEqual(True, os.path.exists(schema_file)) From 01dcd5ee8d0ee6c99ab293526aaac2a43f2b2b22 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 18:47:54 +0000 Subject: [PATCH 17/67] Refactored test_download_schema to utilize mocking. --- intelmq/bots/parsers/shadowserver/parser.py | 6 ++++ .../shadowserver/test_download_schema.py | 30 ++++++++++++------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index fd9fa6b2c..48cbba901 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -23,6 +23,7 @@ import copy import re import os +import tempfile from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue @@ -265,5 +266,10 @@ def run(cls, parsed_args=None): else: super().run(parsed_args=parsed_args) + def test_update_schema(cls): + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) + return config.update_schema() + BOT = ShadowserverParserBot diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index abcd0ca2a..abf27a5bd 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -8,21 +8,29 @@ """ -import unittest -import os -import tempfile import logging -import intelmq.bots.parsers.shadowserver._config as config +import unittest +import unittest.mock as mock +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot import intelmq.lib.utils as utils import intelmq.lib.test as test + @test.skip_internet() -class TestShadowserverSchemaDownload(unittest.TestCase): +class TestShadowserverSchemaDownload(test.BotTestCase, unittest.TestCase): + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.sysconfig = {"logging_level": "DEBUG"} def test_download(self): - if not os.environ.get('INTELMQ_SKIP_INTERNET'): - with tempfile.TemporaryDirectory() as tmp_dir: - schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None, log_level=logging.DEBUG)) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + self.prepare_bot(prepare_source_queue=False, parameters={'test_mode': True}) + result = False + with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): + with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)): + self.log_stream.truncate(0) + result = self.bot.test_update_schema() + self.bot.stop(exitcode=0) + print(self.log_stream.getvalue()) + self.assertEqual(True, result) From 9314c84c19ef06cdfad508c64d3398785d82fff8 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 28 Aug 2023 14:18:22 +0000 Subject: [PATCH 18/67] Added docstring for test_update_schema(). --- intelmq/bots/parsers/shadowserver/parser.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 48cbba901..4485a2602 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -267,6 +267,13 @@ def run(cls, parsed_args=None): super().run(parsed_args=parsed_args) def test_update_schema(cls): + """ + Test schema download to a temporary directory. + + This is necessary as the request session requires mocking in order to function. + + Returns True on success. + """ with tempfile.TemporaryDirectory() as tmp_dir: schema_file = config.prepare_update_schema_test(tmp_dir) return config.update_schema() From 2f11b2a6667393c25eacd2584d14ade09065eb15 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 29 Aug 2023 14:09:33 +0000 Subject: [PATCH 19/67] Removed logging output. --- intelmq/tests/bots/parsers/shadowserver/test_download_schema.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index abf27a5bd..84922bf17 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -29,8 +29,6 @@ def test_download(self): result = False with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)): - self.log_stream.truncate(0) result = self.bot.test_update_schema() self.bot.stop(exitcode=0) - print(self.log_stream.getvalue()) self.assertEqual(True, result) From 46f2ca775df9591b293c07e2a1a049d49264d25a Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 31 Aug 2023 20:52:17 +0000 Subject: [PATCH 20/67] Removed the assertion regarding report fields. --- docs/user/bots.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index 3da99af1e..8d3c7555d 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -1634,8 +1634,6 @@ The parser will automatically reload the configuration when the file changes. Once set in the schema, the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static for a specific report. -Report fields will not be removed from a report. - The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. **Sample configuration** From d0311e0058b7e8c7125fed5ff50e9abe454afc64 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:53:24 +0000 Subject: [PATCH 21/67] remove obsolete tests and data --- .../shadowserver/scan_rdpeudp.csv.license | 2 - .../parsers/shadowserver/test_blocklist.py | 103 ------- .../shadowserver/test_compromised_website.py | 88 ------ .../parsers/shadowserver/test_device_id.py | 116 -------- .../test_event4_ddos_participant.py | 131 --------- .../test_event4_honeypot_darknet.py | 106 ------- .../shadowserver/test_event4_honeypot_ddos.py | 148 ---------- .../test_event4_honeypot_ddos_target.py | 150 ---------- .../test_event4_honeypot_http_scan.py | 109 -------- .../shadowserver/test_event4_ip_spoofer.py | 182 ------------ .../test_event4_microsoft_sinkhole.py | 135 --------- .../test_event4_microsoft_sinkhole_http.py | 202 -------------- .../shadowserver/test_event4_sinkhole.py | 73 ----- .../shadowserver/test_event4_sinkhole_dns.py | 127 --------- .../shadowserver/test_event4_sinkhole_http.py | 189 ------------- .../test_event4_sinkhole_http_referer.py | 213 --------------- .../shadowserver/test_event6_sinkhole_http.py | 146 ---------- .../shadowserver/test_honeypot_brute_force.py | 72 ----- .../shadowserver/test_honeypot_ddos_amp.py | 91 ------ .../parsers/shadowserver/test_malware_url.py | 107 -------- .../parsers/shadowserver/test_phish_url.py | 106 ------- .../test_population_http_proxy.py | 130 --------- .../parsers/shadowserver/test_sandbox_conn.py | 99 ------- .../parsers/shadowserver/test_sandbox_dns.py | 95 ------- .../parsers/shadowserver/test_sandbox_url.py | 104 ------- .../parsers/shadowserver/test_scan_adb.py | 98 ------- .../parsers/shadowserver/test_scan_afp.py | 106 ------- .../parsers/shadowserver/test_scan_amqp.py | 144 ---------- .../parsers/shadowserver/test_scan_ard.py | 111 -------- .../parsers/shadowserver/test_scan_chargen.py | 110 -------- .../test_scan_cisco_smart_install.py | 82 ------ .../parsers/shadowserver/test_scan_coap.py | 121 -------- .../parsers/shadowserver/test_scan_couchdb.py | 128 --------- .../parsers/shadowserver/test_scan_cwmp.py | 103 ------- .../parsers/shadowserver/test_scan_db2.py | 91 ------ .../shadowserver/test_scan_ddos_middlebox.py | 119 -------- .../parsers/shadowserver/test_scan_dns.py | 91 ------ .../parsers/shadowserver/test_scan_docker.py | 159 ----------- .../test_scan_dvr_dhcpdiscover.py | 178 ------------ .../shadowserver/test_scan_elasticsearch.py | 126 --------- .../shadowserver/test_scan_exchange.py | 149 ---------- .../parsers/shadowserver/test_scan_ftp.py | 120 -------- .../parsers/shadowserver/test_scan_hadoop.py | 94 ------- .../parsers/shadowserver/test_scan_http.py | 100 ------- .../shadowserver/test_scan_http_proxy.py | 118 -------- .../shadowserver/test_scan_http_vulnerable.py | 125 --------- .../parsers/shadowserver/test_scan_ics.py | 125 --------- .../parsers/shadowserver/test_scan_ipmi.py | 106 ------- .../parsers/shadowserver/test_scan_ipp.py | 79 ------ .../parsers/shadowserver/test_scan_isakmp.py | 105 ------- .../shadowserver/test_scan_kubernetes.py | 214 --------------- .../shadowserver/test_scan_ldap_tcp.py | 154 ----------- .../shadowserver/test_scan_ldap_udp.py | 162 ----------- .../parsers/shadowserver/test_scan_mdns.py | 127 --------- .../shadowserver/test_scan_memcached.py | 130 --------- .../parsers/shadowserver/test_scan_mongodb.py | 103 ------- .../parsers/shadowserver/test_scan_mqtt.py | 89 ------ .../shadowserver/test_scan_mqtt_anon.py | 173 ------------ .../parsers/shadowserver/test_scan_mssql.py | 123 --------- .../parsers/shadowserver/test_scan_mysql.py | 258 ------------------ .../parsers/shadowserver/test_scan_nat_pmp.py | 116 -------- .../parsers/shadowserver/test_scan_netbios.py | 121 -------- .../shadowserver/test_scan_netis_router.py | 107 -------- .../parsers/shadowserver/test_scan_ntp.py | 161 ----------- .../shadowserver/test_scan_ntpmonitor.py | 108 -------- .../shadowserver/test_scan_portmapper.py | 120 -------- .../shadowserver/test_scan_postgres.py | 199 -------------- .../parsers/shadowserver/test_scan_qotd.py | 119 -------- .../parsers/shadowserver/test_scan_quic.py | 118 -------- .../parsers/shadowserver/test_scan_radmin.py | 236 ---------------- .../parsers/shadowserver/test_scan_rdp.py | 117 -------- .../parsers/shadowserver/test_scan_rdpeudp.py | 109 -------- .../parsers/shadowserver/test_scan_redis.py | 107 -------- .../parsers/shadowserver/test_scan_rsync.py | 116 -------- .../parsers/shadowserver/test_scan_sip.py | 124 --------- .../parsers/shadowserver/test_scan_slp.py | 137 ---------- .../parsers/shadowserver/test_scan_smb.py | 124 --------- .../shadowserver/test_scan_smb_json.py | 123 --------- .../shadowserver/test_scan_smtp_vulnerable.py | 92 ------- .../parsers/shadowserver/test_scan_snmp.py | 120 -------- .../parsers/shadowserver/test_scan_socks.py | 107 -------- .../parsers/shadowserver/test_scan_ssdp.py | 136 --------- .../parsers/shadowserver/test_scan_ssh.py | 182 ------------ .../parsers/shadowserver/test_scan_ssl.py | 218 --------------- .../shadowserver/test_scan_ssl_freak.py | 136 --------- .../shadowserver/test_scan_ssl_poodle.py | 91 ------ .../parsers/shadowserver/test_scan_stun.py | 146 ---------- .../shadowserver/test_scan_synfulknock.py | 117 -------- .../parsers/shadowserver/test_scan_telnet.py | 87 ------ .../parsers/shadowserver/test_scan_tftp.py | 121 -------- .../shadowserver/test_scan_ubiquiti.py | 124 --------- .../parsers/shadowserver/test_scan_vnc.py | 86 ------ .../shadowserver/test_scan_ws_discovery.py | 119 -------- .../parsers/shadowserver/test_scan_xdmcp.py | 117 -------- .../bots/parsers/shadowserver/test_special.py | 106 ------- .../parsers/shadowserver/test_testdata.py | 81 ------ .../shadowserver/testdata/blocklist.csv | 4 - .../testdata/blocklist.csv.license | 2 - .../testdata/botnet_drone.csv.license | 2 - .../testdata/caida_ip_spoofer.csv.license | 2 - .../testdata/compromised_website.csv | 4 - .../testdata/compromised_website.csv.license | 2 - .../shadowserver/testdata/darknet.csv.license | 2 - .../testdata/ddos_amplification.csv.license | 2 - .../shadowserver/testdata/device_id.csv | 4 - .../testdata/device_id.csv.license | 2 - .../testdata/drone_brute_force.csv.license | 2 - .../testdata/event4_ddos_participant.csv | 4 - .../event4_ddos_participant.csv.license | 2 - .../testdata/event4_honeypot_brute_force.csv | 7 - .../event4_honeypot_brute_force.csv.license | 2 - .../testdata/event4_honeypot_darknet.csv | 9 - .../event4_honeypot_darknet.csv.license | 2 - .../testdata/event4_honeypot_ddos.csv | 4 - .../testdata/event4_honeypot_ddos.csv.license | 2 - .../testdata/event4_honeypot_ddos_amp.csv | 6 - .../event4_honeypot_ddos_amp.csv.license | 2 - .../testdata/event4_honeypot_ddos_target.csv | 4 - .../event4_honeypot_ddos_target.csv.license | 2 - .../testdata/event4_honeypot_http_scan.csv | 3 - .../event4_honeypot_http_scan.csv.license | 2 - .../testdata/event4_ip_spoofer.csv | 7 - .../testdata/event4_ip_spoofer.csv.license | 2 - .../testdata/event4_microsoft_sinkhole.csv | 7 - .../event4_microsoft_sinkhole.csv.license | 2 - .../event4_microsoft_sinkhole_http.csv | 6 - ...event4_microsoft_sinkhole_http.csv.license | 2 - .../shadowserver/testdata/event4_sinkhole.csv | 4 - .../testdata/event4_sinkhole.csv.license | 2 - .../testdata/event4_sinkhole_dns.csv | 4 - .../testdata/event4_sinkhole_dns.csv.license | 2 - .../testdata/event4_sinkhole_http.csv | 6 - .../testdata/event4_sinkhole_http.csv.license | 2 - .../testdata/event4_sinkhole_http_referer.csv | 6 - .../event4_sinkhole_http_referer.csv.license | 2 - .../testdata/event6_sinkhole_http.csv | 4 - .../testdata/event6_sinkhole_http.csv.license | 2 - .../testdata/hp_http_scan.csv.license | 2 - .../testdata/hp_ics_scan.csv.license | 2 - .../shadowserver/testdata/malware_url.csv | 4 - .../testdata/malware_url.csv.license | 2 - .../testdata/outdated_dnssec_key.csv.license | 2 - .../shadowserver/testdata/phish_url.csv | 4 - .../testdata/phish_url.csv.license | 2 - .../testdata/population_http_proxy.csv | 4 - .../population_http_proxy.csv.license | 2 - .../shadowserver/testdata/sandbox_conn.csv | 4 - .../testdata/sandbox_conn.csv.license | 2 - .../shadowserver/testdata/sandbox_dns.csv | 4 - .../testdata/sandbox_dns.csv.license | 2 - .../shadowserver/testdata/sandbox_url.csv | 4 - .../testdata/sandbox_url.csv.license | 2 - .../shadowserver/testdata/scan_adb.csv | 3 - .../testdata/scan_adb.csv.license | 2 - .../shadowserver/testdata/scan_afp.csv | 3 - .../testdata/scan_afp.csv.license | 2 - .../shadowserver/testdata/scan_amqp.csv | 4 - .../testdata/scan_amqp.csv.license | 2 - .../shadowserver/testdata/scan_ard.csv | 4 - .../testdata/scan_ard.csv.license | 2 - .../shadowserver/testdata/scan_chargen.csv | 4 - .../testdata/scan_chargen.csv.license | 2 - .../testdata/scan_cisco_smart_install.csv | 3 - .../scan_cisco_smart_install.csv.license | 2 - .../shadowserver/testdata/scan_coap.csv | 4 - .../testdata/scan_coap.csv.license | 2 - .../shadowserver/testdata/scan_couchdb.csv | 4 - .../testdata/scan_couchdb.csv.license | 2 - .../shadowserver/testdata/scan_cwmp.csv | 3 - .../testdata/scan_cwmp.csv.license | 2 - .../shadowserver/testdata/scan_db2.csv | 3 - .../testdata/scan_db2.csv.license | 2 - .../testdata/scan_ddos_middlebox.csv | 4 - .../testdata/scan_ddos_middlebox.csv.license | 2 - .../shadowserver/testdata/scan_dns.csv | 101 ------- .../testdata/scan_dns.csv.license | 2 - .../shadowserver/testdata/scan_docker.csv | 4 - .../testdata/scan_docker.csv.license | 2 - .../testdata/scan_dvr_dhcpdiscover.csv | 4 - .../scan_dvr_dhcpdiscover.csv.license | 2 - .../testdata/scan_elasticsearch.csv | 4 - .../testdata/scan_elasticsearch.csv.license | 2 - .../shadowserver/testdata/scan_exchange.csv | 8 - .../testdata/scan_exchange.csv.license | 2 - .../shadowserver/testdata/scan_ftp.csv | 3 - .../testdata/scan_ftp.csv.license | 2 - .../shadowserver/testdata/scan_hadoop.csv | 3 - .../testdata/scan_hadoop.csv.license | 2 - .../shadowserver/testdata/scan_http.csv | 3 - .../testdata/scan_http.csv.license | 2 - .../shadowserver/testdata/scan_http_proxy.csv | 4 - .../testdata/scan_http_proxy.csv.license | 2 - .../testdata/scan_http_vulnerable.csv | 4 - .../testdata/scan_http_vulnerable.csv.license | 2 - .../shadowserver/testdata/scan_ics.csv | 4 - .../testdata/scan_ics.csv.license | 2 - .../shadowserver/testdata/scan_ipmi.csv | 96 ------- .../testdata/scan_ipmi.csv.license | 2 - .../shadowserver/testdata/scan_ipp.csv | 2 - .../testdata/scan_ipp.csv.license | 2 - .../shadowserver/testdata/scan_isakmp.csv | 3 - .../testdata/scan_isakmp.csv.license | 2 - .../shadowserver/testdata/scan_kubernetes.csv | 4 - .../testdata/scan_kubernetes.csv.license | 2 - .../shadowserver/testdata/scan_ldap_tcp.csv | 4 - .../testdata/scan_ldap_tcp.csv.license | 2 - .../shadowserver/testdata/scan_ldap_udp.csv | 4 - .../testdata/scan_ldap_udp.csv.license | 2 - .../shadowserver/testdata/scan_mdns.csv | 4 - .../testdata/scan_mdns.csv.license | 2 - .../shadowserver/testdata/scan_memcached.csv | 4 - .../testdata/scan_memcached.csv.license | 2 - .../shadowserver/testdata/scan_mongodb.csv | 11 - .../testdata/scan_mongodb.csv.license | 2 - .../shadowserver/testdata/scan_mqtt.csv | 2 - .../testdata/scan_mqtt.csv.license | 2 - .../shadowserver/testdata/scan_mqtt_anon.csv | 4 - .../testdata/scan_mqtt_anon.csv.license | 2 - .../shadowserver/testdata/scan_mssql.csv | 4 - .../testdata/scan_mssql.csv.license | 2 - .../shadowserver/testdata/scan_mysql.csv | 4 - .../testdata/scan_mysql.csv.license | 2 - .../shadowserver/testdata/scan_nat_pmp.csv | 4 - .../testdata/scan_nat_pmp.csv.license | 2 - .../shadowserver/testdata/scan_netbios.csv | 4 - .../testdata/scan_netbios.csv.license | 2 - .../testdata/scan_netis_router.csv | 4 - .../testdata/scan_netis_router.csv.license | 2 - .../shadowserver/testdata/scan_ntp.csv | 4 - .../testdata/scan_ntp.csv.license | 2 - .../shadowserver/testdata/scan_ntpmonitor.csv | 4 - .../testdata/scan_ntpmonitor.csv.license | 2 - .../shadowserver/testdata/scan_portmapper.csv | 4 - .../testdata/scan_portmapper.csv.license | 2 - .../shadowserver/testdata/scan_postgres.csv | 4 - .../testdata/scan_postgres.csv.license | 2 - .../shadowserver/testdata/scan_qotd.csv | 4 - .../testdata/scan_qotd.csv.license | 2 - .../shadowserver/testdata/scan_quic.csv | 4 - .../testdata/scan_quic.csv.license | 2 - .../shadowserver/testdata/scan_radmin.csv | 10 - .../testdata/scan_radmin.csv.license | 2 - .../shadowserver/testdata/scan_rdp.csv | 3 - .../testdata/scan_rdp.csv.license | 2 - .../shadowserver/testdata/scan_rdpeudp.csv | 4 - .../testdata/scan_rdpeudp.csv.license | 2 - .../shadowserver/testdata/scan_redis.csv | 94 ------- .../testdata/scan_redis.csv.license | 2 - .../shadowserver/testdata/scan_rsync.csv | 4 - .../testdata/scan_rsync.csv.license | 2 - .../shadowserver/testdata/scan_sip.csv | 4 - .../testdata/scan_sip.csv.license | 2 - .../shadowserver/testdata/scan_slp.csv | 4 - .../testdata/scan_slp.csv.license | 2 - .../shadowserver/testdata/scan_smb.csv | 4 - .../testdata/scan_smb.csv.license | 2 - .../testdata/scan_smtp_vulnerable.csv | 3 - .../testdata/scan_smtp_vulnerable.csv.license | 2 - .../shadowserver/testdata/scan_snmp.csv | 4 - .../testdata/scan_snmp.csv.license | 2 - .../shadowserver/testdata/scan_socks.csv | 4 - .../testdata/scan_socks.csv.license | 2 - .../shadowserver/testdata/scan_ssdp.csv | 4 - .../testdata/scan_ssdp.csv.license | 2 - .../shadowserver/testdata/scan_ssh.csv | 4 - .../testdata/scan_ssh.csv.license | 2 - .../shadowserver/testdata/scan_ssl.csv | 4 - .../testdata/scan_ssl.csv.license | 2 - .../shadowserver/testdata/scan_ssl_freak.csv | 46 ---- .../testdata/scan_ssl_freak.csv.license | 2 - .../shadowserver/testdata/scan_ssl_poodle.csv | 32 --- .../testdata/scan_ssl_poodle.csv.license | 2 - .../shadowserver/testdata/scan_stun.csv | 4 - .../testdata/scan_stun.csv.license | 2 - .../testdata/scan_synfulknock.csv | 4 - .../testdata/scan_synfulknock.csv.license | 2 - .../shadowserver/testdata/scan_telnet.csv | 3 - .../testdata/scan_telnet.csv.license | 2 - .../shadowserver/testdata/scan_tftp.csv | 4 - .../testdata/scan_tftp.csv.license | 2 - .../shadowserver/testdata/scan_ubiquiti.csv | 4 - .../testdata/scan_ubiquiti.csv.license | 2 - .../shadowserver/testdata/scan_vnc.csv | 3 - .../testdata/scan_vnc.csv.license | 2 - .../testdata/scan_ws_discovery.csv | 4 - .../testdata/scan_ws_discovery.csv.license | 2 - .../shadowserver/testdata/scan_xdmcp.csv | 4 - .../testdata/scan_xdmcp.csv.license | 2 - .../testdata/sinkhole_http_drone.csv.license | 2 - .../parsers/shadowserver/testdata/special.csv | 4 - .../shadowserver/testdata/special.csv.license | 2 - 291 files changed, 12939 deletions(-) delete mode 100644 intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_blocklist.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_device_id.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_malware_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_phish_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_special.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_testdata.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/special.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license diff --git a/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license b/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license deleted file mode 100644 index 043ed079f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py b/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py deleted file mode 100644 index 48509eea0..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/blocklist.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - 'feed.name': 'Block Listed IP Addresses', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-blocklist-test-geo.csv", -} -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.134", - "source.reverse_dns": "host.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.171", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.network": "198.123.245.0/24", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -},] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py b/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py deleted file mode 100644 index 53c5b247b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py +++ /dev/null @@ -1,88 +0,0 @@ -# SPDX-FileCopyrightText: 2017 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/compromised_website.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Compromised Website", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-compromised_website-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'ShadowServer Compromised Website', - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'classification.identifier': 'compromised-website', - 'extra.server': 'Microsoft-IIS/7.5', - 'extra.system': 'WINNT', - 'extra.detected_since': '2015-05-09 05:51:12', - 'protocol.application': 'http', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 64496, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '203.0.113.1', - 'source.port': 80, - 'source.url': 'http://example.com/header.php', - 'source.fqdn': 'example.com', - 'source.reverse_dns': 'example.com', - 'malware.name': 'hacked-webserver-stealrat-t1', - 'event_description.text': 'spam', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-01-16T00:43:48+00:00'}, - {'__type': 'Event', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'classification.identifier': 'compromised-website', - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'event_description.text': 'phishing', - 'feed.name': 'ShadowServer Compromised Website', - 'malware.name': 'phishing', - 'protocol.application': 'http', - 'source.asn': 64496, - 'source.fqdn': 'example.com', - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'GRAZ', - 'source.geolocation.region': 'STEIERMARK', - 'source.ip': '203.0.113.1', - 'source.port': 80, - 'source.url': 'http://example.com/', - 'time.source': '2018-04-09T15:43:41+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_device_id.py b/intelmq/tests/bots/parsers/shadowserver/test_device_id.py deleted file mode 100644 index e8954e03c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_device_id.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/device_id.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Device ID', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-device_id-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'extra.naics' : 517311, - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 2116, - 'source.geolocation.cc' : 'NO', - 'source.geolocation.city' : 'TROMVIK', - 'source.geolocation.region' : 'TROMS OG FINNMARK', - 'source.ip' : '88.84.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 27843, - 'source.geolocation.cc' : 'PE', - 'source.geolocation.city' : 'LIMA', - 'source.geolocation.region' : 'METROPOLITANA DE LIMA', - 'source.ip' : '170.231.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'extra.naics' : 517311, - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 4181, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MILWAUKEE', - 'source.geolocation.region' : 'WISCONSIN', - 'source.ip' : '96.60.0.0', - 'source.port' : 10443, - 'source.reverse_dns' : '96-60-66-218.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py deleted file mode 100644 index badc53a73..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py +++ /dev/null @@ -1,131 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_ddos_participant.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DDoS Participant', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_ddos_participant-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.1', - 'destination.port': 443, - 'destination.reverse_dns': 'node01.example.net', - 'extra.application': 'https', - 'extra.domain': 'www.example.com', - 'extra.http_method': 'GET', - 'extra.http_path': '/??=GovpfOoaWYlk', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 38055, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.2', - 'destination.port': 53, - 'destination.reverse_dns': 'node02.example.net', - 'extra.application': 'dns', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 53, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.3', - 'destination.port': 53, - 'destination.reverse_dns': 'node03.example.net', - 'extra.application': 'dns', - 'extra.device_model': 'Exchange', - 'extra.device_type': 'email', - 'extra.device_vendor': 'Microsoft', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 53, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py deleted file mode 100644 index 1d020f473..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_darknet.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Darknet", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_darknet.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.port': 23, - 'extra.source.naics': 518210, - 'extra.tag': 'mirai', - 'protocol.transport': 'tcp', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 9829, - 'source.geolocation.cc': 'IN', - 'source.geolocation.city': 'CHENGANNUR', - 'source.geolocation.region': 'KERALA', - 'source.ip': '61.3.1.2', - 'source.port': 4717, - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - {'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.port': 23, - 'protocol.transport': 'tcp', - 'extra.source.naics': 517311, - 'extra.tag': 'mirai', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 4766, - 'source.geolocation.cc': 'KR', - 'source.geolocation.city': 'PYEONGCHANG-EUP', - 'source.geolocation.region': 'GANGWON-DO', - 'source.ip': '211.218.3.4', - 'source.port': 4405, - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - }, - {'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.tag': 'mirai', - 'destination.port': 23, - 'protocol.transport': 'tcp', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 266915, - 'source.geolocation.cc': 'BR', - 'source.geolocation.city': 'VITORIA DA CONQUISTA', - 'source.geolocation.region': 'BAHIA', - 'source.ip': '45.225.5.6', - 'source.port': 59777, - 'source.reverse_dns': 'static-45-225-x-x.example.net', - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py deleted file mode 100644 index c62a610fa..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py +++ /dev/null @@ -1,148 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_ddos.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot DDoS Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_honeypot_ddos-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.1', - 'destination.port' : 88, - 'destination.reverse_dns' : 'node01.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '121.12.110.28/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 61234, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.2', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node02.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '180.97.183.94/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 61234, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.3', - 'destination.reverse_dns' : 'node03.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk7', - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '104.237.138.135/32', - 'extra.duration' : 10, - 'extra.family' : 'mirai', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 6379, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py deleted file mode 100644 index f379d1c88..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py +++ /dev/null @@ -1,150 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_ddos_target.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot DDoS Target Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_honeypot_ddos_target-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.1', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node01.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk0', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '115.238.198.85/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 61234, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.2', - 'destination.port' : 43437, - 'destination.reverse_dns' : 'node02.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk0', - 'extra.destination.sector' : 'Information', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '52.184.50.250/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 61234, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.3', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node03.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '211.99.102.216/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 61234, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py deleted file mode 100644 index bcf268ba7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_http_scan.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot-HTTP-Scan', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-08-01T12:00:00+00:00", - "extra.file_name": "2021-08-01-event4_honeypot_http_scan.csv", - } - -EVENTS = [{'__type': 'Event', - 'feed.name': 'Honeypot-HTTP-Scan', - 'classification.identifier': 'honeypot-http-scan', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'destination.asn': 5678, - 'destination.geolocation.cc': 'UK', - 'destination.geolocation.city': 'MAIDENHEAD', - 'destination.geolocation.region': 'WINDSOR AND MAIDENHEAD', - 'destination.ip': '109.87.65.43', - 'destination.port': 80, - 'extra.http_url': '/js/ueditor/wwwroot/way-board.cgi', - 'extra.destination.naics': 518210, - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'extra.public_source': 'CAPRICA-EU', - 'extra.request_raw': 'R0VUIC9qcy91ZWRpdG9yL3d3d3Jvb3Qvd2F5LWJvYXJkLmNnaSBIVFRQLzEuMHJuQWNjZXB0OiB0ZXh0L2h0bWwsYXBwbGljYXRpb24veGh0bWwreG1sLGFwcGxpY2F0aW9uL3htbDtxPTAuOSwqLyo7cT0wLjhybkFjY2VwdC1FbmNvZGluZzogZ3ppcCwgZGVmbGF0ZXJuQWNjZXB0LUxhbmd1YWdlOiBlbi1VUyxlbjtxPTAuNXJuQ29ubmVjdGlvbjogY2xvc2VybkRudDogMXJuSG9zdDogMTA5Ljg3LjY1LjQzcm5PcmlnaW46IGh0dHA6Ly8xMDkuODcuNjUuNDNyblJlZmVyZXI6IGh0dHA6Ly8xMDkuODcuNjUuNDMvcm5Vc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA2LjE7IFdPVzY0KSBBcHBsZVdlYktpdC81MzcuMzYgKEtIVE1MLCBsaWtlIEdlY2tvKSBDaHJvbWUvNTMuMC4yNzg1LjEwNCBTYWZhcmkvNTM3LjM2IENvcmUvMS41My4zMDg0LjQwMCBRUUJyb3dzZXIvOS42LjExMzQ2LjQwMA==', - 'extra.source.naics': 518210, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.version': '3.1.3-dev', - 'malware.name': 'http-scan', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 1234, - 'source.geolocation.cc': 'EE', - 'source.geolocation.city': 'TALLINN', - 'source.geolocation.region': 'HARJUMAA', - 'source.ip': '191.23.45.67', - 'source.port': 36455, - 'source.reverse_dns': '191-23-45-67-host.example.com', - 'time.observation': '2021-08-01T12:00:00+00:00', - 'time.source': '2021-08-01T00:24:08+00:00'}, - {'__type': 'Event', - 'feed.name': 'Honeypot-HTTP-Scan', - 'classification.identifier': 'honeypot-http-scan', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'destination.asn': 23456, - 'destination.geolocation.cc': 'UA', - 'destination.geolocation.city': 'KHARKIV', - 'destination.geolocation.region': "KHARKIVS'KA OBLAST'", - 'destination.ip': '82.41.20.10', - 'destination.port': 8080, - 'extra.http_url': '/', - 'extra.method': 'GET', - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'extra.public_source': 'CAPRICA-EU', - 'extra.request_raw': 'R0VUIC8gSFRUUC8xLjENCkhvc3Q6IDgyLjQxLjIwLjEwOjgwODANCkFjY2VwdDogdGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksKi8qO3E9MC44DQpBY2NlcHQtRW5jb2Rpbmc6IGRlZmxhdGUsIGd6aXAsIGlkZW50aXR5DQpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTO3E9MC42LGVuO3E9MC40DQpVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA1LjE7IHJ2OjkuMC4xKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzkuMC4xDQoNCg==', - 'extra.url_scheme': 'http', - 'extra.user_agent': 'Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1', - 'malware.name': 'http-scan', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 12345, - 'source.geolocation.cc': 'EE', - 'source.geolocation.city': 'TALLINN', - 'source.geolocation.region': 'HARJUMAA', - 'source.ip': '45.67.89.123', - 'source.port': 58610, - 'time.observation': '2021-08-01T12:00:00+00:00', - 'time.source': '2021-08-01T05:21:59+00:00'}, -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py deleted file mode 100644 index d21fb10c5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py +++ /dev/null @@ -1,182 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/event4_ip_spoofer.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "CAIDA", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-event4_ip_spoofer.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T00:42:59+00:00", - "source.ip": "98.191.250.0", - - "source.asn": 22898, - - "source.geolocation.cc": "US", - "source.geolocation.region": "OKLAHOMA", - "source.geolocation.city": "OKLAHOMA CITY", - "source.network": "98.191.250.0/24", - "source.reverse_dns": 'ip-98.191.250.0.atlinkservices.com', - "extra.routedspoof": "received", - "extra.session": '1112907', - "extra.nat": True, - "extra.public_source": "caida", - "extra.source.naics": 517311, - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T01:36:22+00:00", - "source.ip": "191.7.16.0", - - "source.asn": 262485, - - "source.geolocation.cc": "BR", - "source.geolocation.region": "RIO DE JANEIRO", - "source.geolocation.city": "NOVA IGUACU", - "source.network": "191.7.16.0/24", - "extra.routedspoof": "received", - "extra.session": '1112914', - "extra.nat": False, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T02:10:58+00:00", - "source.ip": "202.53.160.0", - - "source.asn": 23923, - - "source.geolocation.cc": "BD", - "source.geolocation.region": "DHAKA", - "source.geolocation.city": "DHAKA", - "source.network": "202.53.160.0/24", - "extra.routedspoof": "received", - "extra.session": '1112931', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T03:41:51+00:00", - "source.ip": "87.121.75.0", - - "source.asn": 134697, - - "source.geolocation.cc": "AU", - "source.geolocation.region": "QUEENSLAND", - "source.geolocation.city": "BRISBANE", - "source.network": "87.121.75.0/24", - "extra.routedspoof": "received", - "extra.session": '1112953', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T06:07:17+00:00", - "source.ip": "189.201.194.0", - - "source.asn": 262944, - - "source.network": "189.201.194.0/24", - "source.geolocation.cc": 'MX', - "source.geolocation.city": 'SALTILLO', - "source.geolocation.region": 'COAHUILA', - "source.reverse_dns": 'ip-189-201-194-0.slw.spectro.mx', - "extra.routedspoof": "received", - "extra.session": '1113015', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py deleted file mode 100644 index f008fd18e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py +++ /dev/null @@ -1,135 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_microsoft_sinkhole.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Microsoft Sinkhole", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_microsoft_sinkhole.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'HK', - 'destination.geolocation.city': 'HONG KONG', - 'destination.geolocation.region': 'HONG KONG', - 'destination.ip': '168.63.134.179', - 'destination.port': 16464, - 'extra.destination.naics': 334111, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 7303, - 'source.geolocation.cc': 'AR', - 'source.geolocation.city': 'CASEROS', - 'source.geolocation.region': 'BUENOS AIRES', - 'source.ip': '190.229.1.2', - 'source.port': 52955, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - {'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'IE', - 'destination.geolocation.city': 'DUBLIN', - 'destination.geolocation.region': 'DUBLIN', - 'destination.ip': '52.169.3.4', - 'destination.port': 16464, - 'extra.destination.naics': 334111, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 5769, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'LAVAL', - 'source.geolocation.region': 'QUEBEC', - 'source.ip': '96.20.3.4', - 'source.port': 16464, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - }, - {'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'HK', - 'destination.geolocation.city': 'HONG KONG', - 'destination.geolocation.region': 'HONG KONG', - 'destination.ip': '168.63.134.179', - 'destination.port': 16464, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 8151, - 'source.geolocation.cc': 'MX', - 'source.geolocation.city': 'MEXICO CITY', - 'source.geolocation.region': "CIUDAD DE MEXICO", - 'source.ip': '187.222.5.6', - 'source.port': 55049, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py deleted file mode 100644 index 2f8c3d8e2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py +++ /dev/null @@ -1,202 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_microsoft_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'HTTP Microsoft Sinkhole IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_microsoft_sinkhole_http.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.infection': 'necurs', - 'extra.tag': 'necurs', - 'protocol.application': 'http', - 'malware.name': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 8386, - 'source.geolocation.cc': 'TR', - 'source.geolocation.city': 'KEPEZ', - 'source.geolocation.region': 'ANTALYA', - 'source.ip': '31.206.1.2', - 'source.port': 49245, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'caphaw', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.fqdn': '3fo8jrthz3y.rgk.cc', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'REDMOND', - 'destination.geolocation.region': 'WASHINGTON', - 'destination.ip': '204.95.99.204', - 'destination.port': 443, - 'destination.url': 'http://3fo8jrthz3y.rgk.cc/index.php', - 'protocol.application': 'http', - 'extra.infection': 'caphaw', - 'extra.tag': 'caphaw', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.http_agent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.6103)', - 'extra.http_referer': 'null', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517312, - 'malware.name': 'caphaw', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 28573, - 'source.geolocation.cc': 'BR', - 'source.geolocation.city': 'SAO PAULO', - 'source.geolocation.region': 'SAO PAULO', - 'source.ip': '177.140.3.4', - 'source.port': 35919, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'protocol.application': 'http', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'malware.name': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn': 132199, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'MANDAUE', - 'source.geolocation.region': 'CEBU', - 'source.ip': '180.190.5.6', - 'source.port': 49264, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.ip': '40.121.206.97', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/news/stream.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'malware.name': 'necurs', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'source.asn': 37129, - 'source.geolocation.cc': 'KE', - 'source.geolocation.city': 'NAIROBI', - 'source.geolocation.region': 'NAIROBI CITY', - 'source.ip': '197.157.7.8', - 'source.port': 55307, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'necurs', - 'protocol.application': 'http', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'source.asn': 812, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'OTTAWA', - 'source.geolocation.region': 'ONTARIO', - 'source.ip': '174.114.9.10', - 'source.port': 59000, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py deleted file mode 100644 index 2bb8aa698..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Sinkhole", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'victorygate.b', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 28753, - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.ip': '178.162.1.2', - 'destination.port': 4455, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.public_source': 'eset', - 'feed.name': 'ShadowServer Sinkhole', - 'malware.name': 'victorygate.b', - 'extra.infection': 'victorygate.b', - 'protocol.transport': 'tcp', - 'source.asn': 12252, - 'source.geolocation.cc': 'PE', - 'source.geolocation.city': 'LIMA', - 'source.geolocation.region': 'METROPOLITANA DE LIMA', - 'source.ip': '190.113.1.2', - 'source.port': 17409, - 'time.source': '2021-03-04T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py deleted file mode 100644 index cf3bdb162..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_sinkhole_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "Sinkhole DNS", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole_dns-test-geo.csv", - } - -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'msexchange', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'calypso', - 'extra.dns_query' : 'YolkIsh.COM', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'extra.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'calypso', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 8220, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'FRANKFURT AM MAIN', - 'source.geolocation.region' : 'HESSEN', - 'source.ip' : '217.110.0.0', - 'source.port' : 29614, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'rat', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'orcus', - 'extra.dns_query' : 'verble.rocks', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'orcus', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 40934, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '209.66.0.0', - 'source.port' : 46189, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'msexchange', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'calypso', - 'extra.dns_query' : 'RAwFuNS.COM', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'extra.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'calypso', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 8220, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'FRANKFURT AM MAIN', - 'source.geolocation.region' : 'HESSEN', - 'source.ip' : '217.110.0.0', - 'source.port' : 3590, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -} -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py deleted file mode 100644 index 60cd6b6ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py +++ /dev/null @@ -1,189 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'HTTP Sinkhole IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole_http.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.1.2', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 134707, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'DEL PILAR', - 'source.geolocation.region': 'NUEVA ECIJA', - 'source.ip': '103.196.1.2', - 'source.port': 60902, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.3.4', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 8708, - 'source.geolocation.cc': 'RO', - 'source.geolocation.city': 'CONSTANTA', - 'source.geolocation.region': 'CONSTANTA', - 'source.ip': '5.14.3.4', - 'source.port': 55002, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'disorderstatus.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.5.6', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.source.naics': 517311, - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn': 9299, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'CEBU', - 'source.geolocation.region': 'CEBU', - 'source.ip': '49.145.5.6', - 'source.port': 31350, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.ip': '184.105.7.8', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.source.naics': 517311, - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'source.asn': 8048, - 'source.geolocation.cc': 'VE', - 'source.geolocation.city': 'VALENCIA', - 'source.geolocation.region': 'CARABOBO', - 'source.ip': '200.44.7.8', - 'source.port': 28063, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.9.10', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'source.asn': 17072, - 'source.geolocation.cc': 'MX', - 'source.geolocation.city': 'JUAREZ', - 'source.geolocation.region': 'CHIHUAHUA', - 'source.ip': '187.189.9.10', - 'source.port': 45335, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py deleted file mode 100644 index b1ccacd31..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py +++ /dev/null @@ -1,213 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole_http_referer.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-03-05T00:00:00+00:00", - "extra.file_name": "2021-03-04-event4_sinkhole_http_referer.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'kovter', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': '12106.mobapptrack.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.ip': '85.17.31.82', - 'destination.port': 80, - 'destination.url': 'http://12106.mobapptrack.com/favicon.ico', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.event_id': '1614816002', - 'malware.name': 'kovter', - 'extra.http_referer': 'http://12106.mobapptrack.com/click/redirect?feed_id=12106&sub_id=7&q=8A5491983C8FBE7743E2D2C36E45EBC4-18307118D2626C9BD756B3F09D14BB910E381EE4', - 'extra.http_referer_asn': 28753, - 'extra.http_referer_city': 'FRANKFURT AM MAIN', - 'extra.http_referer_geo': 'DE', - 'extra.http_referer_hostname': '12106.mobapptrack.com', - 'extra.http_referer_ip': '178.162.203.211', - 'extra.http_referer_naics': 518210, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'HESSEN', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:02+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 28753, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/animalally.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '178.162.1.2', - 'extra.event_id': '1614816011', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://x.noizm.com/jump.php?u=http://freescanonline.com/animalally.com', - 'extra.http_referer_asn': 9370, - 'extra.http_referer_city': 'OSAKA', - 'extra.http_referer_geo': 'JP', - 'extra.http_referer_hostname': 'x.noizm.com', - 'extra.http_referer_naics': 518210, - 'extra.http_referer_port': 80, - 'extra.http_referer_ip': '59.106.1.2', - 'extra.http_referer_region': 'OSAKA', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.source': '2021-03-04T00:00:11+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'kovter', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 28753, - 'destination.fqdn': 'rxrtb.bid', - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.port': 80, - 'destination.url': 'http://rxrtb.bid/getjs?r=0.6393021999392658', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '178.162.1.2', - 'extra.event_id': '1614816012', - 'malware.name': 'kovter', - 'extra.http_referer': 'http://x.blogspot.com/', - 'extra.http_referer_ip': '142.250.3.4', - 'extra.http_referer_asn': 15169, - 'extra.http_referer_city': 'MOUNTAIN VIEW', - 'extra.http_referer_geo': 'US', - 'extra.http_referer_hostname': 'x.blogspot.com', - 'extra.http_referer_naics': 519130, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'CALIFORNIA', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'time.source': '2021-03-04T00:00:12+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.ip': '5.79.71.225', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/personalationmall.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'extra.event_id': '1614816013', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://www.example.com/teams/default.asp?u=EKL&t=c&s=lacrosse&p=remote&url=http://freescanonline.com/personalationmall.com', - 'extra.http_referer_asn': 14618, - 'extra.http_referer_city': 'ASHBURN', - 'extra.http_referer_geo': 'US', - 'extra.http_referer_hostname': 'www.example.com', - 'extra.http_referer_ip': '34.232.5.6', - 'extra.http_referer_naics': 454110, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'VIRGINIA', - 'extra.http_referer_sector': 'Retail Trade', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'time.source': '2021-03-04T00:00:13+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/raftcomply.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '5.79.1.2', - 'extra.event_id': '1614816086', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://x.communes.jp/?url=http://freescanonline.com/raftcomply.com', - 'extra.http_referer_asn': 2516, - 'extra.http_referer_city': 'SAPPORO', - 'extra.http_referer_geo': 'JP', - 'extra.http_referer_hostname': 'x.communes.jp', - 'extra.http_referer_ip': '210.172.7.8', - 'extra.http_referer_naics': 517312, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'HOKKAIDO', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'time.source': '2021-03-04T00:01:26+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py deleted file mode 100644 index d6ff35dc1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event6_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "Sinkhole-Events-HTTP IPv6", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event6_sinkhole_http-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::fe', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)', - 'extra.infection' : 'boaxxe', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 7713, - 'source.geolocation.cc' : 'ID', - 'source.geolocation.city' : 'JAKARTA', - 'source.geolocation.region' : 'JAKARTA RAYA', - 'source.ip' : '2001:448a:1082:4d9b:7491:bf9e:3d5f:a634', - 'source.port' : 49431, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T09:14:19+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::ef', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)', - 'extra.infection' : 'boaxxe', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 7713, - 'source.geolocation.cc' : 'ID', - 'source.geolocation.city' : 'JAKARTA', - 'source.geolocation.region' : 'JAKARTA RAYA', - 'source.ip' : '2001:448a:1082:4d9b:7491:bf9e:3d5f:a634', - 'source.port' : 49460, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T09:15:10+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::fe', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/WMoUNCvuKGzdqSCeQcadP1%2f0B%2f3bzpOmyKBU85Z25HVOhvDQUPFl%2fk8uOcLewS%2b1BsuHXalRAOIgGOYYs2igj6UX8FkdCAmDewWPvfDhPD45nwd2tx1lLf2IoIfuOtIpGR6bN5Q6hGpSBgfERqCa0ImHcwfcZ2EdO%2fWvg7R8H6SLcTiuUC0I4pzvlWt1CRLgLdIEU1hZ0nnFHIHhchb6D7ITEgBQ2chQDxy5TJMrGjm4Dac6dKl%2ft5uYhRhSjAHkLLtgrJjsqVtVbelTAkt5kdcqLlO09m1SH%2fvtAb%2fOvR2DbhBss7%2f64DG7g6cAnghNA6JrFn1uW7sw%2bnKH8koKQwzUjdSsbrQAvmg4r0KDDW8Diq64gfDzxFWkzCLOYifc%2fwlinXPCl7aJiNCoieDC1U98RNQg%2f5td4SZmJnDQ2%2f96CPbFeSpCez5WD1rCjrxLj1h2cqzIgkydEWACceWP9ztxc4QaObzEcgOGxbRckWC7H2aaLeT8jaYEYdKi1pwEKChSL3YdEt4ZIb2IFrWwzNaXEpQzFXf07f902OEdI9vVA1ZdEOBPG6rAIkzMdebfprfVyhKEWtrCd3Skg3COUFtRQks5jzG1nv4sVGijTfSgyn6xE9Taka668Nycik6nmHy8Huj3oC01j3tee%2f1Z3eI6tV7lgM5d3uFJ84slRGHUCwMfVozOGmZRwNo%2fz%2bA', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko', - 'extra.infection' : 'boaxxe', - 'extra.source.naics' : 517311, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 11427, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'GARLAND', - 'source.geolocation.region' : 'TEXAS', - 'source.ip' : '2603:8080:b20a:dc00:f06e:8304:71f6:27e2', - 'source.port' : 62932, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T14:15:10+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py b/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py deleted file mode 100644 index c376a73fb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_honeypot_brute_force.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot-Brute-Force-Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_brute_force.csv" - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'ssh', - 'classification.taxonomy': 'intrusion-attempts', - 'classification.type': 'brute-force', - 'extra.client_version': "b'SSH-2.0-Go'", - 'destination.asn': 26832, - 'destination.geolocation.cc': 'CA', - 'destination.geolocation.city': 'MONTREAL', - 'destination.geolocation.region': 'QUEBEC', - 'destination.ip': '162.250.1.2', - 'destination.port': 22, - 'extra.application': 'ssh', - 'extra.end_time': '2021-03-27T00:00:01.710968+00:00', - 'extra.public_source': 'CAPRICA-EU', - 'extra.start_time': '2021-03-27T00:00:00.521730+00:00', - 'malware.name': 'ssh-brute-force', - 'feed.name': 'Honeypot-Brute-Force-Events', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 209588, - 'source.geolocation.cc': 'NL', - 'source.geolocation.city': 'AMSTERDAM', - 'source.geolocation.region': 'NOORD-HOLLAND', - 'source.ip': '141.98.1.2', - 'source.port': 30123, - 'time.source': '2021-03-27T00:00:00+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py b/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py deleted file mode 100644 index e95e59dcb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_honeypot_ddos_amp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Amplification DDoS Victim', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_ddos_amp.csv" - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Amplification DDoS Victim', - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation': '2019-01-01T00:00:00+00:00', - 'time.source': '2021-03-28T00:00:02+00:00', - 'source.ip': '107.141.1.2', - 'destination.port': 389, - 'source.reverse_dns': '192-0-2-10.example.net', - 'source.asn': 7018, - 'source.geolocation.cc': 'US', - 'source.geolocation.region': 'VISALIA', - 'source.geolocation.city': 'VISALIA', - 'source.geolocation.region': 'CALIFORNIA', - 'extra.end_time': '2021-03-28T00:20:22+00:00', - 'extra.public_source': 'CISPA', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'ddos-amplification', - 'source.reverse_dns': '107-141-x-x.lightspeed.frsnca.sbcglobal.net', - }, - {'__type': 'Event', - 'feed.name': 'Amplification DDoS Victim', - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.observation': '2019-01-01T00:00:00+00:00', - 'time.source': '2021-03-28T00:00:02+00:00', - 'source.ip': '74.59.3.4', - 'destination.port': 389, - 'source.reverse_dns': 'modemcablex-x-59-74.mc.videotron.ca', - 'source.asn': 5769, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'CHICOUTIMI', - 'source.geolocation.region': 'QUEBEC', - 'extra.end_time': '2021-03-28T00:13:50+00:00', - 'extra.public_source': 'CISPA', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'ddos-amplification', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py b/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py deleted file mode 100644 index b19b200b5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/malware_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Malware URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-malware_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'source.url' : 'http://41.86.0.0:50008/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.hash.sha256' : '12013662c71da69de977c04cd7021f13a70cf7bed4ca6c82acbc100464d4b0ef', - 'malware.name' : 'cve-2016-10372', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37203, - 'source.geolocation.cc' : 'LR', - 'source.geolocation.city' : 'MONROVIA', - 'source.geolocation.region' : 'MONTSERRADO', - 'source.ip' : '41.86.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:02:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'extra.source.naics' : 517311, - 'source.url' : 'http://42.225.0.0:38173/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.name' : 'cve-2018-10562', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 4837, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'ZHUMADIAN', - 'source.geolocation.region' : 'HENAN SHENG', - 'source.ip' : '42.225.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:03:14+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'extra.source.naics' : 517311, - 'source.url' : 'http://211.52.0.0:53029/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.name' : 'cve-2018-10562', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 4766, - 'source.geolocation.cc' : 'KR', - 'source.geolocation.city' : 'SAGOK-MYEON', - 'source.geolocation.region' : 'CHUNGCHEONGNAM-DO', - 'source.ip' : '211.52.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:10:26+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py b/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py deleted file mode 100644 index 0783372f9..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/phish_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Phish URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-phish_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'priceless-pare.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.naics' : 518210, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'source.url' : 'https://priceless-pare.example.net/Postal-/acec6/', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'BUFFALO', - 'source.geolocation.region' : 'NEW YORK', - 'source.ip' : '172.245.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'mailyahooattt.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.sector' : 'Professional, Scientific, and Technical Services', - 'source.url' : 'https://mailyahooattt.example.net/', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'SAN FRANCISCO', - 'source.geolocation.region' : 'CALIFORNIA', - 'source.ip' : '199.34.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'www.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.naics' : 519130, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'source.url' : 'https://www.example.net/viewer/vbid-730ec2b1-omsttuer', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'DRAPER', - 'source.geolocation.region' : 'UTAH', - 'source.ip' : '216.58.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py b/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py deleted file mode 100644 index e9f11a47c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py +++ /dev/null @@ -1,130 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/population_http_proxy.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible HTTP Proxy', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-population_http_proxy-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 3741, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"Squid proxy-caching web ' - 'server\\"\\""', - 'extra.server': 'squid/4.10', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3128, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 3833, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"00:23:24:43:1c:34\\"\\""', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3128, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 179, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"Proxy\\"\\""', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3128, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py deleted file mode 100644 index c5da82346..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py +++ /dev/null @@ -1,99 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_conn.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox Connections', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_conn-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'time.windows.com', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'b575ce6dcce6502a8431db5610135c25', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 8075, - 'source.geolocation.cc' : 'US', - 'source.ip' : '40.119.6.228', - 'source.port' : 123, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -}, - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'c0d947f9a8685b0d9f3efdba966389c2', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 3356, - 'source.geolocation.cc' : 'US', - 'source.ip' : '8.252.70.126', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'c0d947f9a8685b0d9f3efdba966389c2', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 8075, - 'source.geolocation.cc' : 'US', - 'source.ip' : '52.109.8.22', - 'source.port' : 443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py deleted file mode 100644 index 70cf1eee5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py +++ /dev/null @@ -1,95 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox DNS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_dns-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'time.windows.com', - 'extra.response' : '40.119.6.228', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : 'b575ce6dcce6502a8431db5610135c25', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:02+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'time.windows.com', - 'extra.response' : '40.119.6.228', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : '807679198a39c80d3ca07e60fd51b581', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:08+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'client-office365-tas.msedge.net', - 'extra.response' : '13.107.5.88', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : 'd97e973b9bf073bd3a217425259cea26', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:20+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py deleted file mode 100644 index 91b0154b8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py +++ /dev/null @@ -1,104 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'www.msftncsi.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://www.msftncsi.com/ncsi.txt', - 'extra.user_agent' : 'Microsoft NCSI', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : '37514b54e679a5313334e830ad780ec7', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'US', - 'source.ip' : '23.196.47.89', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'www.download.windowsupdate.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://www.download.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab', - 'extra.user_agent' : 'Microsoft-CryptoAPI/6.1', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : '37514b54e679a5313334e830ad780ec7', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 15133, - 'source.geolocation.cc' : 'US', - 'source.ip' : '72.21.81.240', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:28+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'crl.microsoft.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://crl.microsoft.com/pki/crl/products/MicTimStaPCA_2010-07-01.crl', - 'extra.user_agent' : 'Microsoft-CryptoAPI/6.1', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : 'e97ea2820c0d79f3f3ca241d4dcd1060', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'US', - 'source.ip' : '23.56.4.57', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:08:24+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py deleted file mode 100644 index 6bc6e6146..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py +++ /dev/null @@ -1,98 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_adb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible ADB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_adb-test-test.csv", - - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible ADB', - 'time.observation': '2018-07-30T00:00:00+00:00', - 'time.source': '2018-07-26T02:07:16+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-adb', - 'protocol.application': 'adb', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 3462, - 'source.geolocation.cc': 'TW', - 'source.geolocation.city': 'TAOYUAN CITY', - 'source.geolocation.region': 'TAOYUAN COUNTY', - 'source.ip': '36.239.124.210', - 'source.port': 5555, - 'extra.name': 'hlteuc', - 'extra.model': 'SAMSUNG-SM-N900A', - 'extra.device': 'hlteatt', - 'extra.tag': 'adb', - 'extra.naics': 518210, - 'extra.sic': 737415, - 'source.reverse_dns': '36-239-124-210.dynamic-ip.hinet.net', - }, - {'__type': 'Event', - 'feed.name': 'Accessible ADB', - 'time.observation': '2018-07-30T00:00:00+00:00', - 'time.source': '2018-07-26T02:07:16+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-adb', - 'protocol.application': 'adb', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 3462, - 'source.geolocation.cc': 'TW', - 'source.geolocation.city': 'TAIPEI', - 'source.geolocation.region': 'TAIPEI CITY', - 'source.ip': '36.236.108.107', - 'source.port': 5555, - 'extra.name': 'marlin', - 'extra.model': 'Pixel XL', - 'extra.device': 'marlin', - 'extra.features': 'cmd,shell_v2', - 'extra.naics': 518210, - 'extra.sic': 737415, - 'extra.tag': 'adb', - 'source.reverse_dns': '36-236-108-107.dynamic-ip.hinet.net', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py deleted file mode 100644 index cc30b1e4c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_afp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible AFP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_afp-test-test.csv", - - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible AFP', - "classification.identifier": "accessible-afp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.afp_versions": "AFP3.3,AFP3.2,AFP3.1", - "extra.flags": "SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient", - "extra.machine_type": "TimeCapsule8,119", - "extra.naics": 517311, - "extra.network_address": "198.33.24.165:548,10.0.1.1:548,fe80:0008:0000:0000:6e70:9fff:fed4::548,fe80:0009:0000:0000:6e70:9fff:fed4::548,179.24.24.165 (DNS address),", - "extra.server_name": "airport-time-capsule-de-jack", - "extra.signature": "4338364e37364442463948350069672d", - "extra.tag": "afp", - "extra.uams": "DHCAST128,DHX2,SRP,Recon1", - "extra.utf8_servername": "AirPort Time Capsule de jack", - "protocol.application": "afp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 6057, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.13.34.22", - "source.port": 548, - "source.reverse_dns": "host.local", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T05:05:53+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Accessible AFP', - "classification.identifier": "accessible-afp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.afp_versions": "AFP3.3,AFP3.2,AFP3.1", - "extra.flags": "SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient", - "extra.machine_type": "TimeCapsule8,119", - "extra.naics": 517311, - "extra.network_address": "0.0.0.1:548,10.0.1.1:548,198.33.42.1:548,fe80:000b:0000:0000:dea4:caff:feba::548,fe80:000c:0000:0000:dea4:caff:feba::548,fe80:000d:0000:0000:4c7d:ffff:fec7::548,0.0.0.1 (DNS address),", - "extra.server_name": "time-capsule-del-jack", - "extra.signature": "433836544b303147463948360069672d", - "extra.tag": "afp", - "extra.uams": "DHCAST128,DHX2,SRP,Recon1", - "extra.utf8_servername": "Time Capsule del Jack", - "protocol.application": "afp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 6057, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.40.27.212", - "source.port": 548, - "source.reverse_dns": "host.local", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T05:05:56+00:00" - },] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py deleted file mode 100644 index df707f30b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py +++ /dev/null @@ -1,144 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_amqp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible AMQP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_amqp-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@iZuf63m0nnq9bwf7lhjxrkZ', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'PLAIN AMQPLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.3.5', - 'extra.naics' : 518210, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37963, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'SHANGHAI', - 'source.geolocation.region' : 'SHANGHAI SHI', - 'source.ip' : '47.103.0.0', - 'source.port' : 5672, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@mtk-breizh', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'AMQPLAIN PLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP 24.0.3', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.8.19', - 'extra.naics' : 518210, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 16276, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'SAARBRUCKEN', - 'source.geolocation.region' : 'SAARLAND', - 'source.ip' : '141.95.0.0', - 'source.port' : 5672, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@1397a0e9629b', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'PLAIN AMQPLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP 24.2', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.9.11', - 'extra.naics' : 454110, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 14618, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '54.234.0.0', - 'source.port' : 5672, - 'source.reverse_dns' : 'ec2-54.234.0.0.compute-1.amazonaws.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py deleted file mode 100644 index 4d8420c3b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py +++ /dev/null @@ -1,111 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Tomas Bellus -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ard.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible ARD', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-07-20T00:00:00+00:00", - "extra.file_name": "2020-01-01-scan_ard-test-test.csv", - - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': 'Macmini (radio)', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3283, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': 'biuro-rip-org-pl', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3283, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': '127.0.0.1', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3283, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py deleted file mode 100644 index 3b72baa8d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py +++ /dev/null @@ -1,110 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_chargen.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Chargen', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_chargen-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 19, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 19, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.sector': 'Government', - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 19, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py deleted file mode 100644 index 46c963a79..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py +++ /dev/null @@ -1,82 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_cisco_smart_install.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Cisco Smart Install', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_cisco_smart_install-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible Cisco Smart Install', - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.type': 'vulnerable-system', - 'classification.taxonomy': 'vulnerable', - 'protocol.application': 'cisco-smart-install', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 8559, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '198.51.100.103', - 'source.port': 4786, - 'extra.tag': 'cisco-smart-install', - 'source.reverse_dns': '198-51-100-103.example.net', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-11-18T08:42:45+00:00'}, - {'__type': 'Event', - 'feed.name': 'Accessible Cisco Smart Install', - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.type': 'vulnerable-system', - 'classification.taxonomy': 'vulnerable', - 'protocol.application': 'cisco-smart-install', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 35609, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '198.51.100.218', - 'source.port': 4786, - 'extra.tag': 'cisco-smart-install', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-11-18T08:47:54+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py deleted file mode 100644 index 773fc04d5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_coap.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-CoAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-06-29T00:00:00+00:00", - "extra.file_name": "2020-06-28-scan_coap-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 2.05, - 'extra.response': ',,', - 'extra.response_size': 43, - 'extra.tag': 'coap', - 'extra.version': '2', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5683, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 5.38, - 'extra.response': ',,,,,,,,,', - 'extra.response_size': 113, - 'extra.tag': 'coap', - 'extra.version': '2', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5683, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 113.5, - 'extra.response': '`EsjAy************************************************************|CoAP ' - 'RFC 7252 ' - '|************************************************************|This ' - 'server is using the Eclipse Californium (Cf) CoAP ' - 'framework|published under EPL+EDL: ' - 'http://www.eclipse.org/californium/||(c) 2014, 2015, 2016 ' - 'Institute for Pervasive Computing, ETH Zurich and ' - 'others|************************************************************', - 'extra.response_size': 454, - 'extra.tag': 'coap', - 'extra.version': '1', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5683, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py deleted file mode 100644 index 1bf6f321c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py +++ /dev/null @@ -1,128 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_couchdb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible CouchDB Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_couchdb-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '1.6.1', - 'extra.server_version' : 'CouchDB/1.6.1 (Erlang OTP/18)', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'Ubuntu 16.04', - 'extra.visible_databases' : '_replicator;_users;test;shops;god', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5984, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '3.2.1', - 'extra.features' : 'access-ready,partitioned,pluggable-storage-engines,reshard,scheduler', - 'extra.git_sha' : '244d428af', - 'extra.server_version' : 'CouchDB/3.2.1 (Erlang OTP/23)', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'The Apache Software Foundation', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5984, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '3.2.1', - 'extra.features' : 'access-ready,partitioned,pluggable-storage-engines,reshard,scheduler', - 'extra.git_sha' : '244d428af', - 'extra.server_version' : 'CouchDB/3.2.1 (Erlang OTP/20)', - 'extra.source.sector' : 'Retail Trade', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'The Apache Software Foundation', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5984, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py deleted file mode 100644 index b508b6450..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_cwmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible CWMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_cwmp-test-test.csv", - - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible CWMP', - "classification.identifier": "open-cwmp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.connection": "keep-alive", - "extra.content_length": 5678, - "extra.content_type": "text/html", - "extra.date": "Wed, 04 Sep 2019 07:42:37 GMT", - "extra.http": "HTTP/1.1", - "extra.http_code": 200, - "extra.http_reason": "OK", - "extra.naics": 517311, - "extra.server": "DNVRS-Webs", - "extra.tag": "cwmp", - "protocol.application": "cwmp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.142", - "source.port": 30005, - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T10:44:55+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Accessible CWMP', - "classification.identifier": "open-cwmp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.content_type": "text/html", - "extra.http": "HTTP/1.1", - "extra.http_code": 404, - "extra.http_reason": "Not Found", - "extra.naics": 517311, - "extra.server": "RomPager/4.07 UPnP/1.0", - "extra.tag": "cwmp", - "extra.transfer_encoding": "chunked", - "protocol.application": "cwmp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.162", - "source.port": 5678, - "source.reverse_dns": "localhost.localdomain", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T11:06:50+00:00" - },] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py deleted file mode 100644 index 423ebe8c5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_db2.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Open-DB2-Discovery-Service", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_db2-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 14.9, - 'extra.db2_hostname': 'NOWAK_SERWER', - 'extra.servername': 'node01.example.com', - 'extra.size': 298, - 'extra.tag': 'db2', - 'feed.name': 'ShadowServer Open-DB2-Discovery-Service', - 'protocol.application': 'db2', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 523, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 14.9, - 'extra.db2_hostname': 'SPZOZ-DZIEWIN', - 'extra.servername': 'node02.example.com', - 'extra.size': 298, - 'extra.tag': 'db2', - 'feed.name': 'ShadowServer Open-DB2-Discovery-Service', - 'protocol.application': 'db2', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 523, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py deleted file mode 100644 index 9038a79ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ddos_middlebox.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DDoS Middlebox', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_ddos_middlebox-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source_port' : '49002', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 80, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.source_port' : '41200', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 80, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source_port' : '47492', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 80, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py deleted file mode 100644 index 3492f82ce..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DNS Open Resolvers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_dns-test-test.csv", - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'DNS Open Resolvers', - "classification.identifier": "dns-open-resolver", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.66", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.179", - "source.port": 53, - "source.reverse_dns": "198-51-100-189.example.net", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:34+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'DNS Open Resolvers', - "classification.identifier": "dns-open-resolver", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.51", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.8", - "source.port": 53, - "source.reverse_dns": "198-51-100-111.example.net", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:36+00:00" -},] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py deleted file mode 100644 index 31d0e4417..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py +++ /dev/null @@ -1,159 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_docker.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Docker Service', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_docker-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.37', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2018-05-09T22:18:36.000000000+00:00', - 'extra.content_type' : 'application/json; charset=UTF-8', - 'extra.date' : 'Fri, 06 May 2022 14:06:30 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : 'f150324', - 'extra.go_version' : 'go1.9.5', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-514.26.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.server' : 'Docker/18.05.0-ce (linux)', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'docker', - 'extra.version' : '18.05.0-ce', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 2375, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.26', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2022-03-02T15:25:43.414574467+00:00', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Fri, 06 May 2022 14:08:07 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : '7d71120/1.13.1', - 'extra.go_version' : 'go1.10.3', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-693.2.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.pkg_version' : 'docker-1.13.1-209.git7d71120.el7.centos.x86_64', - 'extra.server' : 'Docker/1.13.1 (linux)', - 'extra.tag' : 'docker', - 'extra.version' : '1.13.1', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 2375, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.37', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2018-05-09T22:18:36.000000000+00:00', - 'extra.content_type' : 'application/json; charset=UTF-8', - 'extra.date' : 'Fri, 06 May 2022 14:08:06 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : 'f150324', - 'extra.go_version' : 'go1.9.5', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-514.26.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.server' : 'Docker/18.05.0-ce (linux)', - 'extra.tag' : 'docker', - 'extra.version' : '18.05.0-ce', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 2375, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py deleted file mode 100644 index 01e68db94..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py +++ /dev/null @@ -1,178 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dvr_dhcpdiscover.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible DVR DHCPDiscover', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_dvr_dhcpdiscover-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 794.0, - 'extra.device_model': 'BCS-TIP3401IR-E-V', - 'extra.device_serial': '6J0E022PAG35073', - 'extra.device_type': 'IPC', - 'extra.device_vendor': 'General', - 'extra.device_version': '2.800.106F004.0.R', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.1', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::1', - 'extra.ipv6_dhcp_enable': False, - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3ac4:e8ff:fe03:b3e2/64', - 'extra.mac_address': '38:c4:e8:03:b3:e2', - 'extra.machine_name': '6J0E022PAG35073', - 'extra.manufacturer': 'General', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 0, - 'extra.response_size': 794, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.video_input_channels': 1, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 37810, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 761.0, - 'extra.device_model': 'HCVR', - 'extra.device_serial': '2K0488CPAGS0ND6', - 'extra.device_type': 'HCVR', - 'extra.device_vendor': 'Private', - 'extra.device_version': '3.210.1.4', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.2', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::2', - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3eef:8cff:fe18:a507/64', - 'extra.mac_address': '3c:ef:8c:18:a5:07', - 'extra.machine_name': 'HCVR', - 'extra.manufacturer': 'Private', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 9, - 'extra.response_size': 761, - 'extra.video_input_channels': 3, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 37810, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 711.0, - 'extra.device_model': 'BCS-XVR0401-IV', - 'extra.device_serial': '5L034FAPAZA0E30', - 'extra.device_type': 'HCVR', - 'extra.device_vendor': 'General', - 'extra.device_version': '4.000.0000002.11', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.3', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::3', - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3ac4:e8ff:fe02:74da/64', - 'extra.mac_address': '38:c4:e8:02:74:da', - 'extra.machine_name': 'XVR', - 'extra.manufacturer': 'General', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 0, - 'extra.response_size': 711, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.video_input_channels': 4, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 37810, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py deleted file mode 100644 index 4e12a1b07..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py +++ /dev/null @@ -1,126 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_elasticsearch.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Elasticsearch', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_elasticsearch-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': '90f439ff60a3c0f497f91663701e64ccd01edbb4', - 'extra.build_snapshot': False, - 'extra.build_timestamp': '2016-07-27T10:36:52Z', - 'extra.cluster_name': 'elasticsearch', - 'extra.lucene_version': '5.5.0', - 'extra.name': 'Red Skull', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '2.3.5', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 9200, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': 'bee86328705acaa9a6daede7140defd4d9ec56bd', - 'extra.build_snapshot': False, - 'extra.cluster_name': 'docker-cluster', - 'extra.lucene_version': '8.11.1', - 'extra.name': 'allinonepod', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '7.17.0', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 9200, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': '79d65f6e357953a5b3cbcc5e2c7c21073d89aa29', - 'extra.build_snapshot': False, - 'extra.cluster_name': 'docker-cluster', - 'extra.lucene_version': '8.9.0', - 'extra.name': 'f547c2952610', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '7.15.0', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 9200, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py deleted file mode 100644 index aeeffa3c2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py +++ /dev/null @@ -1,149 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/scan_exchange.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "Shadowserver CVE-2021-26855", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-scan_exchange.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:30+00:00", - "source.ip": "12.237.1.2", - "source.port": 443, - "source.asn": 7018, - "source.geolocation.cc": "US", - "source.geolocation.region": "CALIFORNIA", - "source.geolocation.city": "TURLOCK", - "source.reverse_dns": 'afs-exch-cas2.xxx.com', - "extra.version": '15.2.721', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 517311, - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "extra.servername": "AFS-EXCH2019", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:37+00:00", - "source.ip": "98.153.3.4", - "source.port": 443, - "source.asn": 20001, - "source.geolocation.cc": "US", - "source.geolocation.region": "CALIFORNIA", - "source.geolocation.city": "LOS ANGELES", - "source.reverse_dns": 'rrcs-98-153-x-x.west.biz.rr.com', - "extra.version": '15.0.847', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 517311, - "extra.tag": "exchange;webshell", - "classification.taxonomy": "intrusions", - "classification.type": "system-compromise", - "classification.identifier": "exchange-server-webshell", - "extra.servername": "SSAMAIL", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "206.210.5.6", - "source.port": 443, - "source.asn": 17054, - "source.geolocation.cc": "US", - "source.geolocation.region": "PENNSYLVANIA", - "source.geolocation.city": "PITTSBURGH", - "source.reverse_dns": 'webmail.xxx.com', - "extra.source.naics": 518210, - "extra.version": '15.0.1178', - "extra.servername": "OMNYXEXCH02", - "extra.tag": "exchange;webshell", - "classification.taxonomy": "intrusions", - "classification.type": "system-compromise", - "classification.identifier": "exchange-server-webshell", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "12.33.7.8", - "source.port": 443, - "source.asn": 7018, - "source.geolocation.cc": "US", - "source.geolocation.region": "ARKANSAS", - "source.geolocation.city": "LITTLE ROCK", - "source.reverse_dns": 'mail.xxx.org', - "extra.version": '15.1.2176', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 921120, - "extra.servername": "MHASVR02", - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "41.204.9.10", - "source.port": 443, - "source.asn": 21042, - "source.geolocation.cc": 'MG', - "source.geolocation.city": 'ANTANANARIVO', - "source.geolocation.region": 'ANTANANARIVO', - "source.reverse_dns": 'mail.xxx.mg', - "extra.servername": "SABMHQE0232", - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py deleted file mode 100644 index 33daefd75..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ftp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible FTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ftp-test-test.csv", - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible FTP', - 'time.observation': '2019-03-25T00:00:00+00:00', - 'time.source': '2019-03-06T06:37:00+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-ftp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.ip': '61.126.3.70', - 'source.port': 21, - 'protocol.transport': 'tcp', - 'protocol.application': 'ftp', - 'source.reverse_dns': 'arcus-net.co.jp', - 'extra.tag': 'ftp', - 'source.asn': 4713, - 'source.geolocation.cc': 'JP', - 'source.geolocation.region': 'TOKYO', - 'source.geolocation.city': 'TOKYO', - 'extra.naics': 517311, - 'extra.sic': 737401, - 'extra.banner': '220 FTP Server ready.|', - 'extra.handshake': 'TLSv1.2', - 'extra.cipher_suite': 'TLS_RSA_WITH_AES_128_CBC_SHA', - 'extra.cert_length': 2048, - 'extra.subject_common_name': '*.bizmw.com', - 'extra.issuer_common_name': 'GlobalSign Organization Validation CA - SHA256 - G2', - 'extra.cert_issue_date': 'Jan 14 08:04:50 2015 GMT', - 'extra.cert_expiration_date': 'Jan 14 08:04:50 2020 GMT', - 'extra.sha1_fingerprint': 'D9:98:3F:2E:F9:D1:BE:9A:10:1E:DE:51:2C:C1:DF:01:18:0A:20:65', - 'extra.cert_serial_number': '1121DC7421AB7924C3B1D396AEA3707E9E29', - 'extra.ssl_version': 2, - 'extra.signature_algorithm': 'sha256WithRSAEncryption', - 'extra.key_algorithm': 'rsaEncryption', - 'extra.subject_organization_name': 'NTT Communications Corporation', - 'extra.subject_country': 'JP', - 'extra.subject_state_or_province_name': 'Tokyo', - 'extra.subject_locality_name': 'Minato-ku', - 'extra.issuer_organization_name': 'GlobalSign nv-sa', - 'extra.issuer_country': 'BE', - 'extra.sha256_fingerprint': '27:4A:8A:3A:A7:DF:82:D0:43:03:0E:6F:48:30:30:C9:24:77:11:1A:08:EF:F7:B9:74:0C:CE:40:87:03:D2:51', - 'extra.sha512_fingerprint': 'E5:93:8B:72:84:0F:35:52:8E:7A:6C:E3:EF:36:90:4C:F2:86:A7:4D:B2:DD:C0:C6:23:83:18:EF:DD:86:34:92:91:57:22:29:75:45:71:8B:3A:CD:F1:27:A9:CA:5F:70:5E:AC:15:A5:E6:63:FD:6F:BB:C5:E2:45:99:73:E9:E6', - 'extra.md5_fingerprint': 'D1:A7:BC:96:78:1D:16:D0:24:A8:62:7C:3A:95:5A:4A', - 'extra.cert_valid': False, - 'extra.self_signed': False, - 'extra.cert_expired': False, - 'extra.validation_level': 'OV', - 'extra.auth_tls_response': '234 AUTH TLS successful', - }, - { - '__type': 'Event', - 'feed.name': 'Accessible FTP', - 'time.observation': '2019-03-25T00:00:00+00:00', - 'time.source': '2019-03-06T06:37:00+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-ftp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.ip': '62.48.156.65', - 'source.port': 21, - 'protocol.transport': 'tcp', - 'protocol.application': 'ftp', - 'source.reverse_dns': 'dial-62-48-156-65.ptprime.net', - 'extra.tag': 'ftp', - 'source.asn': 15525, - 'source.geolocation.cc': 'PT', - 'source.geolocation.region': 'LISBOA', - 'source.geolocation.city': 'FRIELAS', - 'extra.banner': '220-================================================================| PT Empresas| Acesso Reservado| Acesso nao autorizado punido por lei: 109/91; 67/98| ----------------------------------------------------------------| HENNES & MAURITZ LDA - 149093| SITE: PT303 - Cascais Shopping| MORADA: | NIR: EWS1822940| ================================================================|220 FTP server ready, 1 active clients of 4 simultaneous clients allowed.|', - 'extra.auth_tls_response': '500 Syntax error, command unrecognized.', - 'extra.auth_ssl_response': '500 Syntax error, command unrecognized.' - } - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py deleted file mode 100644 index 0b5794cb7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py +++ /dev/null @@ -1,94 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_hadoop.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Accessible-Hadoop", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_hadoop-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'ShadowServer Accessible-Hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-hadoop', - 'extra.version': '2.7.3, rbaa91f7c6bc9cb92be5982de4719c1c8af91ccff', - 'extra.server_type': 'namenode', - 'extra.clusterid': 'CID-64471a53-60cb-4302-9832-92f321f111fe', - 'extra.total_disk': 41567956992, - 'extra.used_disk': 53248, - 'extra.free_disk': 25160089600, - 'extra.livenodes': 'edmonton:50010', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 15296, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'CALGARY', - 'source.geolocation.region': 'ALBERTA', - 'source.ip': '199.116.235.200', - 'source.port': 50070, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-09-13T02:06:05+00:00'}, - {'__type': 'Event', - 'feed.name': 'ShadowServer Accessible-Hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-hadoop', - 'extra.version': '2.7.1.2.4.0.0-169', - 'extra.naics': 334111, - 'extra.sic': 357101, - 'extra.server_type': 'datanode', - 'extra.clusterid': 'CID-771bae52-9e4f-4ec4-bc1a-c867585751f0', - 'extra.namenodeaddress': 'sandbox.hortonworks.com', - 'extra.volumeinfo': '/hadoop/hdfs/data/current', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 8075, - 'source.geolocation.cc': 'US', - 'source.geolocation.city': 'DES MOINES', - 'source.geolocation.region': 'IOWA', - 'source.ip': '104.43.235.92', - 'source.port': 50075, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-09-13T02:07:48+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py deleted file mode 100644 index 793a95f22..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible HTTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_http-test-test.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible HTTP', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'accessible-http', - 'extra.source.naics': 518111, - 'extra.source.sic': 737401, - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_reason': 'OK', - 'extra.content_type': 'text/html', - 'extra.server': 'lighttpd', - 'extra.transfer_encoding': 'chunked', - 'extra.http_date': '2018-04-19T00:02:28+00:00', - 'extra.tag': 'http', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.reverse_dns': 'c-75-74-78-113.hsd1.fl.comcast.net', - 'source.asn': 7922, - 'source.geolocation.cc': 'US', - 'source.geolocation.city': 'MIAMI', - 'source.geolocation.region': 'FLORIDA', - 'source.ip': '75.74.78.113', - 'source.port': 8080, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2018-04-19T00:02:26+00:00'}, - {'__type': 'Event', - 'feed.name': 'Accessible HTTP', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'accessible-http', - 'extra.source.naics': 518210, - 'extra.source.sic': 737415, - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_reason': 'OK', - 'extra.content_type': 'text/html', - 'extra.content_length': 17729, - 'extra.http_date': '2018-04-19T02:02:28+00:00', - 'extra.tag': 'http', - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.reverse_dns': 'sto95-3-88-162-174-130.fbx.proxad.net', - 'source.asn': 12322, - 'source.geolocation.cc': 'FR', - 'source.geolocation.city': 'SAINT-OUEN-LAUMONE', - 'source.ip': '88.162.174.130', - 'source.port': 8080, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2018-04-19T00:02:26+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py deleted file mode 100644 index dc5e94e5e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http_proxy.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open HTTP Proxy', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_http_proxy-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3128, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'extra.via': 'HTTP/1.1 s_proxy_den1', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3128, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'extra.via': 'HTTP/1.1 s_proxy_yvr', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3128, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py deleted file mode 100644 index d15232eaf..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py +++ /dev/null @@ -1,125 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http_vulnerable.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable HTTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-08-01T09:00:00+00:00", - "extra.file_name": "2021-08-01-scan_http_vulnerable-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.content_length': 149, - 'extra.content_type': 'text/html; charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 401, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Unauthorized', - 'extra.server': 'TwistedWeb/19.7.0', - 'extra.set_cookie': 'TWISTED_SESSION=5473ad3faa3de66685fb3a53bffb390b4fcec2039893009a06caf38e1bec8aa8', - 'extra.tag': 'basic-auth,http', - 'extra.www_authenticate': 'Basic realm=\\\\"OpenWebif\\"\\""', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 8080, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.content_length': 149, - 'extra.content_type': 'text/html; charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 401, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Unauthorized', - 'extra.server': 'TwistedWeb/19.7.0', - 'extra.set_cookie': 'TWISTED_SESSION=d2460d37b7fdbdd6c27dd74423ead5704e553d4f2c230672313edc5602059e33', - 'extra.tag': 'basic-auth,http', - 'extra.www_authenticate': 'Basic realm=\\\\"OpenWebif\\"\\""', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 80, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.detail': 'repositoryformatversion = 0;filemode = false;bare = ' - 'false;logallrefupdates = true;symlinks = false;ignorecase = ' - 'true', - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.tag': 'git-config-file', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 443, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py deleted file mode 100644 index f673f40c8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py +++ /dev/null @@ -1,125 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ics.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Acessible ICS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ics-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 1', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDE=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.1', - 'source.port' : 502, - 'source.reverse_dns' : 'host1.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 2', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDI=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64513, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.2', - 'source.port' : 502, - 'source.reverse_dns' : 'host2.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 3', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDM=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64514, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.3', - 'source.port' : 502, - 'source.reverse_dns' : 'host3.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py deleted file mode 100644 index 08a9082af..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ipmi.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open IPMI', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ipmi-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open IPMI', - "classification.identifier": "open-ipmi", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.anon_login": False, - "extra.defaultkg": "-", - "extra.ipmi_version": "1.5", - "extra.md2_auth": False, - "extra.md5_auth": True, - "extra.none_auth": True, - "extra.nulluser": True, - "extra.oem_auth": False, - "extra.passkey_auth": True, - "extra.permessage_auth": True, - "extra.tag": "ipmi", - "extra.userlevel_auth": True, - "extra.usernames": False, - "protocol.application": "ipmi", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 2914, - "source.geolocation.cc": "DE", - "source.geolocation.city": "BERLIN", - "source.geolocation.region": "BERLIN", - "source.ip": "198.51.100.4", - "source.port": 623, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:09:42+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open IPMI', - "classification.identifier": "open-ipmi", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.anon_login": False, - "extra.defaultkg": "default", - "extra.ipmi_version": "2.0", - "extra.md2_auth": False, - "extra.md5_auth": False, - "extra.none_auth": False, - "extra.nulluser": False, - "extra.oem_auth": False, - "extra.passkey_auth": False, - "extra.permessage_auth": False, - "extra.tag": "ipmi", - "extra.userlevel_auth": True, - "extra.usernames": True, - "protocol.application": "ipmi", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 28753, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.182", - "source.port": 623, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:09:43+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py deleted file mode 100644 index 9adc8485e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ipp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open-IPP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-06-09T00:00:00+00:00", - "extra.file_name": "2020-06-08-scan_ipp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open-IPP', - "classification.identifier": "open-ipp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "ipp", - "extra.ipp_version": "IPP/2.1", - "extra.cups_version": "CUPS/2.0", - "extra.printer_uris": "ipp://123.45.67.89:631/ipp/print", - "extra.printer_name": "NPI3F0D22", - "extra.printer_info": "HP Color LaserJet MFP M277dw", - "extra.printer_more_info": "http://123.45.67.89:631/hp/device/info_config_AirPrint.html?tab=Networking&menu=AirPrintStatus", - "extra.printer_make_and_model": "HP Color LaserJet MFP M277dw", - "extra.printer_firmware_name": "20191203", - "extra.printer_firmware_string_version": "20191203", - "extra.printer_firmware_version": "20191203", - "extra.printer_organization": "org", - "extra.printer_organization_unit": "unit", - "extra.printer_uuid": "urn:uuid:456e4238-4a44-4643-4c42-10e1813f0a18", - "extra.printer_wifi_ssid": "wifissid", - "protocol.application": "ipp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 12345, - "source.geolocation.cc": "AA", - "source.geolocation.city": "CITY", - "source.geolocation.region": "REGION", - "source.ip": "123.45.67.89", - "source.port": 631, - 'source.reverse_dns': 'some.host.com', - "time.observation": "2020-06-09T00:00:00+00:00", - "time.source": "2020-06-08T11:30:14+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py deleted file mode 100644 index 3192f508f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py +++ /dev/null @@ -1,105 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_isakmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable ISAKMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_isakmp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Vulnerable ISAKMP', - "classification.identifier": "open-ike", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.domain_of_interpretation": '00', - "extra.exchange_type": '05', - "extra.flags": '00', - "extra.initiator_spi": "3e35c70729dfedef", - "extra.message_id": "00000000", - "extra.naics": 517311, - "extra.next_payload": '11', - "extra.next_payload2": '00', - "extra.notify_message_type": '14', - "extra.responder_spi": "253acab7cbfda607", - "extra.spi_size": 0, - "extra.tag": "isakmp-vulnerable", - "protocol.application": "ipsec", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.42", - "source.port": 500, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T00:17:25+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Vulnerable ISAKMP', - "classification.identifier": "open-ike", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.domain_of_interpretation": '00', - "extra.exchange_type": '05', - "extra.flags": '00', - "extra.initiator_spi": "3e35c70729dfedef", - "extra.message_id": "00000000", - "extra.next_payload": '11', - "extra.next_payload2": '00', - "extra.notify_message_type": '14', - "extra.responder_spi": "b274460e7adc1bf0", - "extra.spi_size": 0, - "extra.tag": "isakmp-vulnerable", - "protocol.application": "ipsec", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.67", - "source.port": 500, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T00:17:28+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py deleted file mode 100644 index 2bac336a7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py +++ /dev/null @@ -1,214 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_kubernetes.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Kubernetes API Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_kubernetes-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2021-11-17T13:00:29Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:13 GMT', - 'extra.git_commit' : '2444b3347a2c45eb965b182fb836e1f51dc61b70', - 'extra.git_tree_state' : 'clean', - 'extra.git_version' : 'v1.20.13', - 'extra.go_version' : 'go1.15.15', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '20', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 6443, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2022-02-25T06:26:46Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:12 GMT', - 'extra.git_commit' : '6f5a5295923a614a4202a7ad274b38b69f9ca8c0', - 'extra.git_tree_state' : 'clean', - 'extra.git_version' : 'v1.23.3+e419edf', - 'extra.go_version' : 'go1.17.5', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '23', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 6443, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2020-05-08T07:29:59Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:12 GMT', - 'extra.git_commit' : '4f7ea78', - 'extra.git_version' : 'v1.16.9-aliyun.1', - 'extra.go_version' : 'go1.13.9', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '16+', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 6443, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py deleted file mode 100644 index b6abf6eba..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py +++ /dev/null @@ -1,154 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ldap_tcp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open LDAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ldap_tcp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 2, - 'extra.ldap_service_name': 'node01.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'node01.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821124435.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 25029662, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node02.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'node02.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821124539.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.ldap_service_name': 'node03.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'node03.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py deleted file mode 100644 index aa4deefb8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py +++ /dev/null @@ -1,162 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ldap_udp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open LDAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ldap_udp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 58.42, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821044533.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 222537, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node01.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 3038, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'node01.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 58.88, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821044948.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 1478714, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node02.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 3062, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'node02.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 0.69, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.ldap_service_name': 'node03.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 36, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'node03.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py deleted file mode 100644 index 9207aaf36..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mdns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open mDNS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mdns-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_ipv4' : '192.168.0.1', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::1', - 'extra.services' : '_smb._tcp.local.; _device-info._tcp.local.; _http._tcp.local.; _dacp._tcp.local.;', - 'extra.tag' : 'mdns', - 'extra.workstation_ipv4' : '192.168.0.1', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::1', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5353, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_ipv4' : '192.168.0.2', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::2', - 'extra.services' : '_home-assistant._tcp.local.;', - 'extra.tag' : 'mdns', - 'extra.workstation_ipv4' : '192.168.0.2', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::2', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5353, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_info' : '\\\\\"vendor=Synology\\\"\\\" \\\"\\\"model=DS218+\\\"\\\" \\\"\\\"serial=17A0PCN482002\\\"\\\" \\\"\\\"version_major=6\\\"\\\" \\\"\\\"version_minor=2\\\"\\\" \\\"\\\"version_build=25556\\\"\\\" \\\"\\\"admin_port=5000\\\"\\\" \\\"\\\"secure_admin_port=5001\\\"\\\" \\\"\\\"mac_address=00:11:32:80:fd:b5\\\"\\\"\"', - 'extra.http_ipv4' : '192.168.0.3', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::3', - 'extra.http_name' : 'snmeijer.local.', - 'extra.http_port' : 5000, - 'extra.http_ptr' : 'snmeijer._http._tcp.local.', - 'extra.http_target' : 'snmeijer.local.', - 'extra.services' : '_webdav._tcp.local.; _adisk._tcp.local.; _smb._tcp.local.; _http._tcp.local.; _dacp._tcp.local.; _afpovertcp._tcp.local.; _device-info._tcp.local.;', - 'extra.tag' : 'mdns,iot', - 'extra.workstation_ipv4' : '192.168.0.3', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::3', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5353, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py deleted file mode 100644 index b54fc0ea5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py +++ /dev/null @@ -1,130 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_memcached.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Memcached', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_memcached-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 81.71, - 'extra.curr_connections': 243, - 'extra.pid': 1010, - 'extra.pointer_size': 64, - 'extra.response_size': 1144, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:34:06', - 'extra.total_connections': 6106, - 'extra.uptime': 32908114, - 'extra.version': '1.4.15', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 50260, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 75.21, - 'extra.curr_connections': 9, - 'extra.pid': 5316, - 'extra.pointer_size': 64, - 'extra.response_size': 1053, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:39:21', - 'extra.total_connections': 2962, - 'extra.uptime': 9618498, - 'extra.version': '1.4.13', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 11211, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 31.57, - 'extra.curr_connections': 2, - 'extra.pid': 1460, - 'extra.pointer_size': 32, - 'extra.response_size': 442, - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:39:39', - 'extra.total_connections': 534, - 'extra.uptime': 1375159, - 'extra.version': '1.2.6', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 11211, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py deleted file mode 100644 index 3ecf7b21f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mongodb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open MongoDB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mongodb-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open MongoDB', - "classification.identifier": "open-mongodb", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.allocator": "tcmalloc", - "extra.bits": "64", - "extra.gitversion": "a2ddc68ba7c9cee17bfe69ed840383ec3506602b", - "extra.javascriptengine": "V8", - "extra.maxbsonobjectsize": "16777216", - "extra.ok": True, - "extra.sysinfo": "Linux ip-198-51-100-100 198.51.100.103-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49", - "extra.tag": "mongodb", - "extra.version": "2.4.5", - "extra.visible_databases": "local | countly | admin", - "protocol.application": "mongodb", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 20773, - "source.geolocation.cc": "DE", - "source.geolocation.city": "WEEZE", - "source.geolocation.region": "NORDRHEIN-WESTFALEN", - "source.ip": "198.51.100.203", - "source.port": 27017, - "source.reverse_dns": "198-51-100-203.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:40:07+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open MongoDB', - "classification.identifier": "open-mongodb", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.allocator": "tcmalloc", - "extra.bits": "64", - "extra.gitversion": "d73c92b1c85703828b55c2916a5dd4ad46535f6a", - "extra.javascriptengine": "V8", - "extra.maxbsonobjectsize": "16777216", - "extra.ok": True, - "extra.sector": "Information Technology", - "extra.sysinfo": "Linux build5.ny.cbi.10gen.cc 2.6.32-431.3.1.el6.x86_64 #1 SMP Fri Jan 3 21:39:27 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49", - "extra.tag": "mongodb", - "extra.version": "2.6.12", - "extra.visible_databases": "none visible", - "protocol.application": "mongodb", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 24940, - "source.geolocation.cc": "DE", - "source.geolocation.city": "GUNZENHAUSEN", - "source.geolocation.region": "BAYERN", - "source.ip": "198.51.100.42", - "source.port": 27017, - "source.reverse_dns": "198-51-100-208.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:40:07+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py deleted file mode 100644 index 45d19f9ee..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py +++ /dev/null @@ -1,89 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mqtt.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open-MQTT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-03-15T00:00:00+00:00", - "extra.file_name": "2020-03-14-scan_mqtt-test-geo.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.anonymous_access' : False, - 'extra.cert_expiration_date' : '2022-11-14 00:00:00', - 'extra.cert_issue_date' : '2020-08-12 00:00:00', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '085699743A23114C9B6B8DC975A8AF42', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Refused, not authorized', - 'extra.hex_code' : '05', - 'extra.issuer_common_name' : 'Sectigo RSA Domain Validation Secure Server CA', - 'extra.issuer_country' : 'GB', - 'extra.issuer_locality_name' : 'Salford', - 'extra.issuer_organization_name' : 'Sectigo Limited', - 'extra.issuer_state_or_province_name' : 'Greater Manchester', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'DE:2C:98:30:27:2E:7D:C9:ED:A3:9D:AF:9E:CE:14:CC', - 'extra.raw_response' : '20020005', - 'extra.sha1_fingerprint' : '70:84:F1:6D:28:DA:B6:E6:27:60:13:8B:2C:93:52:B6:7B:4B:13:7B', - 'extra.sha256_fingerprint' : 'D2:D7:54:52:EB:86:4E:2D:34:4D:FC:CE:CD:CF:39:41:E1:06:5C:8B:B8:54:E6:0C:DF:FD:6E:E3:F1:B5:41:00', - 'extra.sha512_fingerprint' : '17:57:FB:88:9D:BE:A7:F0:29:A5:31:FC:79:DF:F7:8A:1C:D6:4A:DF:1B:4A:DC:BF:05:E7:E8:2F:79:9A:FA:FE:F7:E8:66:22:CB:B9:4C:72:F7:FB:6C:1D:59:8C:54:63:70:05:DE:7F:3C:2F:BA:B8:37:18:CE:29:6F:11:E8:AB', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.naics' : 454110, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : '*.tracesafe.io', - 'extra.tag' : 'mqtt', - 'feed.name' : 'Open-MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 12345, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'COLUMBUS', - 'source.geolocation.region' : 'OHIO', - 'source.ip' : '18.220.0.0', - 'source.port' : 8883, - 'source.reverse_dns' : '18-220-0-0.example.com', - 'time.observation' : '2020-03-15T00:00:00+00:00', - 'time.source' : '2022-02-07T12:56:53+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py deleted file mode 100644 index 461895724..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py +++ /dev/null @@ -1,173 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_mqtt_anon.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Anonymous MQTT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_mqtt_anon-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2030-05-06 08:07:05', - 'extra.cert_issue_date' : '2020-05-08 08:07:05', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '02', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'RootCA', - 'extra.issuer_country' : 'CN', - 'extra.issuer_organization_name' : 'EMQ', - 'extra.issuer_state_or_province_name' : 'hangzhou', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'AB:A8:E0:2C:EF:AE:BF:9D:DD:FA:70:BA:2F:F2:CA:5C', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '70:1A:1E:1F:EC:5F:7E:A9:12:32:B2:C9:8A:C9:EE:91:8E:0B:82:45', - 'extra.sha256_fingerprint' : '85:26:A2:F2:A2:50:CD:96:33:19:A6:2D:12:2E:97:6B:D3:06:3C:11:EA:01:B4:B7:25:2A:B7:4F:0A:8F:45:40', - 'extra.sha512_fingerprint' : '72:50:07:30:9A:6F:CB:FD:E2:80:69:02:65:62:77:16:C3:B4:0C:98:44:4E:D4:2C:AC:6B:AF:F8:9E:AB:51:C2:FA:A8:72:A3:45:DF:81:09:50:08:18:EB:03:34:FC:92:33:A7:12:46:FE:90:20:91:86:C5:4D:89:48:86:4C:CD', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 518210, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'Server', - 'extra.subject_country' : 'CN', - 'extra.subject_organization_name' : 'EMQ', - 'extra.subject_state_or_province_name' : 'hangzhou', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37963, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'SHENZHEN', - 'source.geolocation.region' : 'GUANGDONG SHENG', - 'source.ip' : '47.106.0.0', - 'source.port' : 8883, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2022-03-06 13:48:03', - 'extra.cert_issue_date' : '2021-12-06 13:48:04', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '06B25BEAD1F43266ABCFCDDE408D3544D04B', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'R3', - 'extra.issuer_country' : 'US', - 'extra.issuer_organization_name' : 'Lets Encrypt', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '23:99:39:C6:77:D8:9F:55:90:FC:A5:FB:BA:72:8B:42', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '20:0E:AC:E7:AF:07:8D:D3:16:7C:63:D1:B9:12:AD:1D:2C:F0:46:86', - 'extra.sha256_fingerprint' : 'DD:7A:4C:8A:1D:66:1D:7C:F5:17:04:5B:A0:B4:C4:E0:80:58:44:B4:DB:A7:5E:61:AE:43:9D:85:4C:9E:DC:83', - 'extra.sha512_fingerprint' : '55:B6:3D:56:A4:39:6E:99:B6:AF:72:AF:4D:3C:7C:C5:A8:C5:4F:A1:79:92:D0:46:8A:A2:9B:2A:48:0D:00:68:39:F0:B8:67:B4:E0:88:51:2A:D7:55:46:83:BD:ED:1E:09:6E:DB:3D:21:E2:AA:DB:42:6A:33:45:1A:2A:DB:4C', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 518210, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 24940, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'WERNIGERODE', - 'source.geolocation.region' : 'SACHSEN-ANHALT', - 'source.ip' : '144.76.0.0', - 'source.port' : 8883, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2030-08-05 16:51:57', - 'extra.cert_issue_date' : '2020-08-07 16:51:57', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'A71541EFAE529B03', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'ClearView2Dev', - 'extra.issuer_organization_name' : 'Sohonet', - 'extra.issuer_organization_unit_name' : 'ClearView2Dev', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '43:0D:A7:89:9E:76:8D:6E:D5:AD:95:CC:F2:91:87:56', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '32:4B:66:98:FA:5B:D2:D1:F2:53:83:21:19:11:5A:A9:BE:85:56:16', - 'extra.sha256_fingerprint' : 'AE:0D:65:34:2F:51:F7:32:1E:DF:B1:DA:12:C7:6A:DE:42:B5:4B:FF:80:2C:E5:EF:99:F6:CC:01:4B:C9:77:68', - 'extra.sha512_fingerprint' : '44:C4:B8:19:FA:39:55:51:EC:E4:6D:C4:6D:0F:A5:46:BB:D5:F9:FD:A6:8D:DF:F3:2D:D2:92:6C:0B:D5:D3:25:CB:19:50:9D:A6:A4:D4:D3:2E:53:10:F5:8D:77:F7:90:F8:65:A7:79:AB:14:62:72:01:F3:EA:38:E2:68:C7:25', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 0, - 'extra.subject_common_name' : 'foo.example.com', - 'extra.subject_locality_name' : '<', - 'extra.subject_organization_name' : 'Sohonet', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 5555, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'BURBANK', - 'source.geolocation.region' : 'CALIFORNIA', - 'source.ip' : '173.0.0.0', - 'source.port' : 8883, - 'source.reverse_dns' : 'example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py deleted file mode 100644 index 0f12014e6..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mssql.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open MSSQL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mssql-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 310.0, - 'extra.instance_name': 'OPTIMA', - 'extra.named_pipe': '\\\\\\\\ERPOPTIMA\\\\pipe\\\\MSSQL$OPTIMA\\\\sql\\\\query', - 'extra.response_size': 310, - 'extra.tag': 'mssql', - 'extra.tcp_port': 49729, - 'extra.version': '13.2.5026.0', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'ERPOPTIMA', - 'source.port': 1434, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 226.0, - 'extra.instance_name': 'MSSQLSERVER', - 'extra.response_size': 226, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'mssql', - 'extra.tcp_port': 1433, - 'extra.version': '13.0.1601.5', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'SERWER', - 'source.port': 1434, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 304.0, - 'extra.instance_name': 'INSERTGT', - 'extra.named_pipe': '\\\\\\\\ILONY\\\\pipe\\\\MSSQL$INSERTGT\\\\sql\\\\query', - 'extra.response_size': 304, - 'extra.tag': 'mssql', - 'extra.tcp_port': 49358, - 'extra.version': '10.50.2500.0', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'ILONY', - 'source.port': 1434, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py deleted file mode 100644 index 3e008f950..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py +++ /dev/null @@ -1,258 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_mysql.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible MySQL Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_mysql-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '5.7.37-0ubuntu0.18.04.1', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 3306, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '5.7.30-0ubuntu0.18.04.1-log', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 3306, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '8.0.23', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 3306, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py deleted file mode 100644 index beeac2717..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_nat_pmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open NATPMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_nat_pmp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.1', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 291278940, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5351, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.2', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 768416, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5351, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.3', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 19629454, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5351, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py deleted file mode 100644 index febe8305c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_netbios.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Netbios', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_netbios-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 4.58, - 'extra.mac_address': '00-00-00-00-00-00', - 'extra.machine_name': 'NBG6503', - 'extra.response_size': 229, - 'extra.tag': 'netbios', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.account': 'NBG6503', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 137, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.86, - 'extra.mac_address': '00-00-00-00-00-00', - 'extra.machine_name': 'NAS-OLD', - 'extra.response_size': 193, - 'extra.tag': 'netbios', - 'extra.workgroup': 'PRACOWNIAELN.', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.account': 'NAS-OLD', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 137, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.14, - 'extra.mac_address': '00-25-90-F0-64-64', - 'extra.machine_name': 'HR-SRV01', - 'extra.response_size': 157, - 'extra.sector': 'Government', - 'extra.tag': 'netbios', - 'extra.workgroup': 'HRSIGMA', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': 'InRpbWVzdGFtcCIsImlwIiwicHJvdG9jb2wiLCJwb3J0IiwiaG9zdG5hbWUiLCJ0YWciLCJtYWNfYWRkcmVzcyIsImFzbiIsImdlbyIsInJlZ2lvbiIsImNpdHkiLCJ3b3JrZ3JvdXAiLCJtYWNoaW5lX25hbWUiLCJ1c2VybmFtZSIsIm5haWNzIiwic2ljIiwic2VjdG9yIiwicmVzcG9uc2Vfc2l6ZSIsImFtcGxpZmljYXRpb24iCiIyMDEwLTAyLTEwIDAwOjAwOjAyIiwxOTIuMTY4LjAuMyx1ZHAsMTM3LG5vZGUwMy5leGFtcGxlLmNvbSxuZXRiaW9zLDAwLTI1LTkwLUYwLTY0LTY0LDY0NTEyLFpaLFJlZ2lvbixDaXR5LEhSU0lHTUEsSFItU1JWMDEsLDAsMCxHb3Zlcm5tZW50LDE1NywzLjE0', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 137, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py deleted file mode 100644 index 043cdf1aa..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_netis_router.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_netis_router-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 53413, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 53413, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 53413, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py deleted file mode 100644 index 85ef710d4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py +++ /dev/null @@ -1,161 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ntp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'NTP Version', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ntp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.0, - 'extra.clock': '0xe6ac3809.363028e7', - 'extra.frequency': 2.018, - 'extra.jitter': 0.977, - 'extra.leap': 0.0, - 'extra.noise': '0.984', - 'extra.offset': 0.557, - 'extra.peer': 18986, - 'extra.poll': 10, - 'extra.precision': -10, - 'extra.refid': '81.15.252.130', - 'extra.reftime': '0xe6ac35ba.2d2e8f2b', - 'extra.response_size': 324, - 'extra.rootdelay': 17.685, - 'extra.rootdispersion': 61.254, - 'extra.stability': '0.027', - 'extra.state': '4', - 'extra.stratum': 4, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 123, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.33, - 'extra.clk_wander': 0.007, - 'extra.clock': '0xE6AC3806.7DF3B7A0', - 'extra.frequency': -20.407, - 'extra.jitter': 8.776, - 'extra.leap': 0.0, - 'extra.mintc': '3', - 'extra.offset': -14.502, - 'extra.peer': 19244, - 'extra.precision': -10, - 'extra.refid': '10.48.21.21', - 'extra.reftime': '0xE6AC3431.B3B64790', - 'extra.response_size': 328, - 'extra.rootdelay': 32.25, - 'extra.rootdispersion': 105.778, - 'extra.sector': 'Transportation and Warehousing', - 'extra.stratum': 8, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.tc': 10, - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 123, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.0, - 'extra.clk_wander': 0.001, - 'extra.clock': '0xE6AC380A.5A1CAD00', - 'extra.frequency': -24.01, - 'extra.jitter': 2.343, - 'extra.leap': 0.0, - 'extra.mintc': '3', - 'extra.offset': 0.49, - 'extra.peer': 51892, - 'extra.precision': -10, - 'extra.refid': '172.28.0.1', - 'extra.reftime': '0xE6AC3020.0C49BA80', - 'extra.response_size': 324, - 'extra.rootdelay': 7.749, - 'extra.rootdispersion': 81.612, - 'extra.stratum': 4, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.tc': 10, - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 123, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py deleted file mode 100644 index ff0e95f3e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py +++ /dev/null @@ -1,108 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ntpmonitor.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'NTP Monitor', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ntpmonitor-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 55.33, - 'extra.packets': 2, - 'extra.size': 664, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 123, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3666.67, - 'extra.packets': 100, - 'extra.size': 44000, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 123, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3666.67, - 'extra.packets': 100, - 'extra.size': 44000, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 123, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py deleted file mode 100644 index 11caec78a..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_portmapper.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Portmapper', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_portmapper-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 111, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 111, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Government', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 111, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py deleted file mode 100644 index 43a297f78..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py +++ /dev/null @@ -1,199 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_postgres.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-PostgreSQL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_postgres-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5432, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5432, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5432, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py deleted file mode 100644 index de52af625..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_qotd.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open QOTD', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_qotd-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 166.0, - 'extra.quote': '_The secret of being miserable is to have leisure to bother ' - 'about whether?? you are happy or not. The cure for it is ' - 'occupation._?? George Bernard Shaw (1856-1950)?', - 'extra.response_size': 166, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 17, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 162.0, - 'extra.quote': '_Oh the nerves, the nerves; the mysteries of this machine ' - 'called man!?? Oh the little that unhinges it, poor creatures ' - 'that we are!_?? Charles Dickens (1812-70)?', - 'extra.response_size': 162, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 17, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 162.0, - 'extra.quote': '_Oh the nerves, the nerves; the mysteries of this machine ' - 'called man!?? Oh the little that unhinges it, poor creatures ' - 'that we are!_?? Charles Dickens (1812-70)?', - 'extra.response_size': 162, - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 17, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py deleted file mode 100644 index 23d11ce99..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_quic.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible QUIC Report', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_quic-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517311, - 'extra.tag' : 'quic', - 'extra.version_field_1' : 'Q050', - 'extra.version_field_3' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 5607, - 'source.geolocation.cc' : 'UK', - 'source.geolocation.city' : 'LONDON', - 'source.geolocation.region' : 'LONDON', - 'source.ip' : '176.255.0.0', - 'source.port' : 443, - 'source.reverse_dns' : 'test1.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517311, - 'extra.tag' : 'quic', - 'extra.version_field_1' : 'Q050', - 'extra.version_field_2' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 6327, - 'source.geolocation.cc' : 'CA', - 'source.geolocation.city' : 'MEACHAM', - 'source.geolocation.region' : 'SASKATCHEWAN', - 'source.ip' : '24.244.0.0', - 'source.port' : 443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517919, - 'extra.tag' : 'quic', - 'extra.version_field_2' : 'Q050', - 'extra.version_field_3' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'JP', - 'source.geolocation.city' : 'OSAKA', - 'source.geolocation.region' : 'OSAKA', - 'source.ip' : '23.60.0.0', - 'source.port' : 443, - 'source.reverse_dns' : 'test3.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py deleted file mode 100644 index 7c052c451..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py +++ /dev/null @@ -1,236 +0,0 @@ -# SPDX-FileCopyrightText: 2020 sinus-x -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/scan_radmin.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "Accessible Radmin", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-scan_radmin-test-test.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin (Details Unknown)", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.asn": 701, - "source.geolocation.cc": "US", - "source.geolocation.city": "BROOKLYN", - "source.geolocation.region": "NEW YORK", - "source.ip": "74.101.218.75", - "source.port": 4899, - "source.reverse_dns": "static-74-101-218-75.nycmny.fios.verizon.net", - "time.source": "2020-07-06T13:55:26+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.asn": 56618, - "source.geolocation.cc": "RU", - "source.geolocation.city": "MURMANSK", - "source.geolocation.region": "MURMANSKAYA OBLAST", - "source.ip": "192.162.189.171", - "source.port": 4899, - "source.reverse_dns": "rubin.an.ru", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin (Details Unknown)", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "CN", - "source.geolocation.city": "BEIJING", - "source.geolocation.region": "BEIJING SHI", - "source.asn": 4808, - "source.ip": "111.197.143.69", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "KR", - "source.geolocation.city": "DAEIN-DONG", - "source.geolocation.region": "GWANGJU-GWANGYEOKSI", - "source.asn": 4766, - "source.ip": "121.147.215.220", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "KR", - "source.geolocation.city": "DAEIN-DONG", - "source.geolocation.region": "GWANGJU-GWANGYEOKSI", - "source.asn": 4766, - "source.ip": "121.147.215.178", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "CN", - "source.geolocation.city": "CHONGQING", - "source.geolocation.region": "CHONGQING SHI", - "source.asn": 9808, - "source.ip": "183.230.5.219", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[6]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "RU", - "source.geolocation.city": "MOSCOW", - "source.geolocation.region": "MOSKVA", - "source.asn": 34300, - "source.ip": "85.93.154.74", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[7]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "BE", - "source.geolocation.city": "BRASSCHAAT", - "source.geolocation.region": "ANTWERPEN", - "source.asn": 5432, - "source.ip": "81.246.135.247", - "source.port": 4899, - "source.reverse_dns": "247.135-246-81.adsl-dyn.isp.belgacom.be", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[8]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "ES", - "source.geolocation.city": "LAS PALMAS DE GRAN CANARIA", - "source.geolocation.region": "LAS PALMAS", - "source.asn": 12430, - "source.ip": "46.27.146.22", - "source.port": 4899, - "source.reverse_dns": "static-22-146-27-46.ipcom.comunitel.net", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[9]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py deleted file mode 100644 index 28a4a02c2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rdp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible RDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rdp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible RDP', - "classification.identifier": "open-rdp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.bluekeep_vulnerable": False, - "extra.cert_expiration_date": "2019-10-29 02:22:06", - "extra.cert_issue_date": "2019-04-29 02:22:06", - "extra.cert_length": 5678, - "extra.cert_serial_number": "1EF2B37AF850C9BF4E88F18177001D6B", - "extra.cve20190708_vulnerable": False, - "extra.issuer_common_name": "KABESRV.KABE.local", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "BC:6E:C3:E2:98:22:EC:BA:5B:30:E2:53:FD:4A:9D:FF", - "extra.naics": 517311, - "extra.rdp_protocol": "RDP", - "extra.sha1_fingerprint": "EC:BB:4D:DB:9F:0C:D3:FF:5B:49:EA:B1:56:62:B6:A7:5D:60:54:42", - "extra.sha256_fingerprint": "B7:C9:F4:07:D5:C0:75:1D:EA:0C:40:E7:26:39:C2:30:C6:13:83:7E:18:46:D8:E9:4C:45:3F:88:1B:0B:70:76", - "extra.sha512_fingerprint": "08:AC:75:FA:EB:A3:2B:44:15:DE:6D:A7:0B:C0:AE:17:94:F3:55:D9:EC:70:AC:5B:B7:94:79:F0:D7:84:83:89:CB:A9:11:E0:08:D7:54:4D:33:85:89:D2:A8:DD:9D:15:F4:CC:95:DE:6A:E3:DF:6B:FA:8B:27:E3:DA:16:AF:0A", - "extra.signature_algorithm": "sha256WithRSAEncryption", - "extra.ssl_version": 2, - "extra.subject_common_name": "KABESRV.KABE.local", - "extra.tag": "rdp", - "protocol.application": "rdp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.178", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T15:45:51+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible RDP', - "classification.identifier": "open-rdp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.bluekeep_vulnerable": False, - "extra.cert_expiration_date": "2019-10-16 06:15:20", - "extra.cert_issue_date": "2019-04-16 06:15:20", - "extra.cert_length": 5678, - "extra.cert_serial_number": "3FF3EBC5CF154BA54D128A8548C8AAF5", - "extra.cve20190708_vulnerable": False, - "extra.issuer_common_name": "RAMBLA01.rambla.local", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "38:73:6A:B3:AA:41:69:C9:BA:E7:3D:D7:40:16:F8:AA", - "extra.naics": 517311, - "extra.rdp_protocol": "RDP", - "extra.sector": "Information Technology", - "extra.sha1_fingerprint": "7A:67:1F:F8:87:C6:B0:AC:A9:84:15:B7:40:EC:CB:19:AA:E3:19:52", - "extra.sha256_fingerprint": "8F:CD:7D:C4:80:2D:8D:9B:06:A0:40:18:9F:ED:73:7A:BA:83:55:BE:1B:56:83:A2:97:DF:BB:B4:06:57:CB:F1", - "extra.sha512_fingerprint": "E8:9B:9A:93:69:B4:58:01:D8:46:C2:DC:01:20:1E:DD:93:E1:EB:E3:9D:6B:65:A0:C5:00:6C:A4:44:08:FE:A4:A6:19:FF:55:79:F2:AA:61:68:C8:1C:B0:CE:78:EB:84:DD:29:9D:64:2F:4E:25:31:3A:6C:B8:02:C9:AF:F5:1F", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.ssl_version": 2, - "extra.subject_common_name": "RAMBLA01.rambla.local", - "extra.tag": "rdp", - "protocol.application": "rdp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.233", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T15:45:51+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py deleted file mode 100644 index 54be35a26..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rdpeudp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible MS RDPEUDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rdpeudp-test-geo.csv", - } - -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '05b28c0c', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '053d355f', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '0567a8cb', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py deleted file mode 100644 index 04552e2ec..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_redis.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Redis', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_redis-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open Redis', - "classification.identifier": "open-redis", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.build_id": "26069fb482f6334b", - "extra.connected_clients": "50", - "extra.gcc_version": "4.7.2", - "extra.git_sha1": "00000000", - "extra.mode": "standalone", - "extra.multiplexing_api": "epoll", - "extra.naics": 541512, - "extra.os.name": "Linux 3.2.0-4-amd64 x86_64", - "extra.process_id": "2127", - "extra.run_id": "d440b0b2fb3d1db655ad607e11e6f38011a0f599", - "extra.sic": 737999, - "extra.tag": "redis", - "extra.uptime": 27946314, - "extra.version": "2.8.19", - "protocol.application": "redis", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 201229, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.152", - "source.port": 6379, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:42:33+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open Redis', - "classification.identifier": "open-redis", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.build_id": "e41bf84a0cecf09d", - "extra.connected_clients": "25376", - "extra.gcc_version": "4.8.4", - "extra.git_sha1": "00000000", - "extra.mode": "standalone", - "extra.multiplexing_api": "epoll", - "extra.os.name": "Linux 3.18.24-sirzion x86_64", - "extra.process_id": "343519", - "extra.run_id": "53d63f23511dc0080b49aaa8e8203d65619f1c8c", - "extra.tag": "redis", - "extra.uptime": 310556, - "extra.version": "3.0.6", - "protocol.application": "redis", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 12586, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.67", - "source.port": 6379, - "source.reverse_dns": "198-51-100-67.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:42:43+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py deleted file mode 100644 index e2a961f71..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rsync.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Rsync', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rsync-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 873, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 873, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 873, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py deleted file mode 100644 index 6b972ec5d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_sip.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-SIP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_sip-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.sip_allow': 'INVITE,ACK,BYE,CANCEL,REGISTER', - 'extra.amplification': 15.57, - 'extra.content_length': 0, - 'extra.response_size': 109, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '489', - 'extra.sip_reason': 'Event Package Not Supported', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5060, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 62.57, - 'extra.content_length': 364, - 'extra.content_type': 'text/plain', - 'extra.response_size': 438, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '400', - 'extra.sip_reason': 'Bad Request', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5060, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.57, - 'extra.content_length': 0, - 'extra.response_size': 46, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '400', - 'extra.sip_reason': 'Bad Request', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5060, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py deleted file mode 100644 index f05973cf5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py +++ /dev/null @@ -1,137 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_slp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SLP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_slp-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 427, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 427, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 427, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py deleted file mode 100644 index 921525122..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_smb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SMB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_smb-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py deleted file mode 100644 index cae83d273..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest -import json - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser_json import ShadowserverJSONParserBot -from intelmq.tests.bots.parsers.shadowserver.test_testdata import csvtojson - -EXAMPLE_FILE = csvtojson(os.path.join(os.path.dirname(__file__), 'testdata/scan_smb.csv')) - -EXAMPLE_REPORT = { - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_smb-test-geo.json", - } - -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[0]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverJSONParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverJSONParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverJSONParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py deleted file mode 100644 index 4428420cf..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_smtp_vulnerable.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable SMTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-07-08T00:00:00+00:00", - "extra.file_name": "2021-07-08-scan_smtp_vulnerable-test-test.csv", - } - -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'vulnerable-smtp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.banner' : '220 smtp-server.invalid ESMTP Exim 4.80 Wed, 11 Jun 2021 10:00:00 +0300|', - 'extra.tag' : 'smtp;21nails', - 'feed.name' : 'Vulnerable SMTP', - 'protocol.application' : 'smtp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 12345, - 'source.geolocation.cc' : 'EE', - 'source.geolocation.city' : 'TALLINN', - 'source.geolocation.region' : 'HARJUMAA', - 'source.ip' : '1.2.3.4', - 'source.port' : 25, - 'source.reverse_dns' : 'smtp-server.invalid', - 'time.observation' : '2021-07-08T00:00:00+00:00', - 'time.source' : '2021-07-08T11:58:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'vulnerable-smtp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.banner' : '220 smtp-out.invalid, ESMTP EXIM 4.86_2|', - 'extra.tag' : 'smtp;21nails', - 'feed.name' : 'Vulnerable SMTP', - 'protocol.application' : 'smtp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 23456, - 'source.geolocation.cc' : 'EE', - 'source.geolocation.city' : 'TALLINN', - 'source.geolocation.region' : 'HARJUMAA', - 'source.ip' : '5.6.7.8', - 'source.port' : 25, - 'source.reverse_dns' : 'smtp-out.invalid', - 'time.observation' : '2021-07-08T00:00:00+00:00', - 'time.source' : '2021-07-08T11:58:44+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py deleted file mode 100644 index e6da5b34f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_snmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SNMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_snmp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.94, - 'extra.community': 'public', - 'extra.response_size': 165, - 'extra.sysdesc': 'Linux localhost 3.18.20 #1 SMP Mon Jul 9 14:11:21 CST 2018 ' - 'armv7l', - 'extra.tag': 'snmp', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 161, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.35, - 'extra.community': 'public', - 'extra.device_sector': 'consumer', - 'extra.device_type': 'router', - 'extra.device_vendor': 'MikroTik', - 'extra.response_size': 115, - 'extra.sysdesc': 'RouterOS CCR1009-8G-1S-1S+', - 'extra.tag': 'snmp,iot', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 161, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.0, - 'extra.community': 'public', - 'extra.response_size': 85, - 'extra.tag': 'snmp', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 161, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py deleted file mode 100644 index 067602aa1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_socks.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SOCKS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_socks-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks4', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 1080, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks5', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 1080, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Retail Trade', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks4', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 1080, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py deleted file mode 100644 index 0811f15ed..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py +++ /dev/null @@ -1,136 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssdp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SSDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssdp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.35, - 'extra.cache_control': 'max-age=100', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node01.example.com', - 'extra.location': 'http://192.168.200.254:49152/description.xml', - 'extra.response_size': 325, - 'extra.search_target': 'upnp:rootdevice', - 'extra.sector': 'Government', - 'extra.server': 'Linux/2.6.26, UPnP/1.0, Portable SDK for UPnP devices/1.3.1', - 'extra.systime': 'Sun, 21 Aug 2022 09:51:13 GMT', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:28802880-2880-1880-a880-001bc502f600::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 60194, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 2.71, - 'extra.cache_control': 'max-age = 1800', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node02.example.com', - 'extra.location': 'http://95.160.216.14:52235/dmr/SamsungMRDesc.xml', - 'extra.response_size': 263, - 'extra.search_target': 'upnp:rootdevice', - 'extra.server': 'Linux/9.0 UPnP/1.0 PROTOTYPE/1.0', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:f144ca92-6816-94b5-b95f-b58180834044::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 38732, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 4.79, - 'extra.cache_control': 'max-age=1800', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node03.example.com', - 'extra.location': 'http://192.168.1.3:8008/ssdp/device-desc.xml', - 'extra.response_size': 465, - 'extra.search_target': 'upnp:rootdevice', - 'extra.sector': 'Government', - 'extra.server': 'Linux/3.10.79, UPnP/1.0, Portable SDK for UPnP ' - 'devices/1.6.18', - 'extra.systime': 'Sun, 03 Jan 2016 21:37:50 GMT', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:62fa0fc8-079d-d00f-2e22-59b49fb488f9::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 57626, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py deleted file mode 100644 index a01383713..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py +++ /dev/null @@ -1,182 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssh.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SSH', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ssh-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ecdsa-sha2-nistp256', - 'extra.available_ciphers' : 'chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com, aes128-cbc, aes192-cbc, aes256-cbc, blowfish-cbc, cast128-cbc, 3des-cbc', - 'extra.available_compression' : 'none, zlib@openssh.com', - 'extra.available_kex' : 'curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha256, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1', - 'extra.available_mac' : 'umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1', - 'extra.ecdsa_curve' : 'P-256', - 'extra.ecdsa_curve25519' : '1xx7ASut7BF4ED8b592bebZBMBKTCzOsmbH4cjwx/0U=', - 'extra.ecdsa_public_key_b' : 'WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=', - 'extra.ecdsa_public_key_gx' : 'axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=', - 'extra.ecdsa_public_key_gy' : 'T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=', - 'extra.ecdsa_public_key_length' : '256', - 'extra.ecdsa_public_key_n' : '/////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=', - 'extra.ecdsa_public_key_p' : '/////wAAAAEAAAAAAAAAAAAAAAD///////////////8=', - 'extra.ecdsa_public_key_x' : 'NIQdotpy2HAHfSu0DjEmA3cb3NeQKabZaFX9OU0GsPY=', - 'extra.ecdsa_public_key_y' : '0fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=', - 'extra.selected_cipher' : 'aes128-ctr', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'curve25519-sha256@libssh.org', - 'extra.selected_mac' : 'hmac-sha2-256', - 'extra.server_cookie' : 'bGjsifbPIDWT7tAu8BMjyg==', - 'extra.server_host_key' : 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDSEHaLacthwB30rtA4xJgN3G9zXkCmm2WhV/TlNBrD20fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=', - 'extra.server_host_key_sha256' : 'a6e4e1c16ba25d51bcddc58a6e16797144575dd18d02d9dedf75093d2b15c557', - 'extra.server_signature_raw' : 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAABKAAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=', - 'extra.server_signature_value' : 'AAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=', - 'extra.serverid_raw' : 'SSH-2.0-OpenSSH_7.4', - 'extra.serverid_software' : 'OpenSSH_7.4', - 'extra.serverid_version' : '2.0', - 'extra.source.naics' : 454110, - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 16509, - 'source.geolocation.cc' : 'JP', - 'source.geolocation.city' : 'TOKYO', - 'source.geolocation.region' : 'TOKYO', - 'source.ip' : '18.179.0.0', - 'source.port' : 22, - 'source.reverse_dns' : 'ec2-18-179-0-0.ap-northeast-1.compute.amazonaws.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ssh-rsa', - 'extra.available_ciphers' : 'aes128-cbc, 3des-cbc, aes256-cbc, twofish256-cbc, twofish-cbc, twofish128-cbc, blowfish-cbc', - 'extra.available_compression' : 'none', - 'extra.available_kex' : 'diffie-hellman-group1-sha1', - 'extra.available_mac' : 'hmac-sha1-96, hmac-sha1, hmac-md5', - 'extra.device_vendor' : 'Arris', - 'extra.rsa_exponent' : '65537', - 'extra.rsa_length' : '1040', - 'extra.rsa_modulus' : 'g6tZBqqsHUFa8gjXKMg2waZy/6JLzNnfTZkdDo6a3E6Amdk2eVlGbdgRsDxNUCyWLJI4b+xsJX0OKoRKhw4N+cDK5Du4/S00Auylt8YrDi7lIpeBZjMT5KJerN/feOcwRqY8NJfwZiZ1A5brcWr5HXfOH6aRTMFzadr248VdUmFXPQ==', - 'extra.selected_cipher' : 'aes128-cbc', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'diffie-hellman-group1-sha1', - 'extra.selected_mac' : 'hmac-sha1', - 'extra.server_cookie' : 'Y4RQS9sdRgEFwNJKVP6bZg==', - 'extra.server_host_key' : 'AAAAB3NzaC1yc2EAAAADAQABAAAAgwCDq1kGqqwdQVryCNcoyDbBpnL/okvM2d9NmR0OjprcToCZ2TZ5WUZt2BGwPE1QLJYskjhv7GwlfQ4qhEqHDg35wMrkO7j9LTQC7KW3xisOLuUil4FmMxPkol6s39945zBGpjw0l/BmJnUDlutxavkdd84fppFMwXNp2vbjxV1SYVc9', - 'extra.server_host_key_sha256' : 'd53fedbfe92e631264629882b2e85bfd213ca4b07b824cd31f8de1fcb8d0ddcb', - 'extra.server_signature_raw' : 'AAAAB3NzaC1yc2EAAACCLQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==', - 'extra.server_signature_value' : 'LQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==', - 'extra.serverid_raw' : 'SSH-2.0-ARRIS_0.50', - 'extra.serverid_software' : 'ARRIS_0.50', - 'extra.serverid_version' : '2.0', - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey, password', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 11976, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MARSHALL', - 'source.geolocation.region' : 'TEXAS', - 'source.ip' : '170.10.0.0', - 'source.port' : 22, - 'source.reverse_dns' : '170-10-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ssh-rsa', - 'extra.available_ciphers' : 'aes128-cbc, 3des-cbc, aes192-cbc, aes256-cbc', - 'extra.available_compression' : 'none', - 'extra.available_kex' : 'diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1', - 'extra.available_mac' : 'hmac-sha1, hmac-sha1-96, hmac-md5, hmac-md5-96', - 'extra.device_sector' : 'enterprise', - 'extra.device_vendor' : 'Cisco', - 'extra.rsa_exponent' : '65537', - 'extra.rsa_length' : '4096', - 'extra.rsa_modulus' : 'yFVwcChoYt+YGm8BzWYugcZbNQRrQ1VWRYcL4U6SSkyoVeE9h5wxRu/hQaWHo3PdsB9Nuln/riRyKZypFUEZ5zlffMyl1uvE8/jp8E/GgUHSyPkGAwu8C8BkX/nDolxAJKTK6djiZnvhsEPe6AXHBMHbto/b3GABUNPngjzX8D63GYcFW9NJLf5qC1UsVkXbAzM0IjQ2X9s3pfhUCAJeXAn2i0gEGtUyF8vEjNdwdG655aXciKrpEEtM1L/zy/+gLH4YC13kAYI7NVyH+qi/mXbULLOQClA7iYK1g3Et58jWUIPwgLfF3SLC57bt2wp/lRgNTv4FBi0tWvRqBnf5UQK5ZjgzbW3bO+Ju4cWgH/4M4NCxSceh4cLm5lQs01xB5feSh2ByqA7wrVDoFJu81LoMVo4bCz30+lH2QsLwmNtUhlWLKBD4k09g4bgBa4jPj0/Nya3rBR4GQ6LG6ltFQotm8wCkgbv76YWqk20nQ6NMYZFvSQm981JFtoHv3vxq48VeHDV0QvV0P12BCFprRf4B0otIvSsHl+LDeUxJAf+Nbw78gzncjyfCbWtCPbwaJQ8CeqnTBzj5TluaFvN8goG5lCTWJGfjIrwAZXOokv9NOqmIiMJJx3s22OX6GHfJAzje2ALLDsAiXBub4iCOdGdTfVbBpFL+bGTK9qfa8vE=', - 'extra.selected_cipher' : 'aes128-cbc', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'diffie-hellman-group14-sha1', - 'extra.selected_mac' : 'hmac-sha1', - 'extra.server_cookie' : 'Z2fOfWsrLlh76Y0bOqa1cw==', - 'extra.server_host_key' : 'AAAAB3NzaC1yc2EAAAADAQABAAACAQDIVXBwKGhi35gabwHNZi6Bxls1BGtDVVZFhwvhTpJKTKhV4T2HnDFG7+FBpYejc92wH026Wf+uJHIpnKkVQRnnOV98zKXW68Tz+OnwT8aBQdLI+QYDC7wLwGRf+cOiXEAkpMrp2OJme+GwQ97oBccEwdu2j9vcYAFQ0+eCPNfwPrcZhwVb00kt/moLVSxWRdsDMzQiNDZf2zel+FQIAl5cCfaLSAQa1TIXy8SM13B0brnlpdyIqukQS0zUv/PL/6AsfhgLXeQBgjs1XIf6qL+ZdtQss5AKUDuJgrWDcS3nyNZQg/CAt8XdIsLntu3bCn+VGA1O/gUGLS1a9GoGd/lRArlmODNtbds74m7hxaAf/gzg0LFJx6HhwubmVCzTXEHl95KHYHKoDvCtUOgUm7zUugxWjhsLPfT6UfZCwvCY21SGVYsoEPiTT2DhuAFriM+PT83JresFHgZDosbqW0VCi2bzAKSBu/vphaqTbSdDo0xhkW9JCb3zUkW2ge/e/GrjxV4cNXRC9XQ/XYEIWmtF/gHSi0i9KweX4sN5TEkB/41vDvyDOdyPJ8Jta0I9vBolDwJ6qdMHOPlOW5oW83yCgbmUJNYkZ+MivABlc6iS/006qYiIwknHezbY5foYd8kDON7YAssOwCJcG5viII50Z1N9VsGkUv5sZMr2p9ry8Q==', - 'extra.server_host_key_sha256' : '06ff3cce443ed832927576d982b69d5a526d0e63334c72e87201deda61679406', - 'extra.server_signature_raw' : 'AAAAB3NzaC1yc2EAAAIAlrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=', - 'extra.server_signature_value' : 'lrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=', - 'extra.serverid_raw' : 'SSH-1.99-Cisco-1.25', - 'extra.serverid_software' : 'Cisco-1.25', - 'extra.serverid_version' : '1.99', - 'extra.source.naics' : 517311, - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey, keyboard-interactive, password', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 33363, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ORLANDO', - 'source.geolocation.region' : 'FLORIDA', - 'source.ip' : '72.17.0.0', - 'source.port' : 22, - 'source.reverse_dns' : '072-017-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py deleted file mode 100644 index f96c03e56..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py +++ /dev/null @@ -1,218 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssl.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SSL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ssl-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: unknown error', - 'extra.browser_trusted' : False, - 'extra.cert_expiration_date' : '2038-01-19 03:14:07', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2014-06-23 09:56:32', - 'extra.cert_length' : 1024, - 'extra.cert_serial_number' : '168CAE', - 'extra.cert_valid' : True, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.content_length' : 131, - 'extra.content_type' : 'text/html', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : 'support', - 'extra.issuer_country' : 'US', - 'extra.issuer_email_address' : 'support@fortinet.com', - 'extra.issuer_locality_name' : 'Sunnyvale', - 'extra.issuer_organization_name' : 'Fortinet', - 'extra.issuer_organization_unit_name' : 'Certificate Authority', - 'extra.issuer_state_or_province_name' : 'California', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '99:45:1F:2E:AE:EB:88:91:27:43:33:79:FA:93:7D:CA', - 'extra.self_signed' : False, - 'extra.server_type' : 'xxxxxxxx-xxxxx', - 'extra.sha1_fingerprint' : '5A:3D:FF:06:F9:E9:25:37:57:F9:09:52:33:A4:85:15:24:2D:88:7F', - 'extra.sha256_fingerprint' : '35:AB:B6:76:2A:3D:17:B2:FB:40:45:1B:FC:0A:99:0A:6E:48:57:F7:30:0A:3B:B1:1A:E6:99:70:5B:7C:32:41', - 'extra.sha512_fingerprint' : '88:7B:16:DB:39:44:0C:47:0E:4A:8F:0B:C5:FB:4D:45:BC:93:5A:00:43:A1:D9:7F:05:1D:86:33:02:F8:FC:57:67:A6:1D:C0:FF:F7:D2:40:D8:9A:21:AE:4E:6D:DC:E7:FF:72:BF:13:CB:EE:A7:5F:CD:83:EA:8A:5E:FB:87:DD', - 'extra.signature_algorithm' : 'sha1WithRSAEncryption', - 'extra.source.naics' : 517311, - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'FGT60D4614030700', - 'extra.subject_country' : 'US', - 'extra.subject_email_address' : 'support@fortinet.com', - 'extra.subject_locality_name' : 'Sunnyvale', - 'extra.subject_organization_name' : 'Fortinet', - 'extra.subject_organization_unit_name' : 'FortiGate', - 'extra.subject_state_or_province_name' : 'California', - 'extra.tag' : 'ssl,vpn', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 4181, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MILWAUKEE', - 'source.geolocation.region' : 'WISCONSIN', - 'source.ip' : '96.60.0.0', - 'source.port' : 10443, - 'source.reverse_dns' : '96-60-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: unknown error', - 'extra.browser_trusted' : False, - 'extra.cert_expiration_date' : '2023-02-06 01:01:34', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2022-01-04 01:01:34', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '36974C4C6B1B3785', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.content_type' : 'text/html; charset=UTF-8', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_connection' : 'keep-alive', - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : '1078-btb-tbi-HungHa-61d39c6d5a7e2', - 'extra.issuer_organization_name' : 'pfSense webConfigurator Self-Signed Certificate', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '16:93:9A:F4:35:7F:9A:85:45:71:91:C7:7C:80:88:00', - 'extra.self_signed' : True, - 'extra.server_type' : 'nginx', - 'extra.set_cookie' : 'PHPSESSID=e15bdfa5739c36877608eb4cf46cc388; path=/; secure; HttpO', - 'extra.sha1_fingerprint' : 'A9:00:BB:E1:54:4D:56:54:59:F1:B7:EA:F1:1A:D5:36:5C:63:90:8E', - 'extra.sha256_fingerprint' : '38:85:F0:44:1E:AD:84:B8:2F:43:68:BA:AC:EE:17:13:A4:BF:86:1D:48:75:7E:22:FA:08:4C:28:5F:AC:3E:5F', - 'extra.sha512_fingerprint' : 'AE:1B:4F:D1:E4:C0:35:9D:2A:4F:7A:37:B8:7B:11:9D:84:25:23:21:AB:EF:B2:0F:DC:C9:F2:A3:72:28:92:E1:74:72:FA:E1:09:6C:E1:F6:B6:E3:A7:61:1C:58:89:34:D7:06:5C:3D:0A:A7:F6:CC:8A:D6:24:D0:04:4C:03:02', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 517311, - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : '1078-btb-tbi-HungHa-61d39c6d5a7e2', - 'extra.subject_organization_name' : 'pfSense webConfigurator Self-Signed Certificate', - 'extra.tag' : 'ssl', - 'extra.transfer_encoding' : 'chunked', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 45899, - 'source.geolocation.cc' : 'VN', - 'source.geolocation.city' : 'THAI BINH', - 'source.geolocation.region' : 'THAI BINH', - 'source.ip' : '113.160.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_trusted' : True, - 'extra.cert_expiration_date' : '2022-11-06 15:30:28', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2021-10-07 15:30:28', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '7B388364A24B88E77E5553B5C6748100', - 'extra.cert_valid' : True, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.content_length' : 131, - 'extra.content_type' : 'text/html', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : 'Entrust Certification Authority - L1K', - 'extra.issuer_country' : 'US', - 'extra.issuer_organization_name' : 'Entrust, Inc.', - 'extra.issuer_organization_unit_name' : '(c) 2012 Entrust, Inc. - for authorized use only', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'E7:34:BC:92:84:FA:39:DE:E1:46:6C:27:DA:5A:01:F4', - 'extra.self_signed' : False, - 'extra.server_type' : 'xxxxxxxx-xxxxx', - 'extra.sha1_fingerprint' : 'AD:19:B2:1C:CB:88:70:9B:DB:8E:7E:F5:65:50:13:D6:43:6C:BE:6E', - 'extra.sha256_fingerprint' : '9A:64:73:0B:8A:FA:DE:22:D4:6D:5A:C6:C4:6F:D4:A4:2A:28:FA:41:1E:FF:81:DC:D4:D9:00:FD:78:DF:C4:DD', - 'extra.sha512_fingerprint' : '9A:B7:BD:68:7D:F3:E7:C1:B7:D3:F4:2F:01:B6:C4:77:90:A3:2B:1E:C0:89:F5:08:EC:43:87:35:60:36:D4:87:61:AA:B8:A8:B3:8A:E9:F1:04:AA:5B:67:12:FF:63:D5:14:80:77:6E:8F:7D:C3:E2:3A:F3:13:DF:08:43:6C:B0', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 454110, - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_country' : 'US', - 'extra.subject_locality_name' : 'Hanover', - 'extra.subject_organization_name' : 'Ciena Corporation', - 'extra.subject_state_or_province_name' : 'Maryland', - 'extra.tag' : 'ssl,vpn', - 'extra.validation_level' : 'OV', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 14618, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '34.224.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py deleted file mode 100644 index 42221bda2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py +++ /dev/null @@ -1,136 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssl_freak.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SSL FREAK Vulnerable Servers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssl_freak-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'SSL FREAK Vulnerable Servers', - 'protocol.application': 'https', - "classification.identifier": "ssl-freak", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.browser_error": "x509: unknown error", - "extra.browser_trusted": False, - "extra.cert_expiration_date": "2032-05-05 00:01:19", - "extra.cert_expired": False, - "extra.cert_issue_date": "2012-05-10 00:01:19", - "extra.cert_length": 1024, - "extra.cert_serial_number": "4FAB054F", - "extra.cert_valid": True, - "extra.cipher_suite": "TLS_RSA_WITH_RC4_128_SHA", - "extra.content_type": "text/html", - "extra.freak_cipher_suite": "TLS_RSA_EXPORT_WITH_RC4_40_MD5", - "extra.freak_vulnerable": True, - "extra.handshake": "TLSv1.0", - "extra.http_code": 200, - "extra.http_date": "2018-04-23T13:25:26+00:00", - "extra.http_reason": "OK", - "extra.http_response_type": "HTTP/1.1", - "extra.issuer_common_name": "usg50_B0B2DC2FA69D", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE", - "extra.self_signed": True, - "extra.sha1_fingerprint": "14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2", - "extra.sha256_fingerprint": "57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1", - "extra.sha512_fingerprint": "E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.subject_common_name": "usg50_B0B2DC2FA69D", - "extra.tag": "ssl-freak", - "extra.transfer_encoding": "chunked", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 8447, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.232", - "source.port": 443, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2018-04-23T13:25:21+00:00" - }, - {'__type': 'Event', - 'feed.name': 'SSL FREAK Vulnerable Servers', - 'protocol.application': 'https', - "classification.identifier": "ssl-freak", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.browser_error": "x509: unknown error", - "extra.browser_trusted": False, - "extra.cert_expiration_date": "2029-12-27 00:00:53", - "extra.cert_expired": False, - "extra.cert_issue_date": "2010-01-01 00:00:53", - "extra.cert_length": 1024, - "extra.cert_serial_number": "4B3D3B35", - "extra.cert_valid": True, - "extra.cipher_suite": "TLS_RSA_WITH_RC4_128_SHA", - "extra.content_type": "text/html", - "extra.freak_cipher_suite": "TLS_RSA_EXPORT_WITH_RC4_40_MD5", - "extra.freak_vulnerable": True, - "extra.handshake": "TLSv1.0", - "extra.http_code": 200, - "extra.http_date": "2018-04-23T13:25:29+00:00", - "extra.http_reason": "OK", - "extra.http_response_type": "HTTP/1.1", - "extra.issuer_common_name": "usg20w_C86C870287EC", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE", - "extra.self_signed": True, - "extra.sha1_fingerprint": "14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2", - "extra.sha256_fingerprint": "57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1", - "extra.sha512_fingerprint": "E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.subject_common_name": "usg20w_C86C870287EC", - "extra.tag": "ssl-freak", - "extra.transfer_encoding": "chunked", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 12577, - "source.geolocation.cc": "AT", - "source.geolocation.city": "BADEN", - "source.geolocation.region": "NIEDEROSTERREICH", - "source.ip": "198.51.100.224", - "source.port": 443, - "source.reverse_dns": "198-51-100-224.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2018-04-23T13:25:26+00:00" - }] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py deleted file mode 100644 index 41535e67a..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ssl_poodle.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SSL POODLE Vulnerable Servers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssl_poodle-test-geo.csv", - } -EVENTS = [{'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'ssl-poodle', - 'extra.browser_error': 'x509: unknown error', - 'extra.browser_trusted': False, - 'extra.cert_expiration_date': '2034-06-20 00:00:42', - 'extra.cert_expired': False, - 'extra.cert_issue_date': '2014-06-25 00:00:42', - 'extra.cert_length': 1024, - 'extra.cert_serial_number': '53AA112A', - 'extra.cert_valid': True, - 'extra.cipher_suite': 'TLS_RSA_WITH_RC4_128_SHA', - 'extra.content_type': 'text/html', - 'extra.handshake': 'TLSv1.0', - 'extra.http_code': 200, - 'extra.http_date': '2018-08-08T00:51:44+00:00', - 'extra.http_reason': 'OK', - 'extra.http_response_type': 'HTTP/1.1', - 'extra.issuer_common_name': 'usg20_107BEF394BA5', - 'extra.key_algorithm': 'rsaEncryption', - 'extra.md5_fingerprint': '33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC', - 'extra.self_signed': True, - 'extra.sha1_fingerprint': '04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3', - 'extra.sha256_fingerprint': '16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E', - 'extra.sha512_fingerprint': '0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE', - 'extra.signature_algorithm': 'sha1WithRSAEncryption', - 'extra.ssl_poodle': True, - 'extra.ssl_version': 2, - 'extra.subject_common_name': 'usg20_107BEF394BA5', - 'extra.tag': 'ssl-poodle', - 'extra.transfer_encoding': 'chunked', - 'feed.name': 'SSL POODLE Vulnerable Servers', - 'protocol.application': 'https', - 'source.asn': 65540, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '203.0.113.85', - 'source.port': 8443, - 'source.reverse_dns': 'example.com', - 'time.source': '2018-08-08T00:51:42+00:00', - "time.observation": "2015-01-01T00:00:00+00:00", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - '__type': 'Event', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py deleted file mode 100644 index 7fd5f6ec2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_stun.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_stun-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 5.4, - 'extra.fingerprint': '0xfaedd06e', - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '192.168.0.1', - 'extra.mapped_family': '01', - 'extra.mapped_port': 3243, - 'extra.message_length': 88, - 'extra.message_type': '0101', - 'extra.response_size': 108, - 'extra.software': "Coturn-4.5.1.1 'dan Eider'", - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '192.168.0.1', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 3243, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3478, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 5.4, - 'extra.fingerprint': '0x21128641', - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '51.77.39.195', - 'extra.mapped_family': '01', - 'extra.mapped_port': 45877, - 'extra.message_length': 88, - 'extra.message_type': '0101', - 'extra.response_size': 108, - 'extra.software': "Coturn-4.5.1.1 'dan Eider'", - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '192.168.0.2', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 45877, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3478, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 4.8, - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '192.168.0.3', - 'extra.mapped_family': '01', - 'extra.mapped_port': 16321, - 'extra.message_length': 76, - 'extra.message_type': '0101', - 'extra.response_size': 96, - 'extra.software': "ApolloProxy-1.20.1.28 'sunflower'", - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '188.68.240.32', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 16321, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3478, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py deleted file mode 100644 index 9b7e1fd3d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_synfulknock.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SYNful Knock', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_synfulknock-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '3cfdfec601e4700f6a9a2000080045000034c3780000f706789442099555b869f7ee0050b20800000000000c123e8012200002aa0000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 18885, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'JERSEY CITY', - 'source.geolocation.region' : 'NEW JERSEY', - 'source.ip' : '66.9.0.0', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:18:23+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '90e2baaf0b84700f6a9a200008004500003434100000f2064382d58337d2b8698b720050916200000000000c123e8012200059d50000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 35805, - 'source.geolocation.cc' : 'GE', - 'source.geolocation.city' : 'TBILISI', - 'source.geolocation.region' : 'TBILISI', - 'source.ip' : '213.131.0.0', - 'source.port' : 80, - 'source.reverse_dns' : 'host-213-131-55-210-customer.wanex.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:19:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '90e2bab9cfd4700f6a9a20000800450000340f1d0000ea068bdad5b2e6914a522f360050eb5200000000000c123e801220001b4a0000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 29256, - 'source.geolocation.cc' : 'SY', - 'source.geolocation.city' : 'DAMASCUS', - 'source.geolocation.region' : 'DIMASHQ', - 'source.ip' : '213.178.0.0', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:27:39+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py deleted file mode 100644 index 66408db4c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py +++ /dev/null @@ -1,87 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_telnet.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Telnet', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_telnet-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible Telnet', - "classification.identifier": "open-telnet", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "|MikroTik v6.5|Login:", - "extra.tag": "telnet-alt", - "protocol.application": "telnet", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.145", - "source.port": 5678, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T12:27:34+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible Telnet', - "classification.identifier": "open-telnet", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "|MikroTik v6.45.3 (stable)|Login:", - "extra.tag": "telnet-alt", - "protocol.application": "telnet", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.145", - "source.port": 5678, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T12:27:40+00:00" - }] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py deleted file mode 100644 index 3cf3688f9..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_tftp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open TFTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_tftp-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.57, - 'extra.error': 'Not defined', - 'extra.errormessage': 'Get not supported', - 'extra.opcode': '5', - 'extra.size': 22, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 35067, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.36, - 'extra.error': 'File not found', - 'extra.errorcode': '1', - 'extra.errormessage': 'File not found', - 'extra.opcode': '5', - 'extra.size': 19, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 56709, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.5, - 'extra.error': 'Access violation', - 'extra.errorcode': '2', - 'extra.errormessage': 'Access violation', - 'extra.opcode': '5', - 'extra.size': 21, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 32785, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py deleted file mode 100644 index 396bff1e3..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ubiquiti.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Ubiquiti', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-03-04T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ubiquiti-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 37.0, - 'extra.essid': 'Kachine-Meta-Lidia-Tereixa', - 'extra.firmwarerev': 'XS5.ar2313.v3.5.4494.091109.1459', - 'extra.mac_address': '00156db98c3a', - 'extra.model': 'NS5', - 'extra.radio_name': 'kachine.meta.lidia.tereixa', - 'extra.response_size': 148, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 10001, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 39.0, - 'extra.essid': 'Adana-Mason-Lanikai-Ozaner', - 'extra.firmwarerev': 'XM.ar7240.v5.6.3.28591.151130.1749', - 'extra.mac_address': '00156d7c9188', - 'extra.model': 'LM5', - 'extra.model_full': 'NanoStation Loco M5', - 'extra.radio_name': 'adana.mason.lanikai.ozaner', - 'extra.response_size': 156, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 10001, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 36.25, - 'extra.essid': 'Tailynn-Kadija-Noreen-Dinkar', - 'extra.firmwarerev': 'XW.ar934x.v5.6.5.29033.160515.2108', - 'extra.mac_address': '0418d6000fd5', - 'extra.model': 'P2B-400', - 'extra.model_full': 'PowerBeam M2 400', - 'extra.radio_name': 'tailynn.kadija.noreen.dinkar', - 'extra.response_size': 145, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 10001, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py deleted file mode 100644 index 457ec4425..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_vnc.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible VNC', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_vnc-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible VNC', - "classification.identifier": "open-vnc", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "RFB 003.889", - "extra.product": "Apple remote desktop vnc", - "protocol.application": "vnc", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.53", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T14:51:44+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible VNC', - "classification.identifier": "open-vnc", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "RFB 005.000", - "extra.naics": 517311, - "extra.product": "RealVNC Enterprise v5.3 or later", - "protocol.application": "vnc", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.112", - "source.port": 5678, - "source.reverse_dns": "localhost.localdomain", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T14:51:44+00:00"}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py deleted file mode 100644 index 41ab55e58..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ws_discovery.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-WS-Discovery-Service', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_ws_discovery-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 164.83, - 'extra.error': 'Validation constraint violation: SOAP message expected', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 989, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3702, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 183.6, - 'extra.error': 'Validation constraint violation: missing root element', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 918, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3702, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 197.8, - 'extra.error': 'Validation constraint violation: SOAP message expected', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 989, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3702, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py deleted file mode 100644 index d17482e71..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_xdmcp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer XDMCP", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_xdmcp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.29, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node01.example.com', - 'extra.size': 44, - 'extra.status': 'Linux 3.0.101-100-default', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 177, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.86, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node02.example.com', - 'extra.size': 48, - 'extra.status': 'Linux 2.6.9-103.ELsmp', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 47074, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.57, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node03.example.com', - 'extra.size': 46, - 'extra.status': '1 user, load: 6,5, 6,6, 6,6', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 177, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_special.py b/intelmq/tests/bots/parsers/shadowserver/test_special.py deleted file mode 100644 index abad86cac..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_special.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/special.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Special', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-special-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Professional, Scientific, and Technical Services', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_testdata.py b/intelmq/tests/bots/parsers/shadowserver/test_testdata.py deleted file mode 100644 index 19cbdd7d7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_testdata.py +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-FileCopyrightText: 2017 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import csv -import json -import os -import os.path -import unittest -import pathlib - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot -from intelmq.bots.parsers.shadowserver.parser_json import ShadowserverJSONParserBot - -def csvtojson(csvfile): - datalist = [] - - with open(csvfile) as fop: - reader = csv.DictReader(fop, restval="") - - for row in reader: - datalist.append(row) - - return json.dumps(datalist, indent=4) - -CSVREPORTS = {} -JSONREPORTS = {} -testdata = pathlib.Path(__file__).parent / 'testdata' -for filename in testdata.glob('*.csv'): - EXAMPLE_FILE = filename.read_text() - shortname = filename.stem - CSVREPORTS[shortname] = {"raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": f"2019-01-01-{shortname}-test-test.csv", - } - JSONREPORTS[shortname] = {"raw": utils.base64_encode(csvtojson(filename)), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": f"2019-01-01-{shortname}-test-test.json", - } - - -def generate_feed_function(feedname, reports): - def test_feed(self): - """ Test if no errors happen for feed %s. """ % feedname - self.input_message = reports[feedname] - self.run_bot() - return test_feed - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - -class TestShadowserverJSONParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverJSONParserBot - -for key in CSVREPORTS: - setattr(TestShadowserverParserBot, 'test_feed_%s' % key, generate_feed_function(key, CSVREPORTS)) -for key in JSONREPORTS: - setattr(TestShadowserverJSONParserBot, 'test_feed_%s' % key, generate_feed_function(key, JSONREPORTS)) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv deleted file mode 100644 index cfadcbb2d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","hostname","source","reason","asn","geo","region","city","naics","sic","sector","tag" -"2019-09-04 07:00:19","198.123.245.134",host.local,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,0, -"2019-09-04 07:00:19","198.123.245.171",,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,, -"2019-09-04 07:00:19","198.123.245.0/24",,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license deleted file mode 100644 index 476908eeb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license deleted file mode 100644 index 456b03316..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv deleted file mode 100644 index 117dd6560..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","application","asn","geo","region","city","url","http_host","category","system","detected_since","server","redirect_target","naics","sic","sector","cc_url","family" -"2017-01-16 00:43:48","203.0.113.1",80,"example.com","hacked-webserver-stealrat-t1","http",64496,"AT","WIEN","VIENNA","/header.php","example.com","spam","WINNT","2015-05-09 05:51:12","Microsoft-IIS/7.5",,0,0,,, -"2018-04-09 15:43:41","203.0.113.1","80","","phishing","http","64496","AT","STEIERMARK","GRAZ","/","example.com","phishing","","","","","0","0","",, -"2022-02-07 21:52:29","66.249.0.0",,"66-249-0-0.example.com","magecart",,1234,"US","CALIFORNIA","MOUNTAIN VIEW",,,"stealer",,,,,519130,,"Communications, Service Provider, and Hosting Service","https://lolfree.pw/ads.txt", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv deleted file mode 100644 index 22cfdd69e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model" -"2022-01-10 00:01:42","88.84.0.0","tcp",10443,,"ssl,vpn",2116,"NO","TROMS OG FINNMARK","TROMVIK",517311,,,"Fortinet","firewall","FortiGate" -"2022-01-10 00:01:42","170.231.0.0","tcp",10443,,"ssl,vpn",27843,"PE","METROPOLITANA DE LIMA","LIMA",,,,"Fortinet","firewall","FortiGate" -"2022-01-10 00:01:42","96.60.0.0","tcp",10443,"96-60-66-218.example.com","ssl,vpn",4181,"US","WISCONSIN","MILWAUKEE",517311,,,"Fortinet","firewall","FortiGate" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv deleted file mode 100644 index 3114c26b1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized","http_agent" -"2010-02-10 00:00:00",tcp,192.168.0.1,38055,64512,ZZ,Region,City,node01.example.com,0,,,,,172.16.0.1,443,65534,ZZ,Region,City,node01.example.net,0,"",,,ddos-participant,,,https,,,,,,,,,www.example.com,,,GET,/??=GovpfOoaWYlk,,,,,,,,,,,,,,,,,, -"2010-02-10 00:00:01",udp,192.168.0.2,53,64512,ZZ,Region,City,node02.example.com,0,,,,,172.16.0.2,53,65534,ZZ,Region,City,node02.example.net,0,"",,,ddos-participant,,,dns,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -"2010-02-10 00:00:02",udp,192.168.0.3,53,64512,ZZ,Region,City,node03.example.com,0,,Microsoft,email,Exchange,172.16.0.3,53,65534,ZZ,Region,City,node03.example.net,0,"",,,ddos-participant,,,dns,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license deleted file mode 100644 index 9f58c89ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv deleted file mode 100644 index 17ff15ee6..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","service","start_time","end_time","client_version","username","password","payload_url","payload_md5" -"2021-03-27 00:00:00","tcp","141.98.1.2",30123,209588,"NL","NOORD-HOLLAND","AMSTERDAM",,,,,,,"162.250.1.2",22,26832,"CA","QUEBEC","MONTREAL",,,,"CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.521730Z","2021-03-27T00:00:01.710968Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","5.188.3.4",55690,57172,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"162.250.3.4",22,26832,"CA","QUEBEC","MONTREAL",,,,"CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.520927Z","2021-03-27T00:00:01.670993Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","45.14.5.6",38636,44220,"RO","BIHOR","ORADEA",,,,,,,"82.118.5.6",23,204957,"PL","MAZOWIECKIE","WARSAW",,,,"CAPRICA-EU","telnet-brute-force",,,"telnet",,,,"2021-03-27T00:00:00.781774Z","2021-03-27T00:00:00.857244Z",,,,, -"2021-03-27 00:00:00","tcp","5.188.6.7",56385,49453,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"102.16.6.7",22,37054,"MG","ANTANANARIVO","ANTANANARIVO",,,"Communications, Service Provider, and Hosting Service","CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.163870Z","2021-03-27T00:00:02.896640Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","45.14.7.8",35802,44220,"RO","BIHOR","ORADEA",,,,,,,"82.118.7.8",23,204957,"PL","MAZOWIECKIE","WARSAW",,,,"CAPRICA-EU","telnet-brute-force",,,"telnet",,,,"2021-03-27T00:00:00.781272Z","2021-03-27T00:00:00.856606Z",,,,, -"2021-03-27 00:00:00","tcp","5.188.9.10",33289,49453,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"60.234.9.10",22,9790,"NZ","WELLINGTON","LOWER HUTT",,,"Communications, Service Provider, and Hosting Service","CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.044871Z","2021-03-27T00:00:00.077322Z","b'SSH-2.0-Go'",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license deleted file mode 100644 index 8b9580cf1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv deleted file mode 100644 index dc78c1c1a..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv +++ /dev/null @@ -1,9 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","count" -"2021-03-07 00:00:00","tcp","61.3.1.2",4717,9829,"IN","KERALA","CHENGANNUR",,518210,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","211.218.3.4",4405,4766,"KR","GANGWON-DO","PYEONGCHANG-EUP",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","45.225.5.6",59777,266915,"BR","BAHIA","VITORIA DA CONQUISTA","static-45-225-x-x.example.net",,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","125.122.7.8",8460,4134,"CN","ZHEJIANG SHENG","HANGZHOU",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","219.77.9.10",21867,4760,"HK","HONG KONG","HONG KONG","n219077092196.example.com",517311,,,,,,5555,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","24.137.11.12",4680,14638,"PR","PUERTO RICO","SAN JUAN","dynamic.libertypr.net",,,,,,,5555,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","119.182.13.14",13175,4837,"CN","SHANDONG SHENG","JINING",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","27.198.15.16",56133,4837,"CN","SHANDONG SHENG","JINAN",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv deleted file mode 100644 index f41cb508f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized","http_agent" -"2010-02-10 00:00:00",,192.168.0.1,61234,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.1,88,65534,ZZ,Region,City,node01.example.net,0,,,,ddos,mirai,mirai,mirai,,,121.12.110.28/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:01",,192.168.0.2,61234,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.2,80,65534,ZZ,Region,City,node02.example.net,0,,,,ddos,mirai,mirai,mirai,,,180.97.183.94/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:02",,192.168.0.3,6379,64512,ZZ,Region,City,node03.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.3,,65534,ZZ,Region,City,node03.example.net,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,104.237.138.135/32,32,atk7,10,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv deleted file mode 100644 index a7d0bc4f1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","request","count","bytes","end_time","duration","avg_pps","max_pps" -"2021-03-28 00:00:02",,"107.141.1.2",,7018,"US","CALIFORNIA","VISALIA","107-141-x-x.lightspeed.frsnca.sbcglobal.net",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,"2021-03-28 00:20:22",,, -"2021-03-28 00:00:02",,"74.59.3.4",,5769,"CA","QUEBEC","CHICOUTIMI","modemcablex-x-59-74.mc.videotron.ca",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,"2021-03-28 00:13:50",,, -"2021-03-28 00:00:02",,"65.131.5.6",,209,"US","WYOMING","CASPER","65-131-x-x.chyn.qwest.net",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, -"2021-03-28 00:00:02",,"104.162.7.8",,12271,"US","NEW YORK","KINGSTON","cpe-104-162-x-x.hvc.res.rr.com",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, -"2021-03-28 00:00:02",,"37.120.178.9.10",,197540,"DE","NIEDERSACHSEN","GIFHORN","v22020111328131649.ultrasrv.de",,,,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license deleted file mode 100644 index 8b9580cf1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv deleted file mode 100644 index 0e5b1e5e9..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","device_vendor","device_type","device_model","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized" -"2010-02-10 00:00:00",,172.16.0.1,80,65534,ZZ,Region,City,node01.example.net,0,,,,,192.168.0.1,61234,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,115.238.198.85/32,32,atk0,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:01",,172.16.0.2,43437,65534,ZZ,Region,City,node02.example.net,0,Information,,,,192.168.0.2,61234,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,52.184.50.250/32,32,atk0,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:02",,172.16.0.3,80,65534,ZZ,Region,City,node03.example.net,0,,,,,192.168.0.3,61234,64512,ZZ,Region,City,node03.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,211.99.102.216/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv deleted file mode 100644 index d9448bd83..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","pattern","http_url","http_agent","http_request_method","url_scheme","session_tags","vulnerability_enum","vulnerability_id","vulnerability_class","vulnerability_score","vulnerability_severity","vulnerability_version","threat_framework","threat_tactic_id","threat_technique_id","target_vendor","target_product","target_class","file_md5","file_sha256","request_raw","body_raw" -"2021-08-01 00:24:08","tcp","191.23.45.67",36455,1234,"EE","HARJUMAA","TALLINN","191-23-45-67-host.example.com",518210,"Communications, Service Provider, and Hosting Service",,,,"109.87.65.43",80,5678,"UK","WINDSOR AND MAIDENHEAD","MAIDENHEAD",,518210,,"CAPRICA-EU","http-scan",,,,"3.1.3-dev",,"unknown","/js/ueditor/wwwroot/way-board.cgi",,,,,,,,,,,,,,,,,,,"GET /js/ueditor/wwwroot/way-board.cgi HTTP/1.0rnAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8rnAccept-Encoding: gzip, deflaternAccept-Language: en-US,en;q=0.5rnConnection: closernDnt: 1rnHost: 109.87.65.43rnOrigin: http://109.87.65.43rnReferer: http://109.87.65.43/rnUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3084.400 QQBrowser/9.6.11346.400", -"2021-08-01 05:21:59","tcp","45.67.89.123",58610,12345,"EE","HARJUMAA","TALLINN",,,,,,,"82.41.20.10",8080,23456,"UA","KHARKIVS'KA OBLAST'","KHARKIV",,,,"CAPRICA-EU","http-scan",,,,,,,"/","Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1","GET","http",,,,,,,,,,,,,,,,"R0VUIC8gSFRUUC8xLjENCkhvc3Q6IDgyLjQxLjIwLjEwOjgwODANCkFjY2VwdDogdGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksKi8qO3E9MC44DQpBY2NlcHQtRW5jb2Rpbmc6IGRlZmxhdGUsIGd6aXAsIGlkZW50aXR5DQpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTO3E9MC42LGVuO3E9MC40DQpVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA1LjE7IHJ2OjkuMC4xKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzkuMC4xDQoNCg==", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license deleted file mode 100644 index c1900637f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv deleted file mode 100644 index 174360bbd..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","network","routedspoof","session","nat" -"2021-03-28 00:42:59","tcp","98.191.250.0",,22898,"US","OKLAHOMA","OKLAHOMA CITY","ip-98.191.250.0.atlinkservices.com",517311,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"98.191.250.0/24","received",1112907,"True" -"2021-03-28 01:36:22","tcp","191.7.16.0",,262485,"BR","RIO DE JANEIRO","NOVA IGUACU",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"191.7.16.0/24","received",1112914,"False" -"2021-03-28 02:10:58","tcp","202.53.160.0",,23923,"BD","DHAKA","DHAKA",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"202.53.160.0/24","received",1112931,"True" -"2021-03-28 03:41:51","tcp","87.121.75.0",,134697,"AU","QUEENSLAND","BRISBANE",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"87.121.75.0/24","received",1112953,"True" -"2021-03-28 06:07:17","tcp","189.201.194.0",,262944,"MX","COAHUILA","SALTILLO","ip-189-201-194-0.slw.spectro.mx",,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"189.201.194.0/24","received",1113015,"True" -"2021-03-28 06:59:53","tcp","197.15.48.0",,37671,"TN","TUNIS","TUNIS",,517311,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"197.15.48.0/24","received",1113035,"True" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv deleted file mode 100644 index eb0cbbab9..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id" -"2021-06-07 00:00:00","tcp","190.229.1.2",52955,7303,"AR","BUENOS AIRES","CASEROS",,517311,,,,,"168.63.134.179",16464,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","96.20.3.4",16464,5769,"CA","QUEBEC","LAVAL",,517311,"Communications, Service Provider, and Hosting Service",,,,"52.169.3.4",16464,8075,"IE","DUBLIN","DUBLIN",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","187.222.5.6",55049,8151,"MX","CIUDAD DE MEXICO","MEXICO CITY",,517311,,,,,"168.63.134.179",16464,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","75.84.7.8",64190,20001,"US","CALIFORNIA","NORTH HOLLYWOOD",,517311,"Communications, Service Provider, and Hosting Service",,,,"52.169.7.8",16464,8075,"IE","DUBLIN","DUBLIN",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","24.15.9.10",60373,7922,"US","ILLINOIS","HOMER GLEN",,517311,"Communications, Service Provider, and Hosting Service",,,,"104.40.6.5",16464,8075,"US","CALIFORNIA","SAN FRANCISCO",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","124.101.11.12",50386,4713,"JP","FUKUOKA","FUKUOKA",,517311,"Communications, Service Provider, and Hosting Service",,,,"23.99.101.165",16465,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-64bit","zeroaccess","b68-zeroaccess-2-64bit",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv deleted file mode 100644 index c56d1f218..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2021-06-07 00:00:00","tcp","31.206.1.2",49245,8386,"TR","ANTALYA","KEPEZ",,,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, -"2021-06-07 00:00:00","tcp","177.140.3.4",35919,28573,"BR","SAO PAULO","SAO PAULO",,517312,,,,,"204.95.99.204",443,8075,"US","WASHINGTON","REDMOND",,334111,"Information","MSDCU","caphaw","caphaw","caphaw",,,,"/index.php","3fo8jrthz3y.rgk.cc","Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.6103)",,,"null" -"2021-06-07 00:00:01","tcp","180.190.5.6",49264,132199,"PH","CEBU","MANDAUE",,517311,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, -"2021-06-07 00:00:01","tcp","197.157.7.8",55307,37129,"KE","NAIROBI CITY","NAIROBI",,,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/news/stream.php","40.121.206.97",,,, -"2021-06-07 00:00:01","tcp","174.114.9.10",59000,812,"CA","ONTARIO","OTTAWA",,517311,"Communications, Service Provider, and Hosting Service",,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv deleted file mode 100644 index c5126c843..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id" -"2021-03-04 00:00:00","tcp","190.113.1.2",17409,12252,"PE","METROPOLITANA DE LIMA","LIMA",,,,,,,"178.162.1.2",4455,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service","eset","victorygate.b","victorygate.b",,,, -"2021-03-04 00:00:00","tcp","35.205.9.10",44696,15169,"BE","BRUXELLES-CAPITALE","BRUSSELS","x.x.205.35.bc.googleusercontent.com",519130,"Communications, Service Provider, and Hosting Service",,,,"148.81.9.10",80,1887,"PL","MAZOWIECKIE","WARSAW",,,,,"virut","virut",,,, -"2021-03-04 00:00:00","tcp","35.197.11.12",36968,15169,"US","OREGON","THE DALLES","x.x.197.35.bc.googleusercontent.com",519130,"Communications, Service Provider, and Hosting Service",,,,"148.81.111.11.12",80,1887,"PL","MAZOWIECKIE","WARSAW",,,,,"virut","virut",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv deleted file mode 100644 index 3e85690d8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","infection","family","tag","query_type","query","count" -"2022-01-06 00:00:02","udp","217.110.0.0",29614,8220,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,,"calypso","calypso","msexchange","A","YolkIsh.COM",1 -"2022-01-06 00:00:02","udp","209.66.0.0",46189,40934,"US","VIRGINIA","ASHBURN",,518210,,,,,"orcus","orcus","rat","A","verble.rocks",1 -"2022-01-06 00:00:02","udp","217.110.0.0",3590,8220,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,,"calypso","calypso","msexchange","A","RAwFuNS.COM",1 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license deleted file mode 100644 index 662bb20b7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv deleted file mode 100644 index 4514f248e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2021-03-04 00:00:00","tcp","103.196.1.2",60902,134707,"PH","NUEVA ECIJA","DEL PILAR",,,,,,,"184.105.1.2",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","5.14.3.4",55002,8708,"RO","CONSTANTA","CONSTANTA",,517311,"Communications, Service Provider, and Hosting Service",,,,"184.105.3.4",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","49.145.5.6",31350,9299,"PH","CEBU","CEBU",,517311,,,,,"184.105.5.6",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"disorderstatus.ru",,,, -"2021-03-04 00:00:00","tcp","200.44.7.8",28063,8048,"VE","CARABOBO","VALENCIA",,517311,,,,,"184.105.7.8",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","187.189.9.10",45335,17072,"MX","CHIHUAHUA","JUAREZ",,,,,,,"184.105.9.10",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv deleted file mode 100644 index 23a3cb2b6..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","http_referer_ip","http_referer_port","http_referer_asn","http_referer_geo","http_referer_region","http_referer_city","http_referer_hostname","http_referer_naics","http_referer_sector","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_referer" -"2021-03-04 00:00:02","tcp","178.162.203.211",80,28753,"DE","HESSEN","FRANKFURT AM MAIN","12106.mobapptrack.com",518210,"Communications, Service Provider, and Hosting Service","85.17.31.82",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"kovter","kovter",,,1614816002,"GET /favicon.ico HTTP/1.1","12106.mobapptrack.com","http://12106.mobapptrack.com/click/redirect?feed_id=12106&sub_id=7&q=8A5491983C8FBE7743E2D2C36E45EBC4-18307118D2626C9BD756B3F09D14BB910E381EE4" -"2021-03-04 00:00:11","tcp","59.106.1.2",80,9370,"JP","OSAKA","OSAKA","x.noizm.com",518210,"Communications, Service Provider, and Hosting Service","178.162.1.2",80,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816011,"GET /animalally.com HTTP/1.1","freescanonline.com","http://x.noizm.com/jump.php?u=http://freescanonline.com/animalally.com" -"2021-03-04 00:00:12","tcp","142.250.3.4",80,15169,"US","CALIFORNIA","MOUNTAIN VIEW","x.blogspot.com",519130,"Communications, Service Provider, and Hosting Service","178.162.1.2",80,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,"kovter","kovter",,,1614816012,"GET /getjs?r=0.6393021999392658 HTTP/1.1","rxrtb.bid","http://x.blogspot.com/" -"2021-03-04 00:00:13","tcp","34.232.5.6",80,14618,"US","VIRGINIA","ASHBURN","www.example.com",454110,"Retail Trade","5.79.71.225",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816013,"GET /personalationmall.com HTTP/1.1","freescanonline.com","http://www.example.com/teams/default.asp?u=EKL&t=c&s=lacrosse&p=remote&url=http://freescanonline.com/personalationmall.com" -"2021-03-04 00:01:26","tcp","210.172.7.8",80,2516,"JP","HOKKAIDO","SAPPORO","x.communes.jp",517312,"Communications, Service Provider, and Hosting Service","5.79.1.2",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816086,"GET /raftcomply.com HTTP/1.1","freescanonline.com","http://x.communes.jp/?url=http://freescanonline.com/raftcomply.com" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license deleted file mode 100644 index c1900637f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv deleted file mode 100644 index 016d2f912..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2022-03-02 09:14:19","tcp","2001:448a:1082:4d9b:7491:bf9e:3d5f:a634",49431,7713,"ID","JAKARTA RAYA","JAKARTA",,,,,,,"2001:470:1:332::fe",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM HTTP/1.1","devps.net","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)",,, -"2022-03-02 09:15:10","tcp","2001:448a:1082:4d9b:7491:bf9e:3d5f:a634",49460,7713,"ID","JAKARTA RAYA","JAKARTA",,,,,,,"2001:470:1:332::ef",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM HTTP/1.1","devps.net","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)",,, -"2022-03-02 14:15:10","tcp","2603:8080:b20a:dc00:f06e:8304:71f6:27e2",62932,11427,"US","TEXAS","GARLAND",,517311,"Communications, Service Provider, and Hosting Service",,,,"2001:470:1:332::fe",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /WMoUNCvuKGzdqSCeQcadP1%2f0B%2f3bzpOmyKBU85Z25HVOhvDQUPFl%2fk8uOcLewS%2b1BsuHXalRAOIgGOYYs2igj6UX8FkdCAmDewWPvfDhPD45nwd2tx1lLf2IoIfuOtIpGR6bN5Q6hGpSBgfERqCa0ImHcwfcZ2EdO%2fWvg7R8H6SLcTiuUC0I4pzvlWt1CRLgLdIEU1hZ0nnFHIHhchb6D7ITEgBQ2chQDxy5TJMrGjm4Dac6dKl%2ft5uYhRhSjAHkLLtgrJjsqVtVbelTAkt5kdcqLlO09m1SH%2fvtAb%2fOvR2DbhBss7%2f64DG7g6cAnghNA6JrFn1uW7sw%2bnKH8koKQwzUjdSsbrQAvmg4r0KDDW8Diq64gfDzxFWkzCLOYifc%2fwlinXPCl7aJiNCoieDC1U98RNQg%2f5td4SZmJnDQ2%2f96CPbFeSpCez5WD1rCjrxLj1h2cqzIgkydEWACceWP9ztxc4QaObzEcgOGxbRckWC7H2aaLeT8jaYEYdKi1pwEKChSL3YdEt4ZIb2IFrWwzNaXEpQzFXf07f902OEdI9vVA1ZdEOBPG6rAIkzMdebfprfVyhKEWtrCd3Skg3COUFtRQks5jzG1nv4sVGijTfSgyn6xE9Taka668Nycik6nmHy8Huj3oC01j3tee%2f1Z3eI6tV7lgM5d3uFJ84slRGHUCwMfVozOGmZRwNo%2fz%2bA HTTP/1.1","devps.net","Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license deleted file mode 100644 index 662bb20b7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv deleted file mode 100644 index ccafbab3f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","url","host","ip","asn","geo","region","city","naics","sector","tag","source","sha256","application" -"2022-01-07 00:02:07","http://41.86.0.0:50008/Mozi.m","41.86.0.0","41.86.0.0",37203,"LR","MONTSERRADO","MONROVIA",,,"CVE-2016-10372",,"12013662c71da69de977c04cd7021f13a70cf7bed4ca6c82acbc100464d4b0ef","http" -"2022-01-07 00:03:14","http://42.225.0.0:38173/Mozi.m","42.225.0.0","42.225.0.0",4837,"CN","HENAN SHENG","ZHUMADIAN",517311,,"CVE-2018-10562",,,"http" -"2022-01-07 00:10:26","http://211.52.0.0:53029/Mozi.m","211.52.0.0","211.52.0.0",4766,"KR","CHUNGCHEONGNAM-DO","SAGOK-MYEON",517311,,"CVE-2018-10562",,,"http" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv deleted file mode 100644 index 965d763a3..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","url","host","ip","asn","geo","region","city","naics","sector","source" -"2022-02-01 08:00:07","https://priceless-pare.example.net/Postal-/acec6/","priceless-pare.example.net","172.245.0.0",64512,"US","NEW YORK","BUFFALO",518210,"Communications, Service Provider, and Hosting Service","openphish.com" -"2022-02-01 08:00:07","https://mailyahooattt.example.net/","mailyahooattt.example.net","199.34.0.0",64512,"US","CALIFORNIA","SAN FRANCISCO",,"Professional, Scientific, and Technical Services","openphish.com" -"2022-02-01 08:00:07","https://www.example.net/viewer/vbid-730ec2b1-omsttuer","www.example.net","216.58.0.0",64512,"US","UTAH","DRAPER",519130,"Communications, Service Provider, and Hosting Service","openphish.com" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv deleted file mode 100644 index d5baa730f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","proxy_authenticate","via","server","content_length","transfer_encoding","http_date" -"2010-02-10 00:00:00",192.168.0.1,tcp,3128,node01.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"Squid proxy-caching web server\"\"",,squid/4.10,3741,,"Wed, 10 Feb 2010 00:00:00 GMT" -"2010-02-10 00:00:01",192.168.0.2,tcp,3128,node02.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"00:23:24:43:1c:34\"\"",,,3833,,"Wed, 10 Feb 2010 00:00:01 GMT" -"2010-02-10 00:00:02",192.168.0.3,tcp,3128,node03.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"Proxy\"\"",,,179,,"Wed, 10 Feb 2010 00:00:02 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv deleted file mode 100644 index 4710af974..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","asn","geo","md5","protocol","port","host","bytes_in","bytes_out" -"2022-01-10 00:00:03","40.119.6.228",8075,"US","b575ce6dcce6502a8431db5610135c25","udp",123,"time.windows.com",0,0 -"2022-01-10 00:00:03","8.252.70.126",3356,"US","c0d947f9a8685b0d9f3efdba966389c2","tcp",80,,0,0 -"2022-01-10 00:00:03","52.109.8.22",8075,"US","c0d947f9a8685b0d9f3efdba966389c2","tcp",443,,0,0 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv deleted file mode 100644 index 697cb6209..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","md5hash","request","type","response","family","tag","source" -"2022-01-10 00:00:02","b575ce6dcce6502a8431db5610135c25","time.windows.com","A","40.119.6.228",,, -"2022-01-10 00:00:08","807679198a39c80d3ca07e60fd51b581","time.windows.com","A","40.119.6.228",,, -"2022-01-10 00:00:20","d97e973b9bf073bd3a217425259cea26","client-office365-tas.msedge.net","A","13.107.5.88",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv deleted file mode 100644 index bbfe596a2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","asn","geo","md5","url","user_agent","host","method" -"2022-01-10 00:01:13","23.196.47.89",20940,"US","37514b54e679a5313334e830ad780ec7","http://www.msftncsi.com/ncsi.txt","Microsoft NCSI","www.msftncsi.com","GET" -"2022-01-10 00:01:28","72.21.81.240",15133,"US","37514b54e679a5313334e830ad780ec7","http://www.download.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab","Microsoft-CryptoAPI/6.1","www.download.windowsupdate.com","GET" -"2022-01-10 00:08:24","23.56.4.57",20940,"US","e97ea2820c0d79f3f3ca241d4dcd1060","http://crl.microsoft.com/pki/crl/products/MicTimStaPCA_2010-07-01.crl","Microsoft-CryptoAPI/6.1","crl.microsoft.com","GET" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv deleted file mode 100644 index c0ff0bdf1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","name","model","device","features","device_vendor","device_type","device_model","device_version","device_sector" -"2018-07-26 02:07:16","36.239.124.210","tcp",5555,"36-239-124-210.dynamic-ip.hinet.net","adb",3462,"TW","TAOYUAN COUNTY","TAOYUAN CITY",518210,737415,"hlteuc","SAMSUNG-SM-N900A","hlteatt",,,,,, -"2018-07-26 02:07:16","36.236.108.107","tcp",5555,"36-236-108-107.dynamic-ip.hinet.net","adb",3462,"TW","TAIPEI CITY","TAIPEI",518210,737415,"marlin","Pixel XL","marlin","cmd,shell_v2",,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv deleted file mode 100644 index c5494d458..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","machine_type","afp_versions","uams","flags","server_name","signature","directory_service","utf8_servername","network_address" -"2019-09-04 05:05:53","198.13.34.22","tcp",548,"host.local","afp",6057,"AA","LOCATION","LOCATION",517311,0,"TimeCapsule8,119","AFP3.3,AFP3.2,AFP3.1","DHCAST128,DHX2,SRP,Recon1","SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient","airport-time-capsule-de-jack","4338364e37364442463948350069672d",,"AirPort Time Capsule de jack","198.33.24.165:548,10.0.1.1:548,fe80:0008:0000:0000:6e70:9fff:fed4::548,fe80:0009:0000:0000:6e70:9fff:fed4::548,179.24.24.165 (DNS address)," -"2019-09-04 05:05:56","198.40.27.212","tcp",548,"host.local","afp",6057,"AA","LOCATION","LOCATION",517311,0,"TimeCapsule8,119","AFP3.3,AFP3.2,AFP3.1","DHCAST128,DHX2,SRP,Recon1","SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient","time-capsule-del-jack","433836544b303147463948360069672d",,"Time Capsule del Jack","0.0.0.1:548,10.0.1.1:548,198.33.42.1:548,fe80:000b:0000:0000:dea4:caff:feba::548,fe80:000c:0000:0000:dea4:caff:feba::548,fe80:000d:0000:0000:4c7d:ffff:fec7::548,0.0.0.1 (DNS address)," diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv deleted file mode 100644 index 92f078af7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","channel","message_length","class","method","version_major","version_minor","capabilities","cluster_name","platform","product","product_version","mechanisms","locales","sector" -"2022-01-10 04:32:13","47.103.0.0","tcp",5672,,"amqp",37963,"CN","SHANGHAI SHI","SHANGHAI",518210,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos","rabbit@iZuf63m0nnq9bwf7lhjxrkZ","Erlang/OTP","RabbitMQ","3.3.5","PLAIN AMQPLAIN","en_US", -"2022-01-10 04:32:13","141.95.0.0","tcp",5672,,"amqp",16276,"DE","SAARLAND","SAARBRUCKEN",518210,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to","rabbit@mtk-breizh","Erlang/OTP 24.0.3","RabbitMQ","3.8.19","AMQPLAIN PLAIN","en_US", -"2022-01-10 04:32:13","54.234.0.0","tcp",5672,"ec2-54.234.0.0.compute-1.amazonaws.com","amqp",14618,"US","VIRGINIA","ASHBURN",454110,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to","rabbit@1397a0e9629b","Erlang/OTP 24.2","RabbitMQ","3.9.11","PLAIN AMQPLAIN","en_US", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv deleted file mode 100644 index 9c43f8598..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","machine_name","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,3283,node01.example.com,ard,64512,ZZ,Region,City,0,0,"Macmini (radio)",1006,201.20 -"2010-02-10 00:00:01",192.168.0.2,udp,3283,node02.example.com,ard,64512,ZZ,Region,City,0,0,biuro-rip-org-pl,1006,201.20 -"2010-02-10 00:00:02",192.168.0.3,udp,3283,node03.example.com,ard,64512,ZZ,Region,City,0,0,127.0.0.1,1006,201.20 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv deleted file mode 100644 index 7bd2b20e0..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","size","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,19,node01.example.com,chargen,,64512,ZZ,Region,City,0,0,,74,74.00 -"2010-02-10 00:00:01",192.168.0.2,udp,19,node02.example.com,chargen,,64512,ZZ,Region,City,0,0,,74,74.00 -"2010-02-10 00:00:02",192.168.0.3,udp,19,node03.example.com,chargen,,64512,ZZ,Region,City,0,0,Government,74,74.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv deleted file mode 100644 index 5182817c1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic" -"2017-11-18 08:42:45","198.51.100.103","tcp",4786,"198-51-100-103.example.net","cisco-smart-install",8559,"AT","WIEN","VIENNA",0,0 -"2017-11-18 08:47:54","198.51.100.218","tcp",4786,,"cisco-smart-install",35609,"AT","WIEN","VIENNA",0,0 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv deleted file mode 100644 index 6d72dac53..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","response","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,5683,node01.example.com,coap,2,64512,ZZ,Region,City,0,0,",,",43,2.05 -"2010-02-10 00:00:01",192.168.0.2,udp,5683,node02.example.com,coap,2,64512,ZZ,Region,City,0,0,",,,,,,,,,",113,5.38 -"2010-02-10 00:00:02",192.168.0.3,udp,5683,node03.example.com,coap,1,64512,ZZ,Region,City,0,0,"`EsjAy************************************************************|CoAP RFC 7252 |************************************************************|This server is using the Eclipse Californium (Cf) CoAP framework|published under EPL+EDL: http://www.eclipse.org/californium/||(c) 2014, 2015, 2016 Institute for Pervasive Computing, ETH Zurich and others|************************************************************",454,113.50 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv deleted file mode 100644 index f4074f3ed..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","server_version","couchdb_message","couchdb_version","git_sha","features","vendor","visible_databases","error","error_reason" -"2010-02-10 00:00:00",192.168.0.1,tcp,5984,node01.example.com,couchdb,64512,ZZ,Region,City,0,0,,"CouchDB/1.6.1 (Erlang OTP/18)",Welcome,1.6.1,,,"Ubuntu 16.04",_replicator;_users;test;shops;god,, -"2010-02-10 00:00:01",192.168.0.2,tcp,5984,node02.example.com,couchdb,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service","CouchDB/3.2.1 (Erlang OTP/23)",Welcome,3.2.1,244d428af,"access-ready,partitioned,pluggable-storage-engines,reshard,scheduler","The Apache Software Foundation",,, -"2010-02-10 00:00:02",192.168.0.3,tcp,5984,node03.example.com,couchdb,64512,ZZ,Region,City,0,0,"Retail Trade","CouchDB/3.2.1 (Erlang OTP/20)",Welcome,3.2.1,244d428af,"access-ready,partitioned,pluggable-storage-engines,reshard,scheduler","The Apache Software Foundation",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv deleted file mode 100644 index 5aebed050..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","date","sector" -"2019-09-04 10:44:55","198.123.245.142","tcp",30005,,"cwmp",5678,"AA","LOCATION","LOCATION",517311,0,"HTTP/1.1",200,"OK","text/html","keep-alive",,,"DNVRS-Webs",5678,,"Wed, 04 Sep 2019 07:42:37 GMT", -"2019-09-04 11:06:50","198.123.245.162","tcp",5678,"localhost.localdomain","cwmp",5678,"AA","LOCATION","LOCATION",517311,0,"HTTP/1.1",404,"Not Found","text/html",,,,"RomPager/4.07 UPnP/1.0",,"chunked",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv deleted file mode 100644 index c4bb32e57..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","db2_hostname","servername","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,523,node01.example.com,db2,64512,ZZ,Region,City,0,0,NOWAK_SERWER,node01.example.com,298,14.90 -"2010-02-10 00:00:01",192.168.0.2,udp,523,node02.example.com,db2,64512,ZZ,Region,City,0,0,SPZOZ-DZIEWIN,node02.example.com,298,14.90 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv deleted file mode 100644 index 25e6f11d0..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","source_port","bytes","amplification","method" -"2010-02-10 00:00:00",192.168.0.1,tcp,80,node01.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,,49002,99,2,SYN+ACK:PSH -"2010-02-10 00:00:01",192.168.0.2,tcp,80,node02.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",41200,99,2,SYN+ACK:PSH -"2010-02-10 00:00:02",192.168.0.3,tcp,80,node03.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,,47492,99,2,SYN+ACK:PSH diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv deleted file mode 100644 index 05b807883..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv +++ /dev/null @@ -1,101 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","dns_version","asn","geo","region","city","min_amplification","p0f_genre","p0f_detail","naics","sic","sector" -"2018-04-14 00:14:34","198.51.100.179","udp",53,"198-51-100-189.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.8","udp",53,"198-51-100-111.example.net","openresolver","dnsmasq-2.51",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:34","198.51.100.179","udp",53,"198-51-100-189.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.8","udp",53,"198-51-100-111.example.net","openresolver","dnsmasq-2.51",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.158","udp",53,,"openresolver",,8447,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:37","198.51.100.7","udp",53,"198-51-100-7.example.net","openresolver","9.9.4-rpz2.13269.14-P2",13292,"AT","STEIERMARK","EISENERZ","4.6190",,,0,0, -"2018-04-14 00:14:38","198.51.100.167","udp",53,"198-51-100-167.example.net","openresolver","ZyWALL DNS",6830,"AT","KARNTEN","VILLACH","4.6667",,,0,0, -"2018-04-14 00:14:40","198.51.100.10","udp",53,"198-51-100-10.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:14:41","198.51.100.191","udp",53,"198-51-100-63.example.net","openresolver",,25255,"AT","TIROL","LIENZ","4.6190",,,0,0, -"2018-04-14 00:14:43","198.51.100.25","udp",53,"198-51-100-187.example.net","openresolver","p.4.0",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:54","198.51.100.174","udp",53,"198-51-100-174.example.net","openresolver",,8445,"AT","SALZBURG","SALZBURG","6.4048",,,0,0, -"2018-04-14 00:14:54","198.51.100.181","udp",53,"198-51-100-181.example.net","openresolver","PowerDNS Recursor tele2/sil",8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:54","198.51.100.80","udp",53,"198-51-100-80.example.net","openresolver",,1901,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:14:57","198.51.100.43","udp",53,"198-51-100-43.example.net","openresolver","vi2zcnsat10, Customer DNS",6830,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:14:58","198.51.100.124","udp",53,"198-51-100-124.example.net","openresolver","dnsmasq-2.47",28919,"AT","TIROL","EIBERG","3.8095",,,0,0, -"2018-04-14 00:15:00","198.51.100.60","udp",53,"198-51-100-60.example.net","openresolver",,24992,"AT","VORARLBERG","DORNBIRN","3.4762",,,0,0, -"2018-04-14 00:15:00","198.51.100.201","udp",53,"198-51-100-201.example.net","openresolver",,1853,"AT","STEIERMARK","GRAZ","4.6190",,,0,0, -"2018-04-14 00:15:01","198.51.100.82","udp",53,"198-51-100-82.example.net","openresolver","9.6-ESV-R7-P2",20811,"AT","TIROL","INNSBRUCK","4.6190",,,0,0, -"2018-04-14 00:15:01","198.51.100.105","udp",53,"198-51-100-105.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:02","198.51.100.173","udp",53,"198-51-100-173.example.net","openresolver",,8445,"AT","NIEDEROSTERREICH","WALD","1.3810",,,0,0, -"2018-04-14 00:15:03","198.51.100.82","udp",53,"198-51-100-82.example.net","openresolver","PowerDNS Recursor tele2/sil",1257,"AT","NIEDEROSTERREICH","MODLING","1.3810",,,518111,737401, -"2018-04-14 00:15:05","198.51.100.39","udp",53,,"openresolver",,8437,"AT","VORARLBERG","LUSTENAU","1.3810",,,0,0, -"2018-04-14 00:15:09","198.51.100.33","udp",53,,"openresolver","dnsmasq-2.55",8447,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:15:09","198.51.100.248","udp",53,"198-51-100-248.example.net","openresolver",,39912,"AT","NIEDEROSTERREICH","HOLLABRUNN","3.8095",,,0,0, -"2018-04-14 00:15:10","198.51.100.119","udp",53,"198-51-100-172.example.net","openresolver","dnsmasq-2.62",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:12","198.51.100.135","udp",53,"198-51-100-135.example.net","openresolver","no access.",43848,"AT","NIEDEROSTERREICH","WIESELBURG","3.8095",,,0,0, -"2018-04-14 00:15:15","198.51.100.64","udp",53,"198-51-100-64.example.net","openresolver",,6830,"AT","VORARLBERG","UBERSAXEN","1.3810",,,0,0, -"2018-04-14 00:15:17","198.51.100.80","udp",53,"198-51-100-80.example.net","openresolver",,42473,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:18","198.51.100.60","udp",53,"198-51-100-60.example.net","openresolver","198-51-100-60.example.net",35369,"AT","OBEROSTERREICH","LINZ","1.3810",,,0,0, -"2018-04-14 00:15:21","198.51.100.50","udp",53,,"openresolver","ZyWALL DNS",8447,"AT","STEIERMARK","TAUPLITZ","4.6667",,,0,0, -"2018-04-14 00:15:23","198.51.100.93","udp",53,,"openresolver","Microsoft DNS 6.1.7601 (1DB15D39)",8447,"AT","NIEDEROSTERREICH","SCHWADORF","1.3810",,,0,0, -"2018-04-14 00:15:24","198.51.100.33","udp",53,,"openresolver",,8447,"AT","STEIERMARK","FURSTENFELD","4.6190",,,0,0, -"2018-04-14 00:15:31","198.51.100.45","udp",53,,"openresolver","dnsmasq-2.52",8245,"AT","BURGENLAND","EISENSTADT","1.3810",,,0,0, -"2018-04-14 00:15:34","198.51.100.13","udp",53,"198-51-100-13.example.net","openresolver",,8447,"AT","WIEN","VIENNA","6.4048",,,518210,737415, -"2018-04-14 00:15:36","198.51.100.190","udp",53,,"openresolver",,8447,"AT","BURGENLAND","PINKAFELD","1.3810",,,0,0, -"2018-04-14 00:15:41","198.51.100.104","udp",53,,"openresolver",,21013,"AT","OBEROSTERREICH","LEONDING","2.6667",,,0,0, -"2018-04-14 00:15:42","198.51.100.101","udp",53,"198-51-100-101.example.net","openresolver",,8447,"AT","STEIERMARK","KAINACH BEI VOITSBERG","1.3810",,,0,0, -"2018-04-14 00:15:44","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver",,1901,"AT","OBEROSTERREICH","GMUNDEN","1.3810",,,518210,737415, -"2018-04-14 00:15:46","198.51.100.186","udp",53,"198-51-100-186.example.net","openresolver",,31239,"AT","WIEN","VIENNA","6.4048",,,0,0, -"2018-04-14 00:15:46","198.51.100.197","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","KIRCHDORF AN DER KREMS","4.6190",,,0,0, -"2018-04-14 00:15:49","198.51.100.16","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","LAAKIRCHEN","4.6190",,,0,0, -"2018-04-14 00:15:50","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver",,6830,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","4.6190",,,0,0, -"2018-04-14 00:15:53","198.51.100.7","udp",53,"198-51-100-7.example.net","openresolver",,198950,"AT","TIROL","REUTTE","4.6190",,,518210,737415, -"2018-04-14 00:15:53","198.51.100.177","udp",53,"198-51-100-177.example.net","openresolver","Microsoft DNS 6.1.7601 (1DB1446A)",12605,"AT","OBEROSTERREICH","LINZ","1.3810",,,0,0, -"2018-04-14 00:15:57","198.51.100.47","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","KOTTINGBRUNN","1.3810",,,0,0, -"2018-04-14 00:15:59","198.51.100.95","udp",53,"198-51-100-67.example.net","openresolver","GNS DNS Version 3",57169,"AT","WIEN","VIENNA","1.3810",,,0,0,"Information Technology" -"2018-04-14 00:16:02","198.51.100.104","udp",53,"198-51-100-104.example.net","openresolver",,6830,"AT","OBEROSTERREICH","BAD WIMSBACH-NEYDHARTING","1.3810",,,0,0, -"2018-04-14 00:16:04","198.51.100.106","udp",53,,"openresolver",,8447,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:16:05","198.51.100.204","udp",53,"198-51-100-204.example.net","openresolver",,12605,"AT","OBEROSTERREICH","LINZ","4.6190",,,0,0, -"2018-04-14 00:16:05","198.51.100.111","udp",53,"198-51-100-111.example.net","openresolver",,8447,"AT","OBEROSTERREICH","LINZ","1.3810",,,518210,737415, -"2018-04-14 00:16:06","198.51.100.131","udp",53,"198-51-100-139.example.net","openresolver","p.4.0",25255,"AT","OBEROSTERREICH","TRAUN","1.3810",,,0,0, -"2018-04-14 00:16:10","198.51.100.240","udp",53,"198-51-100-240.example.net","openresolver",,6830,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:13","198.51.100.9","udp",53,"198-51-100-42.example.net","openresolver",,13026,"AT","STEIERMARK","LEIBNITZ","6.4048",,,0,0, -"2018-04-14 00:16:15","198.51.100.231","udp",53,"198-51-100-74.example.net","openresolver",,25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:17","198.51.100.228","udp",53,"198-51-100-227.example.net","openresolver","u.1.0",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:19","198.51.100.152","udp",53,"198-51-100-152.example.net","openresolver",,34694,"AT","TIROL","WORGL","4.6190",,,0,0, -"2018-04-14 00:16:21","198.51.100.88","udp",53,,"openresolver",,8447,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:22","198.51.100.97","udp",53,"198-51-100-97.example.net","openresolver",,8447,"AT","TIROL","INNSBRUCK","1.3810",,,518210,737415, -"2018-04-14 00:16:23","198.51.100.208","udp",53,"198-51-100-208.example.net","openresolver","dnsmasq-2.62",8447,"AT","TIROL","OTZTAL-BAHNHOF","1.3810",,,0,0, -"2018-04-14 00:16:33","198.51.100.113","udp",53,"198-51-100-121.example.net","openresolver","dnsmasq-2.62",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:35","198.51.100.34","udp",53,"198-51-100-44.example.net","openresolver",,25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:37","198.51.100.236","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","ST. ANDRAE-WOERDERN","4.6190",,,0,0, -"2018-04-14 00:16:40","198.51.100.46","udp",53,"198-51-100-46.example.net","openresolver",,21013,"AT","OBEROSTERREICH","LEONDING","2.6667",,,0,0, -"2018-04-14 00:16:45","198.51.100.72","udp",53,"198-51-100-5.example.net","openresolver",,25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:50","198.51.100.179","udp",53,"198-51-100-179.example.net","openresolver",,31125,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:50","198.51.100.107","udp",53,"198-51-100-107.example.net","openresolver","dnsmasq-2.66",18845,"AT","WIEN","VIENNA","1.3810",,,0,0,"Information Technology" -"2018-04-14 00:16:51","198.51.100.188","udp",53,,"openresolver","9.9.4-RedHat-9.9.4-51.el7_4.2",49322,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:54","198.51.100.232","udp",53,"198-51-100-232.example.net","openresolver",,6830,"AT","SALZBURG","SALZBURG","1.3810",,,0,0, -"2018-04-14 00:16:55","198.51.100.102","udp",53,"198-51-100-102.example.net","openresolver","ZyWALL DNS",6830,"AT","KARNTEN","WERNBERG","3.4762",,,0,0, -"2018-04-14 00:16:59","198.51.100.162","udp",53,"198-51-100-162.example.net","openresolver",,8445,"AT","SALZBURG","SALZBURG","1.3810",,,0,0, -"2018-04-14 00:17:00","198.51.100.110","udp",53,"198-51-100-110.example.net","openresolver",,31543,"AT","TIROL","SOLDEN","4.6190",,,0,0, -"2018-04-14 00:17:02","198.51.100.193","udp",53,"198-51-100-193.example.net","openresolver",,8447,"AT","STEIERMARK","FOHNSDORF","1.3810",,,0,0, -"2018-04-14 00:17:06","198.51.100.45","udp",53,"198-51-100-45.example.net","openresolver",,61201,"AT","KARNTEN","KLAGENFURT AM WORTHERSEE","1.3810",,,0,0, -"2018-04-14 00:17:06","198.51.100.219","udp",53,"198-51-100-219.example.net","openresolver",,8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:10","198.51.100.47","udp",53,"198-51-100-47.example.net","openresolver","unsupported query",8412,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.8095",,,0,0, -"2018-04-14 00:17:13","198.51.100.87","udp",53,"198-51-100-87.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:16","198.51.100.121","udp",53,"198-51-100-121.example.net","openresolver",,8447,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:20","198.51.100.115","udp",53,,"openresolver",,8447,"AT","TIROL","WAIDRING","1.3810",,,0,0, -"2018-04-14 00:17:22","198.51.100.235","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","GRIESKIRCHEN","1.3810",,,0,0, -"2018-04-14 00:17:33","198.51.100.154","udp",53,,"openresolver",,8447,"AT","STEIERMARK","GRAZ","4.6190",,,0,0, -"2018-04-14 00:17:36","198.51.100.36","udp",53,"198-51-100-36.example.net","openresolver","BIND",12605,"AT","OBEROSTERREICH","LINZ","4.6190",,,0,0, -"2018-04-14 00:17:38","198.51.100.100","udp",53,"198-51-100-100.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:41","198.51.100.181","udp",53,"198-51-100-181.example.net","openresolver","Microsoft DNS 6.0.6002 (17724D35)",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:41","198.51.100.242","udp",53,"198-51-100-242.example.net","openresolver","Microsoft DNS 6.0.6002 (17724D35)",34767,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.2857",,,0,0, -"2018-04-14 00:17:42","198.51.100.38","udp",53,,"openresolver","PowerDNS Recursor tele2/sil",8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:43","198.51.100.132","udp",53,"198-51-100-132.example.net","openresolver","PowerDNS Recursor tele2/sil",8437,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:17:49","198.51.100.166","udp",53,"198-51-100-166.example.net","openresolver","9.8.4-rpz2+rl005.12-P1",13292,"AT","STEIERMARK","KINDBERG","4.6190",,,0,0, -"2018-04-14 00:17:49","198.51.100.212","udp",53,"198-51-100-212.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:51","198.51.100.225","udp",53,,"openresolver",,8220,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:17:53","198.51.100.161","udp",53,"198-51-100-161.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:54","198.51.100.12","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","LANGENLOIS","1.3810",,,0,0, -"2018-04-14 00:17:55","198.51.100.113","udp",53,"198-51-100-113.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:57","198.51.100.175","udp",53,"198-51-100-175.example.net","openresolver","PowerDNS Recursor tele2/sil",1257,"AT","NIEDEROSTERREICH","MODLING","1.3810",,,518111,737401, -"2018-04-14 00:17:59","198.51.100.107","udp",53,"198-51-100-107.example.net","openresolver",,50719,"AT","STEIERMARK","TIESCHEN","3.8095",,,0,0, -"2018-04-14 00:17:59","198.51.100.51","udp",53,"198-51-100-68.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:18:04","198.51.100.131","udp",53,,"openresolver","ZyWALL DNS",8447,"AT","TIROL","OBERPERFUSS","3.4762",,,0,0, -"2018-04-14 00:18:05","198.51.100.138","udp",53,"198-51-100-138.example.net","openresolver","unsupported query",8412,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.8095",,,0,0, -"2018-04-14 00:18:06","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver","viezcnsat13, Customer DNS",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:18:07","198.51.100.109","udp",53,"198-51-100-109.example.net","openresolver",,1901,"AT","OBEROSTERREICH","LINZ","6.9524",,,518210,737415, -"2018-04-14 00:18:10","198.51.100.205","udp",53,"198-51-100-205.example.net","openresolver",,8437,"AT","WIEN","VIENNA","1.3810",,,0,0, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv deleted file mode 100644 index 535dc4ea8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","http","http_code","http_reason","content_type","server","date","experimental","api_version","arch","go_version","os","kernel_version","git_commit","min_api_version","build_time","pkg_version" -"2010-02-10 00:00:00",192.168.0.1,tcp,2375,node01.example.com,docker,18.05.0-ce,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",HTTP/1.1,200,OK,"application/json; charset=UTF-8","Docker/18.05.0-ce (linux)","Fri, 06 May 2022 14:06:30 GMT",false,1.37,amd64,go1.9.5,linux,3.10.0-514.26.2.el7.x86_64,f150324,1.12,2018-05-09T22:18:36.000000000+00:00, -"2010-02-10 00:00:01",192.168.0.2,tcp,2375,node02.example.com,docker,1.13.1,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,"Docker/1.13.1 (linux)","Fri, 06 May 2022 14:08:07 GMT",false,1.26,amd64,go1.10.3,linux,3.10.0-693.2.2.el7.x86_64,7d71120/1.13.1,1.12,2022-03-02T15:25:43.414574467+00:00,docker-1.13.1-209.git7d71120.el7.centos.x86_64 -"2010-02-10 00:00:02",192.168.0.3,tcp,2375,node03.example.com,docker,18.05.0-ce,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,"application/json; charset=UTF-8","Docker/18.05.0-ce (linux)","Fri, 06 May 2022 14:08:06 GMT",false,1.37,amd64,go1.9.5,linux,3.10.0-514.26.2.el7.x86_64,f150324,1.12,2018-05-09T22:18:36.000000000+00:00, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv deleted file mode 100644 index 60c711973..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_id","device_serial","machine_name","manufacturer","method","http_port","internal_port","video_input_channels","alarm_input_channels","video_output_channels","alarm_output_channels","remote_video_input_channels","mac_address","ipv4_address","ipv4_gateway","ipv4_subnet_mask","ipv4_dhcp_enable","ipv6_address","ipv6_link_local","ipv6_gateway","ipv6_dhcp_enable","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,37810,node01.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",General,IPC,BCS-TIP3401IR-E-V,2.800.106F004.0.R,,6J0E022PAG35073,6J0E022PAG35073,General,client.notifyDevInfo,80,37777,1,0,0,0,0,38:c4:e8:03:b3:e2,192.168.0.1,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::1,fe80::3ac4:e8ff:fe03:b3e2/64,fd09:4ab5:dae9:b078::ff,0,794,794.00 -"2010-02-10 00:00:01",192.168.0.2,udp,37810,node02.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,,Private,HCVR,HCVR,3.210.1.4,,2K0488CPAGS0ND6,HCVR,Private,client.notifyDevInfo,80,37777,3,0,0,0,9,3c:ef:8c:18:a5:07,192.168.0.2,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::2,fe80::3eef:8cff:fe18:a507/64,fd09:4ab5:dae9:b078::ff,,761,761.00 -"2010-02-10 00:00:02",192.168.0.3,udp,37810,node03.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",General,HCVR,BCS-XVR0401-IV,4.000.0000002.11,,5L034FAPAZA0E30,XVR,General,client.notifyDevInfo,80,37777,4,0,0,0,0,38:c4:e8:02:74:da,192.168.0.3,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::3,fe80::3ac4:e8ff:fe02:74da/64,fd09:4ab5:dae9:b078::ff,,711,711.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv deleted file mode 100644 index c681a8595..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","ok","name","cluster_name","http_code","build_hash","build_timestamp","build_snapshot","lucene_version","tagline","sector" -"2010-02-10 00:00:00",192.168.0.1,tcp,9200,node01.example.com,elasticsearch,2.3.5,64512,ZZ,Region,City,0,0,,"Red Skull",elasticsearch,,90f439ff60a3c0f497f91663701e64ccd01edbb4,2016-07-27T10:36:52Z,false,5.5.0,"You Know, for Search","Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:01",192.168.0.2,tcp,9200,node02.example.com,elasticsearch,7.17.0,64512,ZZ,Region,City,0,0,,allinonepod,docker-cluster,,bee86328705acaa9a6daede7140defd4d9ec56bd,,false,8.11.1,"You Know, for Search","Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,tcp,9200,node03.example.com,elasticsearch,7.15.0,64512,ZZ,Region,City,0,0,,f547c2952610,docker-cluster,,79d65f6e357953a5b3cbcc5e2c7c21073d89aa29,,false,8.9.0,"You Know, for Search","Communications, Service Provider, and Hosting Service" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv deleted file mode 100644 index 4e375a9b4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv +++ /dev/null @@ -1,8 +0,0 @@ -"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","sector","version","servername","url" -"2021-05-14 00:11:30","12.237.1.2",443,"afs-exch-cas2.xxx.com","exchange;cve-2021-26855",7018,"US","CALIFORNIA","TURLOCK",517311,,"Communications, Service Provider, and Hosting Service","15.2.721","AFS-EXCH2019", -"2021-05-14 00:11:37","98.153.3.4",443,"rrcs-98-153-x-x.west.biz.rr.com","exchange;webshell",20001,"US","CALIFORNIA","LOS ANGELES",517311,,"Communications, Service Provider, and Hosting Service","15.0.847","SSAMAIL", -"2021-05-14 00:11:38","206.210.5.6",443,"webmail.xxx.com","exchange;webshell",17054,"US","PENNSYLVANIA","PITTSBURGH",518210,,,"15.0.1178","OMNYXEXCH02", -"2021-05-14 00:11:38","12.33.7.8",443,"mail.xxx.org","exchange;cve-2021-26855",7018,"US","ARKANSAS","LITTLE ROCK",921120,,"Communications, Service Provider, and Hosting Service","15.1.2176","MHASVR02", -"2021-05-14 00:11:38","41.204.9.10",443,"mail.xxx.mg","exchange;cve-2021-26855",21042,"MG","ANTANANARIVO","ANTANANARIVO",,,,,"SABMHQE0232", -"2021-05-14 00:11:38","62.33.11.12",443,,"exchange;cve-2021-26855",20485,"RU","ALTAYSKIY KRAY","BARNAUL",,,,"15.2.659","PV-SRV04", -"2021-05-14 00:11:43","199.33.13.14",443,"mail.xxx.tv","exchange;cve-2021-26855",26481,"US","CALIFORNIA","LOS ANGELES",,,,"15.1.1779","MAIL", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv deleted file mode 100644 index 912e73d84..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","auth_tls_response","auth_ssl_response","tlsv13_support","tlsv13_cipher","jarm","device_vendor","device_type","device_model","device_version","device_sector" -"2019-03-06 06:37:00","61.126.3.70","tcp",21,"arcus-net.co.jp","ftp",4713,"JP","TOKYO","TOKYO",517311,737401,"220 FTP Server ready.|","TLSv1.2","TLS_RSA_WITH_AES_128_CBC_SHA",2048,"*.bizmw.com","GlobalSign Organization Validation CA - SHA256 - G2","Jan 14 08:04:50 2015 GMT","Jan 14 08:04:50 2020 GMT","D9:98:3F:2E:F9:D1:BE:9A:10:1E:DE:51:2C:C1:DF:01:18:0A:20:65","1121DC7421AB7924C3B1D396AEA3707E9E29",2,"sha256WithRSAEncryption","rsaEncryption","NTT Communications Corporation",,"JP","Tokyo","Minato-ku",,,,,,,,"GlobalSign nv-sa",,"BE",,,,,,,,,,"27:4A:8A:3A:A7:DF:82:D0:43:03:0E:6F:48:30:30:C9:24:77:11:1A:08:EF:F7:B9:74:0C:CE:40:87:03:D2:51","E5:93:8B:72:84:0F:35:52:8E:7A:6C:E3:EF:36:90:4C:F2:86:A7:4D:B2:DD:C0:C6:23:83:18:EF:DD:86:34:92:91:57:22:29:75:45:71:8B:3A:CD:F1:27:A9:CA:5F:70:5E:AC:15:A5:E6:63:FD:6F:BB:C5:E2:45:99:73:E9:E6","D1:A7:BC:96:78:1D:16:D0:24:A8:62:7C:3A:95:5A:4A","N","N","N","OV","234 AUTH TLS successful",,,,,,,,, -"2019-03-06 06:37:00","62.48.156.65","tcp",21,"dial-62-48-156-65.ptprime.net","ftp",15525,"PT","LISBOA","FRIELAS",0,0,"220-================================================================| PT Empresas| Acesso Reservado| Acesso nao autorizado punido por lei: 109/91; 67/98| ----------------------------------------------------------------| HENNES & MAURITZ LDA - 149093| SITE: PT303 - Cascais Shopping| MORADA: | NIR: EWS1822940| ================================================================|220 FTP server ready, 1 active clients of 4 simultaneous clients allowed.|",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"500 Syntax error, command unrecognized.","500 Syntax error, command unrecognized.",,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv deleted file mode 100644 index 26f8ccbcf..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","version","asn","geo","region","city","naics","sic","server_type","clusterid","total_disk","used_disk","free_disk","livenodes","namenodeaddress","volumeinfo" -"2017-09-13 02:06:05","199.116.235.200",50070,,"2.7.3, rbaa91f7c6bc9cb92be5982de4719c1c8af91ccff",15296,"CA","ALBERTA","CALGARY",0,0,"namenode","CID-64471a53-60cb-4302-9832-92f321f111fe",41567956992,53248,25160089600,"edmonton:50010",, -"2017-09-13 02:07:48","104.43.235.92",50075,,"2.7.1.2.4.0.0-169",8075,"US","IOWA","DES MOINES",334111,357101,"datanode","CID-771bae52-9e4f-4ec4-bc1a-c867585751f0",,,,,"sandbox.hortonworks.com","/hadoop/hdfs/data/current" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv deleted file mode 100644 index a7e3eb707..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","http_date" -"2018-04-19 00:02:26","75.74.78.113","tcp",8080,"c-75-74-78-113.hsd1.fl.comcast.net","http",7922,"US","FLORIDA","MIAMI",518111,737401,"HTTP/1.1",200,"OK","text/html",,,,"lighttpd",,"chunked","Thu, 19 Apr 2018 00:02:28 GMT" -"2018-04-19 00:02:26","88.162.174.130","tcp",8080,"sto95-3-88-162-174-130.fbx.proxad.net","http",12322,"FR",,"SAINT-OUEN-LAUMONE",518210,737415,"HTTP/1.1",200,"OK","text/html",,,,,17729,,"Thu, 19 Apr 2018 02:02:28 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv deleted file mode 100644 index b1f2330f1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","proxy_authenticate","via","server","content_length","transfer_encoding","http_date" -"2010-02-10 00:00:00",192.168.0.1,tcp,3128,node01.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,,,,,"Wed, 10 Feb 2010 00:00:00 GMT" -"2010-02-10 00:00:01",192.168.0.2,tcp,3128,node02.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,"HTTP/1.1 s_proxy_den1",,,,"Wed, 10 Feb 2010 00:00:01 GMT" -"2010-02-10 00:00:02",192.168.0.3,tcp,3128,node03.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,"HTTP/1.1 s_proxy_yvr",,,,"Wed, 10 Feb 2010 00:00:02 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv deleted file mode 100644 index 195342533..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","http_date","version","build_date","detail" -"2010-02-10 00:00:00",192.168.0.1,tcp,8080,node01.example.com,"basic-auth,http",64512,ZZ,Region,City,0,0,HTTP/1.1,401,Unauthorized,"text/html; charset=utf-8",,"Basic realm=\"\"OpenWebif\"\"",TWISTED_SESSION=5473ad3faa3de66685fb3a53bffb390b4fcec2039893009a06caf38e1bec8aa8,TwistedWeb/19.7.0,149,,"Wed, 10 Feb 2010 00:00:00 GMT",,, -"2010-02-10 00:00:01",192.168.0.2,tcp,80,node02.example.com,"basic-auth,http",64512,ZZ,Region,City,0,0,HTTP/1.1,401,Unauthorized,"text/html; charset=utf-8",,"Basic realm=\"\"OpenWebif\"\"",TWISTED_SESSION=d2460d37b7fdbdd6c27dd74423ead5704e553d4f2c230672313edc5602059e33,TwistedWeb/19.7.0,149,,"Wed, 10 Feb 2010 00:00:01 GMT",,, -"2010-02-10 00:00:02",192.168.0.3,tcp,443,node03.example.com,git-config-file,64512,ZZ,Region,City,0,0,,,,,,,,,,,"Wed, 10 Feb 2010 00:00:02 GMT",,,"repositoryformatversion = 0;filemode = false;bare = false;logallrefupdates = true;symlinks = false;ignorecase = true" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv deleted file mode 100644 index d327f1f3b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_id","response_size","raw_response" -2022-03-02 00:34:22,192.168.0.1,tcp,502,host1.example.net,modbus,64512,ZZ,REGION,CITY,0,0,Sector,Vendor 1,device_type,device_model,device_version,0,5,dGVzdDE= -2022-03-02 00:34:22,192.168.0.2,tcp,502,host2.example.net,modbus,64513,ZZ,REGION,CITY,0,0,Sector,Vendor 2,device_type,device_model,device_version,0,5,dGVzdDI= -2022-03-02 00:34:22,192.168.0.3,tcp,502,host3.example.net,modbus,64514,ZZ,REGION,CITY,0,0,Sector,Vendor 3,device_type,device_model,device_version,0,5,dGVzdDM= diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv deleted file mode 100644 index 87a98157f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv +++ /dev/null @@ -1,96 +0,0 @@ -"timestamp","ip","port","hostname","tag","ipmi_version","asn","geo","region","city","none_auth","md2_auth","md5_auth","passkey_auth","oem_auth","defaultkg","permessage_auth","userlevel_auth","usernames","nulluser","anon_login","error","deviceid","devicerev","firmwarerev","version","manufacturerid","manufacturername","productid","productname","naics","sic","sector" -"2016-07-24 00:09:42","198.51.100.4",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.182",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:42","198.51.100.4",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.182",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.176",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.221",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:44","198.51.100.176",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:45","198.51.100.174",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:45","198.51.100.167",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:46","198.51.100.60",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:47","198.51.100.7",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:48","198.51.100.24",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.86",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.231",623,,"ipmi","1.5",20686,"DE","BAYERN","HAPPURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.197",623,,"ipmi","2.0",3320,"DE","BERLIN","BERLIN","no","no","yes","yes","yes","default","enabled","enabled","yes","no","yes",,,,,,,,,,541690,874899, -"2016-07-24 00:09:49","198.51.100.87",623,,"ipmi","1.5",3209,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:49","198.51.100.6",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:51","198.51.100.193",623,,"ipmi","2.0",15598,"DE","BAYERN","NUREMBERG","no","yes","yes","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:51","198.51.100.63",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:52","198.51.100.179",623,,"ipmi","2.0",3320,"DE","BAYERN","DENKLINGEN","no","no","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,518210,737415, -"2016-07-24 00:09:53","198.51.100.112",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:53","198.51.100.189",623,,"ipmi","2.0",30134,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0,"Communications" -"2016-07-24 00:09:54","198.51.100.44",623,"198-51-100-44.example.net","ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","yes","yes","no","no","default","disabled","disabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:54","198.51.100.215",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.231",623,"198-51-100-231.example.net","ipmi","2.0",6805,"DE","HAMBURG","HAMBURG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.234",623,,"ipmi","2.0",31103,"DE","THURINGEN","ERFURT","no","no","yes","no","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.165",623,,"ipmi","2.0",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:56","198.51.100.170",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:56","198.51.100.66",623,,"ipmi","2.0",41412,"DE","BAYERN","REGENSBURG","no","yes","yes","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:56","198.51.100.150",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:57","198.51.100.222",623,,"ipmi","2.0",34309,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:57","198.51.100.19",623,,"ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:58","198.51.100.83",623,,"ipmi","1.5",3209,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:00","198.51.100.61",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:00","198.51.100.94",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:01","198.51.100.242",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:03","198.51.100.251",623,,"ipmi","2.0",553,"DE","BADEN-WURTTEMBERG","HEIDELBERG","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:03","198.51.100.41",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:04","198.51.100.160",623,"198-51-100-160.example.net","ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:04","198.51.100.243",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.190",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.29",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.224",623,,"ipmi","2.0",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:06","198.51.100.143",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","HEMER","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.120",623,,"ipmi","2.0",13003,"DE","SACHSEN","LEIPZIG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.196",623,,"ipmi","1.5",20686,"DE","BAYERN","HAPPURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.123",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.122",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.192",623,,"ipmi","2.0",34171,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,518210,737415, -"2016-07-24 00:10:08","198.51.100.146",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.127",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:09","198.51.100.112",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:09","198.51.100.45",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:09","198.51.100.46",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","NEUSS","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:10","198.51.100.202",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:11","198.51.100.6",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:11","198.51.100.34",623,,"ipmi","2.0",3320,"DE","HESSEN","LEUN","no","yes","yes","no","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,518210,737415, -"2016-07-24 00:10:12","198.51.100.210",623,,"ipmi","2.0",3320,"DE","BADEN-WURTTEMBERG","AALEN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,541690,874899, -"2016-07-24 00:10:12","198.51.100.97",623,,"ipmi","2.0",42730,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:12","198.51.100.172",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.20",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.181",623,,"ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","yes","yes","no","no","default","disabled","disabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.244",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.85",623,,"ipmi","2.0",34309,"DE","HESSEN","FRANKFURT AM MAIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.150",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.154",623,,"ipmi","2.0",196763,"DE","SAARLAND","ST. INGBERT","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.83",623,,"ipmi","2.0",31342,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.6",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.228",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.150",623,,"ipmi","2.0",44066,"DE","BAYERN","MUNICH","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:15","198.51.100.71",623,,"ipmi","2.0",44066,"DE","BAYERN","MUNICH","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:15","198.51.100.239",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:17","198.51.100.46",623,"198-51-100-53.example.net","ipmi","2.0",29083,"DE","BRANDENBURG","MAHLOW","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:17","198.51.100.78",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:18","198.51.100.164",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,812990,489999, -"2016-07-24 00:10:18","198.51.100.142",623,,"ipmi","2.0",34568,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:18","198.51.100.85",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.173",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.180",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.119",623,,"ipmi","2.0",12843,"DE","RHEINLAND-PFALZ","SPEYER","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.183",623,,"ipmi","1.5",12348,"DE","BAYERN","NUREMBERG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:20","198.51.100.108",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:20","198.51.100.221",623,"198-51-100-156.example.net","ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:21","198.51.100.200",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.162",623,,"ipmi","1.5",30766,"DE","HESSEN","BENSHEIM","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.140",623,,"ipmi","2.0",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.121",623,,"ipmi","2.0",34549,"DE","HESSEN","FRANKFURT AM MAIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:22","198.51.100.33",623,,"ipmi","2.0",47215,"DE","NORDRHEIN-WESTFALEN","GUTERSLOH","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:22","198.51.100.203",623,,"ipmi","2.0",201011,"DE","BAYERN","NUREMBERG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:23","198.51.100.16",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:24","198.51.100.166",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","no","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.135",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.154",623,"198-51-100-154.example.net","ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.237",623,,"ipmi","2.0",12586,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.45",623,,"ipmi","2.0",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv deleted file mode 100644 index a585db6eb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv +++ /dev/null @@ -1,2 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","ipp_version","cups_version","printer_uris","printer_name","printer_info","printer_more_info","printer_make_and_model","printer_firmware_name","printer_firmware_string_version","printer_firmware_version","printer_organization","printer_organization_unit","printer_uuid","printer_wifi_ssid","device_vendor","device_type","device_model","device_version","device_sector" -"2020-06-08 11:30:14","123.45.67.89","tcp",631,"some.host.com","ipp",12345,"AA","REGION","CITY",517311,0,"IPP/2.1","CUPS/2.0","ipp://123.45.67.89:631/ipp/print","NPI3F0D22","HP Color LaserJet MFP M277dw","http://123.45.67.89:631/hp/device/info_config_AirPrint.html?tab=Networking&menu=AirPrintStatus","HP Color LaserJet MFP M277dw",20191203,20191203,20191203,"org","unit","urn:uuid:456e4238-4a44-4643-4c42-10e1813f0a18","wifissid",,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license deleted file mode 100644 index 476908eeb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv deleted file mode 100644 index cef6b027c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","initiator_spi","responder_spi","next_payload","exchange_type","flags","message_id","next_payload2","domain_of_interpretation","protocol_id","spi_size","notify_message_type" -"2019-09-04 00:17:25","198.123.245.42","udp",500,"example.local","isakmp-vulnerable",5678,"AA","LOCATION","LOCATION",517311,0,"3e35c70729dfedef","253acab7cbfda607",11,05,00,00000000,00,00,,0,14 -"2019-09-04 00:17:28","198.123.245.67","udp",500,"example.local","isakmp-vulnerable",20255,"AA","LOCATION","LOCATION",0,0,"3e35c70729dfedef","b274460e7adc1bf0",11,05,00,00000000,00,00,,0,14 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv deleted file mode 100644 index ab71b9a15..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","http","http_code","http_reason","content_type","server","date","major","minor","git_version","git_commit","git_tree_state","build_date","go_version","compiler","platform","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,6443,node01.example.com,kubernetes,,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:13 GMT",1,20,v1.20.13,2444b3347a2c45eb965b182fb836e1f51dc61b70,clean,2021-11-17T13:00:29Z,go1.15.15,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:01",192.168.0.2,tcp,6443,node02.example.com,kubernetes,,64512,ZZ,Region,City,0,0,"Retail Trade",HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:12 GMT",1,23,v1.23.3+e419edf,6f5a5295923a614a4202a7ad274b38b69f9ca8c0,clean,2022-02-25T06:26:46Z,go1.17.5,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:02",192.168.0.3,tcp,6443,node03.example.com,kubernetes,,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:12 GMT",1,16+,v1.16.9-aliyun.1,4f7ea78,,2020-05-08T07:29:59Z,go1.13.9,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv deleted file mode 100644 index 54121fd3b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","size","configuration_naming_context","current_time","default_naming_context","dns_host_name","domain_controller_functionality","domain_functionality","ds_service_name","forest_functionality","highest_committed_usn","is_global_catalog_ready","is_synchronized","ldap_service_name","naming_contexts","root_domain_naming_context","schema_naming_context","server_name","subschema_subentry","supported_capabilities","supported_control","supported_ldap_policies","supported_ldap_version","supported_sasl_mechanisms","amplification" -"2010-02-10 00:00:00",192.168.0.1,tcp,389,node01.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",,"DC=ad,DC=example,DC=com",node01.example.com,7,,"CN=Configuration,DC=ad,DC=example,DC=com",2,,,,node01.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5, -"2010-02-10 00:00:01",192.168.0.2,tcp,389,node02.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",20220821124435.0Z,"DC=ad,DC=example,DC=com",node02.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,25029662,TRUE,TRUE,node02.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5, -"2010-02-10 00:00:02",192.168.0.3,tcp,389,node03.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",20220821124539.0Z,"DC=ad,DC=example,DC=com",node03.example.com,,,"CN=Configuration,DC=ad,DC=example,DC=com",,,,,node03.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv deleted file mode 100644 index 3cd5021c5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","size","configuration_naming_context","current_time","default_naming_context","dns_host_name","domain_controller_functionality","domain_functionality","ds_service_name","forest_functionality","highest_committed_usn","is_global_catalog_ready","is_synchronized","ldap_service_name","naming_contexts","root_domain_naming_context","schema_naming_context","server_name","subschema_subentry","supported_capabilities","supported_control","supported_ldap_policies","supported_ldap_version","supported_sasl_mechanisms","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,389,node01.example.com,ldap-udp,64512,ZZ,Region,City,0,0,3038,"CN=Configuration,DC=ad,DC=example,DC=com",20220821044533.0Z,"DC=ad,DC=example,DC=com",node01.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,222537,TRUE,TRUE,node01.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5,58.42 -"2010-02-10 00:00:01",192.168.0.2,udp,389,node02.example.com,ldap-udp,64512,ZZ,Region,City,0,0,3062,"CN=Configuration,DC=ad,DC=example,DC=com",20220821044948.0Z,"DC=ad,DC=example,DC=com",node02.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,1478714,TRUE,TRUE,node02.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5,58.88 -"2010-02-10 00:00:02",192.168.0.3,udp,389,node03.example.com,ldap-udp,64512,ZZ,Region,City,0,0,36,"CN=Configuration,DC=ad,DC=example,DC=com",,"DC=ad,DC=example,DC=com",node03.example.com,,,"CN=Configuration,DC=ad,DC=example,DC=com",,,,,node03.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,,,,,0.69 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv deleted file mode 100644 index 4a97121e7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","mdns_name","mdns_ipv4","mdns_ipv6","services","workstation_name","workstation_ipv4","workstation_ipv6","workstation_info","http_name","http_ipv4","http_ipv6","http_ptr","http_info","http_target","http_port","spotify_name","spotify_ipv4","spotify_ipv6","opc_ua_discovery" -"2010-02-10 00:00:00",192.168.0.1,udp,5353,node01.example.com,mdns,64512,ZZ,Region,City,0,0,,,,"_smb._tcp.local.; _device-info._tcp.local.; _http._tcp.local.; _dacp._tcp.local.;",,192.168.0.1,fd09:4ab5:dae9:b078::1,,,192.168.0.1,fd09:4ab5:dae9:b078::1,,,,,,,, -"2010-02-10 00:00:01",192.168.0.2,udp,5353,node02.example.com,mdns,64512,ZZ,Region,City,0,0,,,,_home-assistant._tcp.local.;,,192.168.0.2,fd09:4ab5:dae9:b078::2,,,192.168.0.2,fd09:4ab5:dae9:b078::2,,,,,,,, -"2010-02-10 00:00:02",192.168.0.3,udp,5353,node03.example.com,"mdns,iot",64512,ZZ,Region,City,0,0,,,,"_webdav._tcp.local.; _adisk._tcp.local.; _smb._tcp.local.; _http._tcp.local.; _dacp._tcp.local.; _afpovertcp._tcp.local.; _device-info._tcp.local.;",,192.168.0.3,fd09:4ab5:dae9:b078::3,,snmeijer.local.,192.168.0.3,fd09:4ab5:dae9:b078::3,snmeijer._http._tcp.local.,"\"\"vendor=Synology\"\" \"\"model=DS218+\"\" \"\"serial=17A0PCN482002\"\" \"\"version_major=6\"\" \"\"version_minor=2\"\" \"\"version_build=25556\"\" \"\"admin_port=5000\"\" \"\"secure_admin_port=5001\"\" \"\"mac_address=00:11:32:80:fd:b5\"\"",snmeijer.local.,5000,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv deleted file mode 100644 index 6a1d445e7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","pid","pointer_size","uptime","time","curr_connections","total_connections","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,50260,node01.example.com,memcached,1.4.15,64512,ZZ,Region,City,0,0,1010,64,32908114,"2022-08-21 10:34:06",243,6106,"Communications, Service Provider, and Hosting Service",1144,81.71 -"2010-02-10 00:00:01",192.168.0.2,udp,11211,node02.example.com,memcached,1.4.13,64512,ZZ,Region,City,0,0,5316,64,9618498,"2022-08-21 10:39:21",9,2962,"Communications, Service Provider, and Hosting Service",1053,75.21 -"2010-02-10 00:00:02",192.168.0.3,udp,11211,node03.example.com,memcached,1.2.6,64512,ZZ,Region,City,0,0,1460,32,1375159,"2022-08-21 10:39:39",2,534,,442,31.57 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv deleted file mode 100644 index 1228dcfc6..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv +++ /dev/null @@ -1,11 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","gitversion","sysinfo","opensslversion","allocator","javascriptengine","bits","maxbsonobjectsize","ok","visible_databases","sector" -"2016-07-24 00:40:07","198.51.100.203","tcp",27017,"198-51-100-203.example.net","mongodb","2.4.5",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"a2ddc68ba7c9cee17bfe69ed840383ec3506602b","Linux ip-198-51-100-100 198.51.100.103-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49",,"tcmalloc","V8",64,16777216,1,"local | countly | admin", -"2016-07-24 00:40:07","198.51.100.42","tcp",27017,"198-51-100-208.example.net","mongodb","2.6.12",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"d73c92b1c85703828b55c2916a5dd4ad46535f6a","Linux build5.ny.cbi.10gen.cc 2.6.32-431.3.1.el6.x86_64 #1 SMP Fri Jan 3 21:39:27 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49",,"tcmalloc","V8",64,16777216,1,"none visible","Information Technology" -"2016-07-24 00:40:07","198.51.100.225","tcp",27017,"198-51-100-225.example.net","mongodb","3.0.6",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,"1ef45a23a4c5e3480ac919b28afcba3c615488f2","Linux ip-198-51-100-100 3.4.43-43.43.amzn1.x86_64 #1 SMP Mon May 6 18:04:41 UTC 2013 x86_64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.0-fips 29 Mar 2010","tcmalloc","V8",64,16777216,1,"bluu | local","Communications" -"2016-07-24 00:40:07","198.51.100.144","tcp",27017,"198-51-100-144.example.net","mongodb","2.2.2",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"d1b43b61a5308c4ad0679d34b262c5af9d664267","Linux ip-198-51-100-100 198.51.100.252-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49",,,,64,16777216,1,"errbit_production | DELETED_BECAUSE_YOU_DIDNT_PASSWORD_PROTECT_YOUR_MONGODB | admin | local", -"2016-07-24 00:40:07","198.51.100.68","tcp",27017,,"mongodb","3.2.6",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,"05552b562c7a0b3143a729aaa0838e558dc49b25","deprecated",,"tcmalloc","mozjs",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.101","tcp",27017,,"mongodb","3.0.9",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,"20d60d3491908f1ae252fe452300de3978a040c7","Linux ip-198-51-100-100 3.13.0-24-generic #46-Ubuntu SMP Thu Apr 10 19:11:08 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.1f 6 Jan 2014","tcmalloc","V8",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.53","tcp",27017,"198-51-100-162.example.net","mongodb","3.2.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"05552b562c7a0b3143a729aaa0838e558dc49b25","deprecated",,"tcmalloc","mozjs",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.206","tcp",27017,"198-51-100-206.example.net","mongodb","2.4.10",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"e3d78955d181e475345ebd60053a4738a4c5268a","Linux bs-linux32.10gen.cc 198.51.100.34-2.fc8xen #1 SMP Fri Feb 15 12:39:36 EST 2008 i686 BOOST_LIB_VERSION=1_49",,"system","V8",32,16777216,1,"sharelatex | test1 | local | tmp | lococms_production", -"2016-07-24 00:40:10","198.51.100.157","tcp",27017,"198-51-100-157.example.net","mongodb","2.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"nogitversion","Linux biber 2.6.32-5-amd64 #1 SMP Mon Feb 25 00:26:11 UTC 2013 i686 BOOST_LIB_VERSION=1_49",,,,32,16777216,1,"none visible", -"2016-07-24 00:40:10","198.51.100.173","tcp",27017,"198-51-100-173.example.net","mongodb","2.6.12",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"nogitversion","FreeBSD 101amd64-default-job-24 10.1-RELEASE-p33 FreeBSD 10.1-RELEASE-p33 amd64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.1l-freebsd 15 Jan 2015","system","V8",64,16777216,1,"none visible", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv deleted file mode 100644 index cfe4f0061..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv +++ /dev/null @@ -1,2 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","anonymous_access","raw_response","hex_code","code","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serialNumber" -"2022-02-07 12:56:53","18.220.0.0","tcp",8883,"18-220-0-0.example.com","mqtt",12345,"US","OHIO","COLUMBUS",454110,,"N",20020005,05,"Connection Refused, not authorized","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"*.tracesafe.io","Sectigo RSA Domain Validation Secure Server CA","2020-08-12 00:00:00","2022-11-14 00:00:00","70:84:F1:6D:28:DA:B6:E6:27:60:13:8B:2C:93:52:B6:7B:4B:13:7B","D2:D7:54:52:EB:86:4E:2D:34:4D:FC:CE:CD:CF:39:41:E1:06:5C:8B:B8:54:E6:0C:DF:FD:6E:E3:F1:B5:41:00","17:57:FB:88:9D:BE:A7:F0:29:A5:31:FC:79:DF:F7:8A:1C:D6:4A:DF:1B:4A:DC:BF:05:E7:E8:2F:79:9A:FA:FE:F7:E8:66:22:CB:B9:4C:72:F7:FB:6C:1D:59:8C:54:63:70:05:DE:7F:3C:2F:BA:B8:37:18:CE:29:6F:11:E8:AB","DE:2C:98:30:27:2E:7D:C9:ED:A3:9D:AF:9E:CE:14:CC","085699743A23114C9B6B8DC975A8AF42",2,"sha256WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,"Sectigo Limited",,"GB","Greater Manchester","Salford",,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license deleted file mode 100644 index 476908eeb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv deleted file mode 100644 index e0ab4b929..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","raw_response","hex_code","code","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serialNumber" -"2022-01-10 00:59:34","47.106.0.0","tcp",8883,,"mqtt,mqtt-anon",37963,"CN","GUANGDONG SHENG","SHENZHEN",518210,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"Server","RootCA","2020-05-08 08:07:05","2030-05-06 08:07:05","70:1A:1E:1F:EC:5F:7E:A9:12:32:B2:C9:8A:C9:EE:91:8E:0B:82:45","85:26:A2:F2:A2:50:CD:96:33:19:A6:2D:12:2E:97:6B:D3:06:3C:11:EA:01:B4:B7:25:2A:B7:4F:0A:8F:45:40","72:50:07:30:9A:6F:CB:FD:E2:80:69:02:65:62:77:16:C3:B4:0C:98:44:4E:D4:2C:AC:6B:AF:F8:9E:AB:51:C2:FA:A8:72:A3:45:DF:81:09:50:08:18:EB:03:34:FC:92:33:A7:12:46:FE:90:20:91:86:C5:4D:89:48:86:4C:CD","AB:A8:E0:2C:EF:AE:BF:9D:DD:FA:70:BA:2F:F2:CA:5C",02,2,"sha256WithRSAEncryption","rsaEncryption","EMQ",,"CN","hangzhou",,,,,,,,,"EMQ",,"CN","hangzhou",,,,,,,, -"2022-01-10 00:59:34","144.76.0.0","tcp",8883,,"mqtt,mqtt-anon",24940,"DE","SACHSEN-ANHALT","WERNIGERODE",518210,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"example.com","R3","2021-12-06 13:48:04","2022-03-06 13:48:03","20:0E:AC:E7:AF:07:8D:D3:16:7C:63:D1:B9:12:AD:1D:2C:F0:46:86","DD:7A:4C:8A:1D:66:1D:7C:F5:17:04:5B:A0:B4:C4:E0:80:58:44:B4:DB:A7:5E:61:AE:43:9D:85:4C:9E:DC:83","55:B6:3D:56:A4:39:6E:99:B6:AF:72:AF:4D:3C:7C:C5:A8:C5:4F:A1:79:92:D0:46:8A:A2:9B:2A:48:0D:00:68:39:F0:B8:67:B4:E0:88:51:2A:D7:55:46:83:BD:ED:1E:09:6E:DB:3D:21:E2:AA:DB:42:6A:33:45:1A:2A:DB:4C","23:99:39:C6:77:D8:9F:55:90:FC:A5:FB:BA:72:8B:42","06B25BEAD1F43266ABCFCDDE408D3544D04B",2,"sha256WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,"Lets Encrypt",,"US",,,,,,,,, -"2022-01-10 00:59:34","173.0.0.0","tcp",8883,"example.com","mqtt,mqtt-anon",5555,"US","CALIFORNIA","BURBANK",,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",2048,"foo.example.com","ClearView2Dev","2020-08-07 16:51:57","2030-08-05 16:51:57","32:4B:66:98:FA:5B:D2:D1:F2:53:83:21:19:11:5A:A9:BE:85:56:16","AE:0D:65:34:2F:51:F7:32:1E:DF:B1:DA:12:C7:6A:DE:42:B5:4B:FF:80:2C:E5:EF:99:F6:CC:01:4B:C9:77:68","44:C4:B8:19:FA:39:55:51:EC:E4:6D:C4:6D:0F:A5:46:BB:D5:F9:FD:A6:8D:DF:F3:2D:D2:92:6C:0B:D5:D3:25:CB:19:50:9D:A6:A4:D4:D3:2E:53:10:F5:8D:77:F7:90:F8:65:A7:79:AB:14:62:72:01:F3:EA:38:E2:68:C7:25","43:0D:A7:89:9E:76:8D:6E:D5:AD:95:CC:F2:91:87:56","A71541EFAE529B03",0,"sha256WithRSAEncryption","rsaEncryption","Sohonet",,,,"<",,,,,,,,"Sohonet","ClearView2Dev",,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv deleted file mode 100644 index c12a6063e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","server_name","instance_name","tcp_port","named_pipe","response_size","amplification","sector" -"2010-02-10 00:00:00",192.168.0.1,udp,1434,node01.example.com,mssql,13.2.5026.0,64512,ZZ,Region,City,0,0,ERPOPTIMA,OPTIMA,49729,"\\\\ERPOPTIMA\\pipe\\MSSQL$OPTIMA\\sql\\query",310,310.00, -"2010-02-10 00:00:01",192.168.0.2,udp,1434,node02.example.com,mssql,13.0.1601.5,64512,ZZ,Region,City,0,0,SERWER,MSSQLSERVER,1433,,226,226.00,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,udp,1434,node03.example.com,mssql,10.50.2500.0,64512,ZZ,Region,City,0,0,ILONY,INSERTGT,49358,"\\\\ILONY\\pipe\\MSSQL$INSERTGT\\sql\\query",304,304.00, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv deleted file mode 100644 index 25fed2166..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","mysql_protocol_version","server_version","error_code","error_id","error_message","client_can_handle_expired_passwords","client_compress","client_connect_attrs","client_connect_with_db","client_deprecated_eof","client_found_rows","client_ignore_sigpipe","client_ignore_space","client_interactive","client_local_files","client_long_flag","client_long_password","client_multi_results","client_multi_statements","client_no_schema","client_odbc","client_plugin_auth","client_plugin_auth_len_enc_client_data","client_protocol_41","client_ps_multi_results","client_reserved","client_secure_connection","client_session_track","client_ssl","client_transactions","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,3306,node01.example.com,mysql,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",10,5.7.37-0ubuntu0.18.04.1,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:01",192.168.0.2,tcp,3306,node02.example.com,mysql,,64512,ZZ,Region,City,0,0,,10,5.7.30-0ubuntu0.18.04.1-log,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:02",192.168.0.3,tcp,3306,node03.example.com,mysql,,64512,ZZ,Region,City,0,0,"Retail Trade",10,8.0.23,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv deleted file mode 100644 index e8a1108d5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","opcode","uptime","external_ip","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,5351,node01.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,291278940,192.168.0.1,,12,6.00 -"2010-02-10 00:00:01",192.168.0.2,udp,5351,node02.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,768416,192.168.0.2,,12,6.00 -"2010-02-10 00:00:02",192.168.0.3,udp,5351,node03.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,19629454,192.168.0.3,,12,6.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv deleted file mode 100644 index 932225b0b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","mac_address","asn","geo","region","city","workgroup","machine_name","username","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,137,node01.example.com,netbios,00-00-00-00-00-00,64512,ZZ,Region,City,,NBG6503,NBG6503,0,0,,229,4.58 -"2010-02-10 00:00:01",192.168.0.2,udp,137,node02.example.com,netbios,00-00-00-00-00-00,64512,ZZ,Region,City,PRACOWNIAELN.,NAS-OLD,NAS-OLD,0,0,,193,3.86 -"2010-02-10 00:00:02",192.168.0.3,udp,137,node03.example.com,netbios,00-25-90-F0-64-64,64512,ZZ,Region,City,HRSIGMA,HR-SRV01,,0,0,Government,157,3.14 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv deleted file mode 100644 index 4e9159356..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","response","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,53413,node01.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 -"2010-02-10 00:00:01",192.168.0.2,53413,node02.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 -"2010-02-10 00:00:02",192.168.0.3,53413,node03.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv deleted file mode 100644 index cc3cf6fc2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","version","clk_wander","clock","error","frequency","jitter","leap","mintc","noise","offset","peer","phase","poll","precision","processor","refid","reftime","rootdelay","rootdispersion","stability","state","stratum","system","tai","tc","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,123,node01.example.com,ntpversion,64512,ZZ,Region,City,4,,0xe6ac3809.363028e7,,2.018,0.977,0,,0.984,0.557,18986,,10,-10,unknown,81.15.252.130,0xe6ac35ba.2d2e8f2b,17.685,61.254,0.027,4,4,UNIX,,,0,0,,324,27.00 -"2010-02-10 00:00:01",192.168.0.2,udp,123,node02.example.com,ntpversion,64512,ZZ,Region,City,4,0.007,0xE6AC3806.7DF3B7A0,,-20.407,8.776,0,3,,-14.502,19244,,,-10,unknown,10.48.21.21,0xE6AC3431.B3B64790,32.25,105.778,,,8,UNIX,,10,0,0,"Transportation and Warehousing",328,27.33 -"2010-02-10 00:00:02",192.168.0.3,udp,123,node03.example.com,ntpversion,64512,ZZ,Region,City,4,0.001,0xE6AC380A.5A1CAD00,,-24.01,2.343,0,3,,0.49,51892,,,-10,unknown,172.28.0.1,0xE6AC3020.0C49BA80,7.749,81.612,,,4,UNIX,,10,0,0,,324,27.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv deleted file mode 100644 index dca5386d9..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","packets","size","asn","geo","region","city","naics","sic","sector","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,123,node01.example.com,2,664,64512,ZZ,Region,City,0,0,,55.33 -"2010-02-10 00:00:01",192.168.0.2,udp,123,node02.example.com,100,44000,64512,ZZ,Region,City,0,0,,3666.67 -"2010-02-10 00:00:02",192.168.0.3,udp,123,node03.example.com,100,44000,64512,ZZ,Region,City,0,0,,3666.67 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv deleted file mode 100644 index c32bc3d4d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","programs","mountd_port","exports","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,111,node01.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0","Communications, Service Provider, and Hosting Service",148,3.70 -"2010-02-10 00:00:01",192.168.0.2,udp,111,node02.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0","Communications, Service Provider, and Hosting Service",148,3.70 -"2010-02-10 00:00:02",192.168.0.3,udp,111,node03.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0",Government,148,3.70 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv deleted file mode 100644 index 8c1d6f725..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","supported_protocols","protocol_error_code","protocol_error_file","protocol_error_line","protocol_error_message","protocol_error_routine","protocol_error_severity","protocol_error_severity_v","startup_error_code","startup_error_file","startup_error_line","startup_error_message","startup_error_routine","startup_error_severity","startup_error_severity_v","client_ssl","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,5432,node01.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, -"2010-02-10 00:00:01",192.168.0.2,tcp,5432,node02.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, -"2010-02-10 00:00:02",192.168.0.3,tcp,5432,node03.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv deleted file mode 100644 index 857699376..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","quote","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,17,node01.example.com,qotd,"_The secret of being miserable is to have leisure to bother about whether?? you are happy or not. The cure for it is occupation._?? George Bernard Shaw (1856-1950)?",64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",166,166.00 -"2010-02-10 00:00:01",192.168.0.2,udp,17,node02.example.com,qotd,"_Oh the nerves, the nerves; the mysteries of this machine called man!?? Oh the little that unhinges it, poor creatures that we are!_?? Charles Dickens (1812-70)?",64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",162,162.00 -"2010-02-10 00:00:02",192.168.0.3,udp,17,node03.example.com,qotd,"_Oh the nerves, the nerves; the mysteries of this machine called man!?? Oh the little that unhinges it, poor creatures that we are!_?? Charles Dickens (1812-70)?",64512,ZZ,Region,City,0,0,,162,162.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv deleted file mode 100644 index c9fb18896..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","version_field_1","version_field_2","version_field_3","version_field_4" -"2022-01-10 14:31:17","176.255.0.0","udp",443,"test1.example.com","quic",5607,"UK","LONDON","LONDON",517311,,"Q050",,"Q046","Q043" -"2022-01-10 14:31:17","24.244.0.0","udp",443,,"quic",6327,"CA","SASKATCHEWAN","MEACHAM",517311,,"Q050","Q046",,"Q043" -"2022-01-10 14:31:17","23.60.0.0","udp",443,"test3.example.com","quic",20940,"JP","OSAKA","OSAKA",517919,,,"Q050","Q046","Q043" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv deleted file mode 100644 index 76b388aca..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv +++ /dev/null @@ -1,10 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic" -"2020-07-06 13:55:26","74.101.218.75","tcp",4899,"static-74-101-218-75.nycmny.fios.verizon.net","radmin","Radmin (Details Unknown)",701,"US","NEW YORK","BROOKLYN",517312, -"2020-07-06 13:55:27","192.162.189.171","tcp",4899,"rubin.an.ru","radmin","Radmin v3.X Radmin Authentication",56618,"RU","MURMANSKAYA OBLAST","MURMANSK",0, -"2020-07-06 13:55:27","111.197.143.69","tcp",4899,,"radmin","Radmin (Details Unknown)",4808,"CN","BEIJING SHI","BEIJING",517311, -"2020-07-06 13:55:27","121.147.215.220","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",4766,"KR","GWANGJU-GWANGYEOKSI","DAEIN-DONG",517311, -"2020-07-06 13:55:27","121.147.215.178","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",4766,"KR","GWANGJU-GWANGYEOKSI","DAEIN-DONG",517311, -"2020-07-06 13:55:27","183.230.5.219","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",9808,"CN","CHONGQING SHI","CHONGQING",517312, -"2020-07-06 13:55:27","85.93.154.74","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",34300,"RU","MOSKVA","MOSCOW",0, -"2020-07-06 13:55:27","81.246.135.247","tcp",4899,"247.135-246-81.adsl-dyn.isp.belgacom.be","radmin","Radmin v3.X Radmin Authentication",5432,"BE","ANTWERPEN","BRASSCHAAT",517311, -"2020-07-06 13:55:27","46.27.146.22","tcp",4899,"static-22-146-27-46.ipcom.comunitel.net","radmin","Radmin v3.X Radmin Authentication",12430,"ES","LAS PALMAS","LAS PALMAS DE GRAN CANARIA",517312, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license deleted file mode 100644 index 833024a75..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 sinus-x -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv deleted file mode 100644 index 4bac90f19..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","rdp_protocol","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","naics","sic","sector","tlsv13_support","tlsv13_cipher","cve20190708_vulnerable","bluekeep_vulnerable","jarm" -"2019-09-04 15:45:51","198.123.245.178",5678,,"rdp",,5678,"AA","LOCATION","LOCATION","RDP",5678,"KABESRV.KABE.local","KABESRV.KABE.local","2019-04-29 02:22:06","2019-10-29 02:22:06","EC:BB:4D:DB:9F:0C:D3:FF:5B:49:EA:B1:56:62:B6:A7:5D:60:54:42","1EF2B37AF850C9BF4E88F18177001D6B",2,"sha256WithRSAEncryption","rsaEncryption","B7:C9:F4:07:D5:C0:75:1D:EA:0C:40:E7:26:39:C2:30:C6:13:83:7E:18:46:D8:E9:4C:45:3F:88:1B:0B:70:76","08:AC:75:FA:EB:A3:2B:44:15:DE:6D:A7:0B:C0:AE:17:94:F3:55:D9:EC:70:AC:5B:B7:94:79:F0:D7:84:83:89:CB:A9:11:E0:08:D7:54:4D:33:85:89:D2:A8:DD:9D:15:F4:CC:95:DE:6A:E3:DF:6B:FA:8B:27:E3:DA:16:AF:0A","BC:6E:C3:E2:98:22:EC:BA:5B:30:E2:53:FD:4A:9D:FF",517311,0,,,,"N","N" -"2019-09-04 15:45:51","198.123.245.233",5678,,"rdp",,5678,"AA","LOCATION","LOCATION","RDP",5678,"RAMBLA01.rambla.local","RAMBLA01.rambla.local","2019-04-16 06:15:20","2019-10-16 06:15:20","7A:67:1F:F8:87:C6:B0:AC:A9:84:15:B7:40:EC:CB:19:AA:E3:19:52","3FF3EBC5CF154BA54D128A8548C8AAF5",2,"sha1WithRSAEncryption","rsaEncryption","8F:CD:7D:C4:80:2D:8D:9B:06:A0:40:18:9F:ED:73:7A:BA:83:55:BE:1B:56:83:A2:97:DF:BB:B4:06:57:CB:F1","E8:9B:9A:93:69:B4:58:01:D8:46:C2:DC:01:20:1E:DD:93:E1:EB:E3:9D:6B:65:A0:C5:00:6C:A4:44:08:FE:A4:A6:19:FF:55:79:F2:AA:61:68:C8:1C:B0:CE:78:EB:84:DD:29:9D:64:2F:4E:25:31:3A:6C:B8:02:C9:AF:F5:1F","38:73:6A:B3:AA:41:69:C9:BA:E7:3D:D7:40:16:F8:AA",517311,0,"Information Technology",,,"N","N" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv deleted file mode 100644 index 73d0d55ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sessionid","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,3389,node01.example.com,rdpeudp,64512,ZZ,Region,City,0,0,05b28c0c,1232,77.00 -"2010-02-10 00:00:01",192.168.0.2,udp,3389,node02.example.com,rdpeudp,64512,ZZ,Region,City,0,0,053d355f,1232,77.00 -"2010-02-10 00:00:02",192.168.0.3,udp,3389,node03.example.com,rdpeudp,64512,ZZ,Region,City,0,0,0567a8cb,1232,77.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv deleted file mode 100644 index dc9760cf2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv +++ /dev/null @@ -1,94 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","git_sha1","git_dirty_flag","build_id","mode","os","architecture","multiplexing_api","gcc_version","process_id","run_id","uptime","connected_clients","sector" -"2016-07-24 00:42:33","198.51.100.152","tcp",6379,,"redis","2.8.19",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"26069fb482f6334b","standalone","Linux 3.2.0-4-amd64 x86_64",,"epoll","4.7.2",2127,"d440b0b2fb3d1db655ad607e11e6f38011a0f599",27946314,50, -"2016-07-24 00:42:43","198.51.100.67","tcp",6379,"198-51-100-67.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310556,25376, -"2016-07-24 00:42:43","198.51.100.125","tcp",6379,"198-51-100-125.example.net","redis","2.8.17",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5937320cdd80c1e4","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.9.2",11573,"0d58143df099738a7ce9330ee5ec2367d11b1187",25888041,4, -"2016-07-24 00:42:43","198.51.100.203","tcp",6379,"198-51-100-203.example.net","redis","2.8.4",31103,"DE","THURINGEN","ERFURT",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-83-generic x86_64",,"epoll","4.8.2",3847,"4f7765dee91d8c4b1b24604cc5f0c29fca1a4f32",3068554,38, -"2016-07-24 00:42:43","198.51.100.240","tcp",6379,"198-51-100-30.example.net","redis","3.0.7",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.4",1011,"864c8d7df1e72c662a4edd77b6df6cd30161af6e",2476542,2,"Information Technology" -"2016-07-24 00:42:49","198.51.100.69","tcp",6379,"198-51-100-69.example.net","redis","3.0.6",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"315c8c74805fca88","standalone","Linux 3.2.0-98-generic x86_64",,"epoll","4.6.3",28961,"bc705102c854ea1818213e4740a3c6fd9b9f1716",4633191,1, -"2016-07-24 00:42:53","198.51.100.50","tcp",6379,"198-51-100-50.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"6afb1e1f0d80abd0","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.4.5",1717,"f729595b3642b48f3ac9e098bcccab1d6ef82e3e",6345372,3, -"2016-07-24 00:43:49","198.51.100.113","tcp",6379,,"redis","3.0.6",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310623,24628, -"2016-07-24 00:43:49","198.51.100.228","tcp",6379,"198-51-100-131.example.net","redis","2.8.210",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"83ad777ec89a946b","standalone","Windows",,"winsock_IOCP",,1948,"f5d6ad26e423039636afaf3918ee7e6a7e0b5b68",2214134,4,"Information Technology" -"2016-07-24 00:43:59","198.51.100.155","tcp",6379,,"redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"f09a0843cc9876c3","standalone","Linux 3.13.0-79-generic x86_64",,"epoll","4.9.2",1,"5f4f5b7158f928cc96e3ae6af6092a163ace15eb",2897902,24, -"2016-07-24 00:43:59","198.51.100.171","tcp",6379,,"redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310633,25031, -"2016-07-24 00:44:09","198.51.100.230","tcp",6379,"198-51-100-230.example.net","redis","2.8.11",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"f26bfdf4b8265fc","standalone","Linux 3.8.0-29-generic x86_64",,"epoll","4.6.3",14551,"0d001175cf26cee88486d814b4f0c972a5aa89b9",21038337,9, -"2016-07-24 00:44:09","198.51.100.182","tcp",6379,"198-51-100-182.example.net","redis","3.0.7",197540,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"fd24f54fec00684b","standalone","Linux 3.13.0-85-generic x86_64",,"epoll","4.8.4",949,"b11fdf2b95251b8e6c3e9e782409ef82fc8b89aa",8643389,11, -"2016-07-24 00:44:10","198.51.100.23","tcp",6379,"198-51-100-116.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 4.2.0-27-generic x86_64",,"epoll","4.8.2",335,"90079d58e970a1ae94aa91bc0ea0236a0e55269c",4930922,2,"Information Technology" -"2016-07-24 00:44:19","198.51.100.51","tcp",6379,"198-51-100-51.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310652,26257, -"2016-07-24 00:44:22","198.51.100.88","tcp",6379,,"redis","3.0.6",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310656,26371, -"2016-07-24 00:44:22","198.51.100.107","tcp",6379,"octopus-dev","redis","2.8.14",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"78be6d5e32e34139","standalone","Linux 2.6.32-042stab108.2 x86_64",,"epoll","4.8.2",21205,"b98a41b6ea690c207527587f60bff1f1d24236b4",9364864,4, -"2016-07-24 00:44:22","198.51.100.75","tcp",6379,,"redis","3.0.0",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"2b5201a6bfd5f75e","standalone","Linux 3.11.0-19-generic x86_64",,"epoll","4.8.2",832,"2bdcda8b3b59cef244785b58935d68daf48645be",6745479,5, -"2016-07-24 00:44:25","198.51.100.12","tcp",6379,,"redis","3.0.6",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.13.0-79-generic x86_64",,"epoll","4.8.4",899,"94550e510bf770aa315cc3983ce9958853c77cfe",7816856,9, -"2016-07-24 00:44:27","198.51.100.13","tcp",6379,"198-51-100-13.example.net","redis","3.0.7",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"6f8b503a2787e3a6","standalone","Linux 4.4.5-15.26.amzn1.x86_64 x86_64",,"epoll","4.9.2",1,"e050f40e755a739ffecdb2468e1333f371e2abca",7124048,6,"Communications" -"2016-07-24 00:44:29","198.51.100.12","tcp",6379,"198-51-100-12.example.net","redis","2.8.3",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"992c97be25a6b6d2","standalone","Linux 2.6.32-042stab111.12 x86_64",,"epoll","4.4.5",12340,"d7cda18212cf4bcdfd7c42fff33e506a4e9a2614",16874891,8, -"2016-07-24 00:44:38","198.51.100.66","tcp",6379,"198-51-100-66.example.net","redis","3.2.1",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ed627d97d5dc311e","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"4a6beb721ddbaa411f53e5268e6112127903cae3",2029470,3,"Chemical" -"2016-07-24 00:44:38","198.51.100.170","tcp",6379,,"redis","3.0.6",8881,"DE","SACHSEN","RADEBEUL",0,0,00000000,0,"1b14d17ce6fea422","standalone","Linux 4.2.6-1-pve x86_64",,"epoll","4.9.2",728,"c423ba856285690a2fae350b03514cec80db9d5e",1679635,1, -"2016-07-24 00:44:38","198.51.100.67","tcp",6379,"198-51-100-67.example.net","redis","2.8.23",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"8e819a849ea2d7f8","standalone","Linux 4.2.0-23-generic x86_64",,"epoll","4.9.2",1,"7ee1dc403540ff4d1fc0a80d9f0b2910857b6c1b",9451832,68,"Information Technology" -"2016-07-24 00:44:44","198.51.100.238","tcp",6379,,"redis","2.8.4",44066,"DE","BAYERN","MUNICH",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 2.6.32-19-pve x86_64",,"epoll","4.8.2",2207,"6a079396cc44c1aca745edab13f4014c394da3ab",10338949,3, -"2016-07-24 00:44:44","198.51.100.84","tcp",6379,"198-51-100-84.example.net","redis","3.0.2",51862,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"4795df119e2d77fe","standalone","Linux 3.13.0-91-generic x86_64",,"epoll","4.7.2",1,"c120481a551c232b8e1a9cff20d9e0968a402dd9",1040551,7, -"2016-07-24 00:44:44","198.51.100.23","tcp",6379,"198-51-100-23.example.net","redis","3.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"98c227055d7fa7b6","standalone","Linux 3.10.0-327.10.1.el7.x86_64 x86_64",,"epoll","4.8.5",35198,"424b15e04ce09f26299ff19b252a920916d4e4be",8875355,2, -"2016-07-24 00:44:47","198.51.100.160","tcp",6379,"198-51-100-160.example.net","redis","2.8.210",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"83ad777ec89a946b","standalone","Windows",,"winsock_IOCP",,2284,"9bde76afda6f81acfb241ea5ee3a9e878ad53881",742778,2, -"2016-07-24 00:44:47","198.51.100.111","tcp",6379,"198-51-100-98.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e19bb8c3d1c28291","standalone","Linux 3.10.0-327.22.2.el7.x86_64 x86_64",,"epoll","5.3.0",1,"c951371f430c1d94299bfc93759f6940d8bfce78",208557,2, -"2016-07-24 00:44:48","198.51.100.227","tcp",6379,"198-51-100-227.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310681,26496, -"2016-07-24 00:44:54","198.51.100.18","tcp",6379,"198-51-100-18.example.net","redis","2.8.9",8972,"DE","NORDRHEIN-WESTFALEN","COLOGNE",0,0,00000000,0,"52c7b9284559eb20","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.4.5",31887,"e5b1da35862482c4df8d4fce635ec89a36476a4d",14393072,6, -"2016-07-24 00:44:54","198.51.100.248","tcp",6379,"198-51-100-248.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310687,26112, -"2016-07-24 00:44:57","198.51.100.228","tcp",6379,"198-51-100-228.example.net","redis","3.0.7",8972,"DE","NORDRHEIN-WESTFALEN","COLOGNE",0,0,00000000,0,"5e03212a543f54f8","standalone","Linux 3.13.0-042stab116.1 x86_64",,"epoll","4.8.4",719,"537e3e824a45414c3199ef20201b4362b752eeb5",1263367,2, -"2016-07-24 00:45:04","198.51.100.227","tcp",6379,"198-51-100-227.example.net","redis","2.8.12",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ff040dde4a39b4ff","standalone","Windows",,"winsock_IOCP","0.0.0",1872,"c78751c65793a9a72f6fb0318efa532eb4fc87de",277953,18,"Chemical" -"2016-07-24 00:45:07","198.51.100.132","tcp",6379,,"redis","3.0.5",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"30405cba8f6c2d55","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",2500,"10b4084b930d5a77e5f09e89cf0b21702027bd60",10028956,695, -"2016-07-24 00:46:10","198.51.100.47","tcp",6379,"198-51-100-185.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"6a943c0b5bf37fa1","standalone","Linux 4.4.0-24-generic x86_64",,"epoll","5.3.1",1023,"de9c9c0da3d971f689bd7366c1edc93a00fd1506",2791106,1, -"2016-07-24 01:23:27","198.51.100.246","tcp",6379,"198-51-100-190.example.net","redis","2.8.19",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"665519ce00ddac9b","standalone","Linux 2.6.32-43-pve x86_64",,"epoll","4.9.2",2310,"94595838457eddb30a60184a9db66212268e6f82",9481199,4, -"2016-07-24 01:23:29","198.51.100.187","tcp",6379,"198-51-100-63.example.net","redis","2.8.19",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"c0359e7aa3798aa2","standalone","Linux 3.10.0-229.7.2.el7.x86_64 x86_64",,"epoll","4.8.3",14050,"e67a19de4bd2dc485b98ca353eb6fdc65e8fed4a",14051444,10, -"2016-07-24 01:23:29","198.51.100.228","tcp",6379,"198-51-100-228.example.net","redis","2.8.4",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-32-generic x86_64",,"epoll","4.8.2",22837,"daf5dba760d3db12716c6dc1d0bfe6d5e7b33749",10916038,8, -"2016-07-24 01:23:43","198.51.100.180","tcp",6379,"198-51-100-180.example.net","redis","3.2.1",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ed627d97d5dc311e","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"569881874d8d5e1508d584a3fd9dff0ac3515839",1677711,1,"Chemical" -"2016-07-24 01:23:56","198.51.100.5","tcp",6379,"198-51-100-207.example.net","redis","3.0.7",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.4",1011,"864c8d7df1e72c662a4edd77b6df6cd30161af6e",2479015,2,"Information Technology" -"2016-07-24 01:24:03","198.51.100.226","tcp",6379,"198-51-100-226.example.net","redis","3.0.5",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"b33bc3e2f8ad13f6","standalone","Linux 2.6.32-573.12.1.el6.x86_64 x86_64",,"epoll","4.4.7",1801,"7f4bb7ed008cdbd665672e88d57fc55616b6dbf2",13189200,9, -"2016-07-24 01:24:14","198.51.100.253","tcp",6379,"198-51-100-136.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.19.0-39-generic x86_64",,"epoll","4.8.2",28272,"13a889aa846c6302dc8f5453e35e051a6f359e9a",14046610,185, -"2016-07-24 01:24:28","198.51.100.206","tcp",6379,,"redis","3.0.6",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313061,26695, -"2016-07-24 01:24:35","198.51.100.73","tcp",6379,,"redis","3.0.2",28753,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"6be7fc9e6b88f79","standalone","Linux 2.6.32-71.29.1.el6.x86_64 x86_64",,"epoll","4.4.7",811,"bbd4dce247ab51d029a64243810aeb900e00d1d6",10082205,15, -"2016-07-24 01:24:35","198.51.100.83","tcp",6379,"198-51-100-174.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"7e7b61a55b95e8e7","standalone","Linux 4.2.0-41-generic x86_64",,"epoll","4.8.4",1076,"48f5f780ca53553fc4c0bbdbb32a5cb06a0551cd",814255,88,"Information Technology" -"2016-07-24 01:25:30","198.51.100.182","tcp",6379,,"redis","3.0.7",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM",0,0,00000000,0,"d9ceac045f7983a9","standalone","FreeBSD 10.1-RELEASE-p26 amd64",,"kqueue","4.2.1",957,"48f37d15b3f5169f11aa5d7194fdfccc7f8df20b",6364747,1, -"2016-07-24 01:25:30","198.51.100.211","tcp",6379,"198-51-100-118.example.net","redis","2.8.17",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e4968abcd4b78b2e","standalone","Linux 3.13.0-36-generic x86_64",,"epoll","4.8.2",1643,"665565b1b1fb6e773039707a0f680bbc417186be",20180649,4,"Information Technology" -"2016-07-24 01:25:35","198.51.100.249","tcp",6379,,"redis","3.0.2",28753,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"6be7fc9e6b88f79","standalone","Linux 2.6.32-71.29.1.el6.x86_64 x86_64",,"epoll","4.4.7",811,"bbd4dce247ab51d029a64243810aeb900e00d1d6",10082265,15, -"2016-07-24 01:25:40","198.51.100.55","tcp",6379,,"redis","3.2.1",3320,"DE","NORDRHEIN-WESTFALEN","SOLINGEN",518210,737415,00000000,0,"e19bb8c3d1c28291","standalone","Linux 4.4.0-24-generic x86_64",,"epoll","5.3.0",1,"49687ba2a5be7f7b6cdf0c837e06307442f6a369",494739,1, -"2016-07-24 01:25:42","198.51.100.62","tcp",6379,"198-51-100-62.example.net","redis","3.0.7",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"2b87841ee28adfc3","standalone","Linux 3.13.0-042stab113.11 x86_64",,"epoll","4.8.4",525,"4045d68fd2e59a1135bb303206d7cd0439ba7ffd",6971251,4, -"2016-07-24 01:25:55","198.51.100.127","tcp",6379,"198-51-100-25.example.net","redis","2.8.4",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.2",11492,"3de3e977405eef9392a77db4a50d99a5caa2f2d9",2194103,3,"Information Technology" -"2016-07-24 01:26:08","198.51.100.92","tcp",6379,"198-51-100-92.example.net","redis","2.8.10",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5fce0c4aab65e01","standalone","Linux 2.6.32-042stab113.11 x86_64",,"epoll","4.6.3",490,"15abe68a10b011972f50d0abb3bb18f1735994a5",7505621,4, -"2016-07-24 01:26:17","198.51.100.218","tcp",6379,,"redis","3.0.7",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"dc142e699f115c40","standalone","Linux 3.2.60-grsec-x86_64 x86_64",,"epoll","4.7.3",8006,"53a093bd4d0a7b72b2d084ec3767d23b18b8b947",4024979,7, -"2016-07-24 01:26:29","198.51.100.168","tcp",6379,"198-51-100-168.example.net","redis","3.0.6",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.13.0-37-generic x86_64",,"epoll","4.8.4",1279,"8218bd77a0dcb0e00bd77dbb9478115757c70ba5",2405965,1, -"2016-07-24 01:26:29","198.51.100.155","tcp",6379,"198-51-100-155.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"d9155128f7b25ea0","standalone","Linux 3.19.0-25-generic x86_64",,"epoll","4.8.4",27030,"0ede623cb268643672abc04d0267f684a5ee7a0d",6880190,5,"Information Technology" -"2016-07-24 01:26:34","198.51.100.185","tcp",6379,,"redis","2.8.4",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-43-generic x86_64",,"epoll","4.8.2",1196,"ae80fcbb54017f521212caf257418885cd6836a0",5412584,5, -"2016-07-24 01:26:34","198.51.100.1","tcp",6379,"198-51-100-1.example.net","redis","3.2.0",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"5382f69a4e75566b","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"ff8990f109ff5b2d4e0eee47e5ebc66acc43f9e3",4615889,4,"Chemical" -"2016-07-24 01:26:39","198.51.100.51","tcp",6379,"198-51-100-164.example.net","redis","3.0.0",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"9526f4809583faaa","standalone","Linux 2.6.32-042stab113.21 x86_64",,"epoll","4.4.5",14528,"d7271feff55175f434ace92d199f332ad35776a9",7440370,16, -"2016-07-24 01:26:44","198.51.100.138","tcp",6379,,"redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313197,26452, -"2016-07-24 01:26:47","198.51.100.16","tcp",6379,,"redis","2.8.17",25074,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5937320cdd80c1e4","standalone","Linux 2.6.32-43-pve x86_64",,"epoll","4.9.2",266,"e1d403f2daff849a64b178f74c672db6712f217a",351253,1, -"2016-07-24 01:26:54","198.51.100.171","tcp",6379,"198-51-100-171.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313207,26601, -"2016-07-24 01:27:14","198.51.100.89","tcp",6379,"198-51-100-89.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313227,26358, -"2016-07-24 01:27:24","198.51.100.65","tcp",6379,,"redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-85-generic x86_64",,"epoll","4.8.4",21575,"3ec40168300e14f5776d82a48ba873a3999caec1",1897530,1, -"2016-07-24 01:27:24","198.51.100.248","tcp",6379,"198-51-100-248.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313237,25902, -"2016-07-24 01:27:33","198.51.100.17","tcp",6379,,"redis","2.8.17",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"43dd9e14444e6aea","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",556,"3e8fc2878511cc72f79b765fca86cefe21346912",2607965,72, -"2016-07-24 01:27:33","198.51.100.134","tcp",6379,"198-51-100-134.example.net","redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"6f8b503a2787e3a6","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",1,"b85b2419cf35dd81ff5b9ba6e8bf802cf1d439f6",128621,33, -"2016-07-24 01:27:42","198.51.100.186","tcp",6379,"198-51-100-186.example.net","redis","2.8.13",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"d588bf57ea0dfa69","standalone","Linux 4.4.8-jb1 i686",,"epoll","4.6.3",2460,"97b8d49e62d340d94a38c96c5104abfcacbfa4cb",181557,1, -"2016-07-24 01:27:42","198.51.100.21","tcp",6379,"198-51-100-21.example.net","redis","2.8.19",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"920d7eda78149e99","standalone","Linux 4.4.8-x86_64-jb1 x86_64",,"epoll","4.7.2",3722,"74dfd8a7d87cbb9ecc590ceafd438c85d5073903",183984,1, -"2016-07-24 01:27:43","198.51.100.128","tcp",6379,"198-51-100-203.example.net","redis","3.0.5",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"f3bd5bc2b8b4c486","standalone","Linux 2.6.32-573.8.1.el6.x86_64 x86_64",,"epoll","4.4.7",1968,"0d92b1323fea791ba4b0a43435a156b6ec0aac1c",2967611,2,"Information Technology" -"2016-07-24 01:27:44","198.51.100.216","tcp",6379,"198-51-100-229.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.16.0-30-generic x86_64",,"epoll","4.8.2",1470,"e76cd0cf25eec5d254c880965189ae011a119220",302420,1, -"2016-07-24 01:27:53","198.51.100.242","tcp",6379,"198-51-100-242.example.net","redis","3.0.2",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"6a04b5ede30cd4cd","standalone","Linux 3.13.0-32-generic x86_64",,"epoll","4.8.4",29725,"1b7e8dc53dec8fb29a8a2d76f516fd3dcb8df652",5815739,7, -"2016-07-24 01:27:53","198.51.100.54","tcp",6379,"198-51-100-54.example.net","redis","2.8.4",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-91-generic x86_64",,"epoll","4.8.2",2903,"0e02514dec6031018eb148b13a4a9639cab3e8aa",905886,1, -"2016-07-24 01:27:54","198.51.100.225","tcp",6379,"198-51-100-225.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313267,25281, -"2016-07-24 01:27:57","198.51.100.38","tcp",6379,"198-51-100-38.example.net","redis","3.0.5",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"3b863f97501297e9","standalone","Linux 3.13.0-042stab111.12 x86_64",,"epoll","4.8.4",2088,"31a8cececad2e4a33310a741143d85cdef3479b4",11906868,10, -"2016-07-24 01:27:58","198.51.100.22","tcp",6379,"198-51-100-22.example.net","redis","2.8.9",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"2ac6afaedfd3ea15","standalone","Linux 3.13.0-86-generic x86_64",,"epoll","4.8.4",9082,"8e5d9d74c86a9f148a7012733eb52a21938c3c04",5833880,5, -"2016-07-24 01:28:05","198.51.100.106","tcp",6379,"198-51-100-106.example.net","redis","2.8.19",36351,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"9968db13395be4aa","standalone","Windows",,"winsock_IOCP","0.0.0",4372,"89716352a10cd53b5c10e6d5e6cd1d46f5f53a30",485031,4,"Information Technology" -"2016-07-24 01:28:06","198.51.100.130","tcp",6379,"198-51-100-130.example.net","redis","2.8.3",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"542faa6f897d2236","standalone","Linux 2.6.32-573.3.1.el6.x86_64 x86_64",,"epoll","4.4.7",25531,"9d7606a883f764e744d766b7bf0036ba61f7fb6e",496133,5, -"2016-07-24 01:28:08","198.51.100.37","tcp",6379,"198-51-100-37.example.net","redis","2.8.23",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"50630e46be5feb4f","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.9.2",1,"62d16be721c3c62d6c4d080a9bdbe9502c57ca86",3481683,9,"Communications" -"2016-07-24 01:28:32","198.51.100.148","tcp",6379,"198-51-100-148.example.net","redis","3.0.5",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"83dc15dcf8ee3eb8","standalone","Linux 4.1.7-15.23.amzn1.x86_64 x86_64",,"epoll","4.8.3",2304,"883accf76dc364c60902b4eab7861dd1a7eac71d",10981957,10,"Communications" -"2016-07-24 01:28:49","198.51.100.247","tcp",6379,"198-51-100-247.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"3e971e94fbe2eaa6","standalone","Linux 3.2.0-4-amd64 x86_64",,"epoll","4.7.2",2535,"d223aab0621cdd2e4ab752978ad3009ad3814d8b",7715188,57, -"2016-07-24 02:08:46","198.51.100.220","tcp",6379,"198-51-100-220.example.net","redis","3.0.6",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"1f8e4c92f1ca309","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.8.4",3355,"dd517756bb6ee81e1929fa605972318b2baebb93",5211978,10, -"2016-07-24 02:08:46","198.51.100.239","tcp",6379,"198-51-100-239.example.net","redis","2.8.23",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"83a5616190c5a1aa","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",711,"4117960b13fa313b823c79b0e9f188d8ec6aa3ac",10156283,6, -"2016-07-24 02:08:50","198.51.100.233","tcp",6379,,"redis","2.8.11",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"f26bfdf4b8265fc","standalone","Linux 3.8.0-29-generic x86_64",,"epoll","4.6.3",14551,"0d001175cf26cee88486d814b4f0c972a5aa89b9",21043417,9, -"2016-07-24 02:08:51","198.51.100.208","tcp",6379,"198-51-100-181.example.net","redis","3.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 4.2.0-38-generic x86_64",,"epoll","4.8.4",809,"14c5ec7f9669e42ea45a40ff26a6501d593695c0",2405839,19, -"2016-07-24 02:08:51","198.51.100.60","tcp",6379,"198-51-100-60.example.net","redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"4ed99bd9c45dfc14","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",1144,"9e28c29ff40017e2fbe32fb97755caf801f95793",843538,2, -"2016-07-24 02:08:51","198.51.100.107","tcp",6379,"198-51-100-39.example.net","redis","3.2.0",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"82b2619163aabc80","standalone","Linux 4.2.0-25-generic x86_64",,"epoll","4.9.2",1,"98f6640bbde04b1214730937212e1fd4e58d03a8",2195657,12, -"2016-07-24 02:08:54","198.51.100.31","tcp",6379,,"redis","2.8.4",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-042stab111.12 x86_64",,"epoll","4.8.2",1112,"9c4e55b5ebd06045c5d89d43fa202e219ec8b42c",8839783,7, -"2016-07-24 02:08:56","198.51.100.221","tcp",6379,,"redis","3.0.7",44066,"DE","BAYERN","MUNICH",0,0,00000000,0,"49f951dce0725d71","standalone","FreeBSD 10.0-RELEASE-p7 amd64",,"kqueue","4.2.1",932,"28c6af3c4dedcd9b71cf51a7ebc4e84899196aee",8000949,1, -"2016-07-24 02:09:01","198.51.100.155","tcp",6379,"198-51-100-155.example.net","redis","2.8.22",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"fcdf45e47686c89b","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",7,"946ec6b96fe9925d2b677ce02b6c56097c5e69a8",8449694,6, -"2016-07-24 02:09:02","198.51.100.219","tcp",6379,"198-51-100-219.example.net","redis","2.8.4",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.8.2",1047,"9b83d6a6e7a6ffe50e75dac88cdc5e06f6203c9c",966148,1,"Chemical" -"2016-07-24 02:09:02","198.51.100.193","tcp",6379,"198-51-100-193.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"fd640d8ef55a22dd","standalone","Linux 4.2.0-42-generic x86_64",,"epoll","4.8.4",1397,"ed5ec17d78d089af53afd4abc339f7decf4641d4",651175,2,"Information Technology" -"2016-07-24 02:09:20","198.51.100.120","tcp",6379,"198-51-100-120.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"ed627d97d5dc311e","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",1,"f524508ad29334eee2fcf7bdda5c80b9f99d3dfe",987580,167, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv deleted file mode 100644 index a61e4573e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","module","motd","has_password" -"2010-02-10 00:00:00",192.168.0.1,tcp,873,node01.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N -"2010-02-10 00:00:01",192.168.0.2,tcp,873,node02.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N -"2010-02-10 00:00:02",192.168.0.3,tcp,873,node03.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv deleted file mode 100644 index ee0a625e5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","sip","sip_code","sip_reason","user_agent","sip_via","sip_to","sip_from","content_length","content_type","server","contact","cseq","call_id","allow","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,5060,node01.example.com,sip,64512,ZZ,Region,City,SIP/2.0,489,"Event Package Not Supported",,,,,0,,,,,,"INVITE,ACK,BYE,CANCEL,REGISTER",15.57,109 -"2010-02-10 00:00:01",192.168.0.2,udp,5060,node02.example.com,sip,64512,ZZ,Region,City,SIP/2.0,400,"Bad Request",,,,,364,text/plain,,,,,,62.57,438 -"2010-02-10 00:00:02",192.168.0.3,udp,5060,node03.example.com,sip,64512,ZZ,Region,City,SIP/2.0,400,"Bad Request",,,,,0,,,,,,,6.57,46 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license deleted file mode 100644 index 9f58c89ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv deleted file mode 100644 index 256dd78f6..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","version","function","function_text","flags","next_extension_offset","xid","language_tag_length","language_tag","error_code","error_code_text","response_size","raw_response" -"2010-02-10 00:00:00",192.168.0.1,tcp,427,node01.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== -"2010-02-10 00:00:01",192.168.0.2,tcp,427,node02.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== -"2010-02-10 00:00:02",192.168.0.3,tcp,427,node03.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license deleted file mode 100644 index 9f58c89ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv deleted file mode 100644 index fc7fe2fff..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","smb_implant","arch","key","smbv1_support","smb_major_number","smb_minor_number","smb_revision","smb_version_string" -"2010-02-10 00:00:00",192.168.0.1,445,node01.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" -"2010-02-10 00:00:01",192.168.0.2,445,node02.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" -"2010-02-10 00:00:02",192.168.0.3,445,node03.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv deleted file mode 100644 index 19eb56053..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" -"2021-07-08 11:58:42","1.2.3.4","tcp",25,"smtp-server.invalid","smtp;21nails",12345,"EE","HARJUMAA","TALLINN",,,"220 smtp-server.invalid ESMTP Exim 4.80 Wed, 11 Jun 2021 10:00:00 +0300|" -"2021-07-08 11:58:44","5.6.7.8","tcp",25,"smtp-out.invalid","smtp;21nails",23456,"EE","HARJUMAA","TALLINN",,,"220 smtp-out.invalid, ESMTP EXIM 4.86_2|" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license deleted file mode 100644 index c1900637f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv deleted file mode 100644 index f489261c4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","sysdesc","sysname","asn","geo","region","city","version","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_sector","tag","community","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,161,node01.example.com,"Linux localhost 3.18.20 #1 SMP Mon Jul 9 14:11:21 CST 2018 armv7l",,64512,ZZ,Region,City,2,0,0,,,,,,,snmp,public,165,1.94 -"2010-02-10 00:00:01",192.168.0.2,udp,161,node02.example.com,"RouterOS CCR1009-8G-1S-1S+",,64512,ZZ,Region,City,2,0,0,,MikroTik,router,,,consumer,"snmp,iot",public,115,1.35 -"2010-02-10 00:00:02",192.168.0.3,udp,161,node03.example.com,,,64512,ZZ,Region,City,2,0,0,,,,,,,snmp,public,85,1.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv deleted file mode 100644 index c591a5c09..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector" -"2010-02-10 00:00:00",192.168.0.1,tcp,1080,node01.example.com,socks4,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:01",192.168.0.2,tcp,1080,node02.example.com,socks5,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,tcp,1080,node03.example.com,socks4,64512,ZZ,Region,City,0,0,"Retail Trade" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv deleted file mode 100644 index 460be32c5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","header","asn","geo","region","city","systime","cache_control","location","server","search_target","unique_service_name","host","nts","nt","content_type","naics","sic","sector","server_port","instance","version","updated_at","resource_identifier","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,60194,node01.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,"Sun, 21 Aug 2022 09:51:13 GMT",max-age=100,http://192.168.200.254:49152/description.xml,"Linux/2.6.26, UPnP/1.0, Portable SDK for UPnP devices/1.3.1",upnp:rootdevice,uuid:28802880-2880-1880-a880-001bc502f600::upnp:rootdevice,node01.example.com,,,,0,0,Government,,,,,,3.35,325 -"2010-02-10 00:00:01",192.168.0.2,udp,38732,node02.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,,"max-age = 1800",http://95.160.216.14:52235/dmr/SamsungMRDesc.xml,"Linux/9.0 UPnP/1.0 PROTOTYPE/1.0",upnp:rootdevice,uuid:f144ca92-6816-94b5-b95f-b58180834044::upnp:rootdevice,node02.example.com,,,,0,0,,,,,,,2.71,263 -"2010-02-10 00:00:02",192.168.0.3,udp,57626,node03.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,"Sun, 03 Jan 2016 21:37:50 GMT",max-age=1800,http://192.168.1.3:8008/ssdp/device-desc.xml,"Linux/3.10.79, UPnP/1.0, Portable SDK for UPnP devices/1.6.18",upnp:rootdevice,uuid:62fa0fc8-079d-d00f-2e22-59b49fb488f9::upnp:rootdevice,node03.example.com,,,,0,0,Government,,,,,,4.79,465 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv deleted file mode 100644 index 837adbad1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","serverid_raw","serverid_version","serverid_software","serverid_comment","server_cookie","available_kex","available_ciphers","available_mac","available_compression","selected_kex","algorithm","selected_cipher","selected_mac","selected_compression","server_signature_value","server_signature_raw","server_host_key","server_host_key_sha256","rsa_prime","rsa_prime_length","rsa_generator","rsa_generator_length","rsa_public_key","rsa_public_key_length","rsa_exponent","rsa_modulus","rsa_length","dss_prime","dss_prime_length","dss_generator","dss_generator_length","dss_public_key","dss_public_key_length","dss_dsa_public_g","dss_dsa_public_p","dss_dsa_public_q","dss_dsa_public_y","ecdsa_curve25519","ecdsa_curve","ecdsa_public_key_length","ecdsa_public_key_b","ecdsa_public_key_gx","ecdsa_public_key_gy","ecdsa_public_key_n","ecdsa_public_key_p","ecdsa_public_key_x","ecdsa_public_key_y","ed25519_curve25519","ed25519_cert_public_key_nonce","ed25519_cert_public_key_bytes","ed25519_cert_public_key_raw","ed25519_cert_public_key_sha256","ed25519_cert_public_key_serial","ed25519_cert_public_key_type_id","ed25519_cert_public_key_type_name","ed25519_cert_public_key_keyid","ed25519_cert_public_key_principles","ed25519_cert_public_key_valid_after","ed25519_cert_public_key_valid_before","ed25519_cert_public_key_duration","ed25519_cert_public_key_sigkey_bytes","ed25519_cert_public_key_sigkey_raw","ed25519_cert_public_key_sigkey_sha256","ed25519_cert_public_key_sigkey_value","ed25519_cert_public_key_sig_raw","banner","userauth_methods","device_vendor","device_type","device_model","device_version","device_sector" -"2022-01-10 02:20:37","18.179.0.0","tcp",22,"ec2-18-179-0-0.ap-northeast-1.compute.amazonaws.com","ssh",16509,"JP","TOKYO","TOKYO",454110,,"SSH-2.0-OpenSSH_7.4","2.0","OpenSSH_7.4",,"bGjsifbPIDWT7tAu8BMjyg==","curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha256, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1","chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com, aes128-cbc, aes192-cbc, aes256-cbc, blowfish-cbc, cast128-cbc, 3des-cbc","umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1","none, zlib@openssh.com","curve25519-sha256@libssh.org","ecdsa-sha2-nistp256","aes128-ctr","hmac-sha2-256","none","AAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=","AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAABKAAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=","AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDSEHaLacthwB30rtA4xJgN3G9zXkCmm2WhV/TlNBrD20fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=","a6e4e1c16ba25d51bcddc58a6e16797144575dd18d02d9dedf75093d2b15c557",,,,,,,,,,,,,,,,,,,,"1xx7ASut7BF4ED8b592bebZBMBKTCzOsmbH4cjwx/0U=","P-256",256,"WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=","axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=","T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=","/////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=","/////wAAAAEAAAAAAAAAAAAAAAD///////////////8=","NIQdotpy2HAHfSu0DjEmA3cb3NeQKabZaFX9OU0GsPY=","0fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=",,,,,,,,,,,,,,,,,,,,"publickey",,,,, -"2022-01-10 02:20:37","170.10.0.0","tcp",22,"170-10-0-0.example.com","ssh",11976,"US","TEXAS","MARSHALL",,,"SSH-2.0-ARRIS_0.50","2.0","ARRIS_0.50",,"Y4RQS9sdRgEFwNJKVP6bZg==","diffie-hellman-group1-sha1","aes128-cbc, 3des-cbc, aes256-cbc, twofish256-cbc, twofish-cbc, twofish128-cbc, blowfish-cbc","hmac-sha1-96, hmac-sha1, hmac-md5","none","diffie-hellman-group1-sha1","ssh-rsa","aes128-cbc","hmac-sha1","none","LQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==","AAAAB3NzaC1yc2EAAACCLQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==","AAAAB3NzaC1yc2EAAAADAQABAAAAgwCDq1kGqqwdQVryCNcoyDbBpnL/okvM2d9NmR0OjprcToCZ2TZ5WUZt2BGwPE1QLJYskjhv7GwlfQ4qhEqHDg35wMrkO7j9LTQC7KW3xisOLuUil4FmMxPkol6s39945zBGpjw0l/BmJnUDlutxavkdd84fppFMwXNp2vbjxV1SYVc9","d53fedbfe92e631264629882b2e85bfd213ca4b07b824cd31f8de1fcb8d0ddcb",,,,,,,65537,"g6tZBqqsHUFa8gjXKMg2waZy/6JLzNnfTZkdDo6a3E6Amdk2eVlGbdgRsDxNUCyWLJI4b+xsJX0OKoRKhw4N+cDK5Du4/S00Auylt8YrDi7lIpeBZjMT5KJerN/feOcwRqY8NJfwZiZ1A5brcWr5HXfOH6aRTMFzadr248VdUmFXPQ==",1040,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"publickey, password","Arris",,,, -"2022-01-10 02:20:37","72.17.0.0","tcp",22,"072-017-0-0.example.com","ssh",33363,"US","FLORIDA","ORLANDO",517311,,"SSH-1.99-Cisco-1.25","1.99","Cisco-1.25",,"Z2fOfWsrLlh76Y0bOqa1cw==","diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1","aes128-cbc, 3des-cbc, aes192-cbc, aes256-cbc","hmac-sha1, hmac-sha1-96, hmac-md5, hmac-md5-96","none","diffie-hellman-group14-sha1","ssh-rsa","aes128-cbc","hmac-sha1","none","lrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=","AAAAB3NzaC1yc2EAAAIAlrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=","AAAAB3NzaC1yc2EAAAADAQABAAACAQDIVXBwKGhi35gabwHNZi6Bxls1BGtDVVZFhwvhTpJKTKhV4T2HnDFG7+FBpYejc92wH026Wf+uJHIpnKkVQRnnOV98zKXW68Tz+OnwT8aBQdLI+QYDC7wLwGRf+cOiXEAkpMrp2OJme+GwQ97oBccEwdu2j9vcYAFQ0+eCPNfwPrcZhwVb00kt/moLVSxWRdsDMzQiNDZf2zel+FQIAl5cCfaLSAQa1TIXy8SM13B0brnlpdyIqukQS0zUv/PL/6AsfhgLXeQBgjs1XIf6qL+ZdtQss5AKUDuJgrWDcS3nyNZQg/CAt8XdIsLntu3bCn+VGA1O/gUGLS1a9GoGd/lRArlmODNtbds74m7hxaAf/gzg0LFJx6HhwubmVCzTXEHl95KHYHKoDvCtUOgUm7zUugxWjhsLPfT6UfZCwvCY21SGVYsoEPiTT2DhuAFriM+PT83JresFHgZDosbqW0VCi2bzAKSBu/vphaqTbSdDo0xhkW9JCb3zUkW2ge/e/GrjxV4cNXRC9XQ/XYEIWmtF/gHSi0i9KweX4sN5TEkB/41vDvyDOdyPJ8Jta0I9vBolDwJ6qdMHOPlOW5oW83yCgbmUJNYkZ+MivABlc6iS/006qYiIwknHezbY5foYd8kDON7YAssOwCJcG5viII50Z1N9VsGkUv5sZMr2p9ry8Q==","06ff3cce443ed832927576d982b69d5a526d0e63334c72e87201deda61679406",,,,,,,65537,"yFVwcChoYt+YGm8BzWYugcZbNQRrQ1VWRYcL4U6SSkyoVeE9h5wxRu/hQaWHo3PdsB9Nuln/riRyKZypFUEZ5zlffMyl1uvE8/jp8E/GgUHSyPkGAwu8C8BkX/nDolxAJKTK6djiZnvhsEPe6AXHBMHbto/b3GABUNPngjzX8D63GYcFW9NJLf5qC1UsVkXbAzM0IjQ2X9s3pfhUCAJeXAn2i0gEGtUyF8vEjNdwdG655aXciKrpEEtM1L/zy/+gLH4YC13kAYI7NVyH+qi/mXbULLOQClA7iYK1g3Et58jWUIPwgLfF3SLC57bt2wp/lRgNTv4FBi0tWvRqBnf5UQK5ZjgzbW3bO+Ju4cWgH/4M4NCxSceh4cLm5lQs01xB5feSh2ByqA7wrVDoFJu81LoMVo4bCz30+lH2QsLwmNtUhlWLKBD4k09g4bgBa4jPj0/Nya3rBR4GQ6LG6ltFQotm8wCkgbv76YWqk20nQ6NMYZFvSQm981JFtoHv3vxq48VeHDV0QvV0P12BCFprRf4B0otIvSsHl+LDeUxJAf+Nbw78gzncjyfCbWtCPbwaJQ8CeqnTBzj5TluaFvN8goG5lCTWJGfjIrwAZXOokv9NOqmIiMJJx3s22OX6GHfJAzje2ALLDsAiXBub4iCOdGdTfVbBpFL+bGTK9qfa8vE=",4096,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"publickey, keyboard-interactive, password","Cisco",,,,"enterprise" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv deleted file mode 100644 index 0b125001b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","ssl_poodle","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","freak_vulnerable","freak_cipher_suite","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","tlsv13_support","tlsv13_cipher","jarm" -"2022-01-10 00:01:42","96.60.0.0",10443,"96-60-0-0.example.com","ssl,vpn","TLSv1.2",4181,"US","WISCONSIN","MILWAUKEE","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","N",1024,"FGT60D4614030700","support","2014-06-23 09:56:32","2038-01-19 03:14:07","5A:3D:FF:06:F9:E9:25:37:57:F9:09:52:33:A4:85:15:24:2D:88:7F","168CAE",2,"sha1WithRSAEncryption","rsaEncryption","Fortinet","FortiGate","US","California","Sunnyvale",,,,,"support@fortinet.com",,,"Fortinet","Certificate Authority","US","California","Sunnyvale",,,,,"support@fortinet.com",,,517311,,"N",,,"35:AB:B6:76:2A:3D:17:B2:FB:40:45:1B:FC:0A:99:0A:6E:48:57:F7:30:0A:3B:B1:1A:E6:99:70:5B:7C:32:41","88:7B:16:DB:39:44:0C:47:0E:4A:8F:0B:C5:FB:4D:45:BC:93:5A:00:43:A1:D9:7F:05:1D:86:33:02:F8:FC:57:67:A6:1D:C0:FF:F7:D2:40:D8:9A:21:AE:4E:6D:DC:E7:FF:72:BF:13:CB:EE:A7:5F:CD:83:EA:8A:5E:FB:87:DD","99:45:1F:2E:AE:EB:88:91:27:43:33:79:FA:93:7D:CA","HTTP/1.1",200,"OK","text/html",,,,"xxxxxxxx-xxxxx",131,,"Mon, 10 Jan 2022 00:01:44 GMT","Y","N","N","N","unknown","x509: unknown error",,, -"2022-01-10 00:01:42","113.160.0.0",10443,"","ssl","TLSv1.2",45899,"VN","THAI BINH","THAI BINH","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","N",2048,"1078-btb-tbi-HungHa-61d39c6d5a7e2","1078-btb-tbi-HungHa-61d39c6d5a7e2","2022-01-04 01:01:34","2023-02-06 01:01:34","A9:00:BB:E1:54:4D:56:54:59:F1:B7:EA:F1:1A:D5:36:5C:63:90:8E","36974C4C6B1B3785",2,"sha256WithRSAEncryption","rsaEncryption","pfSense webConfigurator Self-Signed Certificate",,,,,,,,,,,,"pfSense webConfigurator Self-Signed Certificate",,,,,,,,,,,,517311,,"N",,,"38:85:F0:44:1E:AD:84:B8:2F:43:68:BA:AC:EE:17:13:A4:BF:86:1D:48:75:7E:22:FA:08:4C:28:5F:AC:3E:5F","AE:1B:4F:D1:E4:C0:35:9D:2A:4F:7A:37:B8:7B:11:9D:84:25:23:21:AB:EF:B2:0F:DC:C9:F2:A3:72:28:92:E1:74:72:FA:E1:09:6C:E1:F6:B6:E3:A7:61:1C:58:89:34:D7:06:5C:3D:0A:A7:F6:CC:8A:D6:24:D0:04:4C:03:02","16:93:9A:F4:35:7F:9A:85:45:71:91:C7:7C:80:88:00","HTTP/1.1",200,"OK","text/html; charset=UTF-8","keep-alive",,"PHPSESSID=e15bdfa5739c36877608eb4cf46cc388; path=/; secure; HttpO","nginx",,"chunked","Mon, 10 Jan 2022 00:01:44 GMT","N","Y","N","N","unknown","x509: unknown error",,, -"2022-01-10 00:01:42","34.224.0.0",10443,"","ssl,vpn","TLSv1.2",14618,"US","VIRGINIA","ASHBURN","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","N",2048,"","Entrust Certification Authority - L1K","2021-10-07 15:30:28","2022-11-06 15:30:28","AD:19:B2:1C:CB:88:70:9B:DB:8E:7E:F5:65:50:13:D6:43:6C:BE:6E","7B388364A24B88E77E5553B5C6748100",2,"sha256WithRSAEncryption","rsaEncryption","Ciena Corporation",,"US","Maryland","Hanover",,,,,,,,"Entrust, Inc.","(c) 2012 Entrust, Inc. - for authorized use only","US",,,,,,,,,,454110,,"N",,"Retail Trade","9A:64:73:0B:8A:FA:DE:22:D4:6D:5A:C6:C4:6F:D4:A4:2A:28:FA:41:1E:FF:81:DC:D4:D9:00:FD:78:DF:C4:DD","9A:B7:BD:68:7D:F3:E7:C1:B7:D3:F4:2F:01:B6:C4:77:90:A3:2B:1E:C0:89:F5:08:EC:43:87:35:60:36:D4:87:61:AA:B8:A8:B3:8A:E9:F1:04:AA:5B:67:12:FF:63:D5:14:80:77:6E:8F:7D:C3:E2:3A:F3:13:DF:08:43:6C:B0","E7:34:BC:92:84:FA:39:DE:E1:46:6C:27:DA:5A:01:F4","HTTP/1.1",200,"OK","text/html",,,,"xxxxxxxx-xxxxx",131,,"Mon, 10 Jan 2022 00:01:44 GMT","Y","N","N","Y","OV",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv deleted file mode 100644 index ab28456b4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv +++ /dev/null @@ -1,46 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","freak_vulnerable","freak_cipher_suite","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","device_model","device_sector","device_type","device_vendor","device_version","jarm","page_sha256fp","raw_cert","raw_cert_chain","tlsv13_cipher","tlsv13_support" -"2018-04-23 13:25:21","198.51.100.232","443",,"ssl-freak","TLSv1.0","8447","AT","WIEN","VIENNA","TLS_RSA_WITH_RC4_128_SHA","1024","usg50_B0B2DC2FA69D","usg50_B0B2DC2FA69D","2012-05-10 00:01:19","2032-05-05 00:01:19","14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2","4FAB054F","sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,"0","0","Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5",,"57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1","E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87","1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE","HTTP/1.1","200","OK","text/html",,,,,,"chunked","Mon, 23 Apr 2018 13:25:26 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -"2018-04-23 13:25:26","198.51.100.224","443","198-51-100-224.example.net","ssl-freak","TLSv1.0","12577","AT","NIEDEROSTERREICH","BADEN","TLS_RSA_WITH_RC4_128_SHA","1024","usg20w_C86C870287EC","usg20w_C86C870287EC","2010-01-01 00:00:53","2029-12-27 00:00:53","14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2","4B3D3B35","sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,"0","0","Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5",,"57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1","E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87","1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE","HTTP/1.1","200","OK","text/html",,,,,,"chunked","Mon, 23 Apr 2018 13:25:29 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -2018-04-23 13:25:21,198.51.100.232,443,,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC2FA69D,usg50_B0B2DC2FA69D,2012-05-10 00:01:19,2032-05-05 00:01:19,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FAB054F,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:26 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:26,198.51.100.224,443,198-51-100-224.example.net,ssl-freak,TLSv1.0,12577,AT,NIEDEROSTERREICH,BADEN,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_C86C870287EC,usg20w_C86C870287EC,2010-01-01 00:00:53,2029-12-27 00:00:53,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4B3D3B35,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:29 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:33,198.51.100.67,443,,ssl-freak,TLSv1.0,8447,AT,NIEDEROSTERREICH,WAIDHOFEN AN DER THAYA,TLS_RSA_WITH_RC4_128_SHA,1024,Technicolor TG670,Technicolor TG670,2005-01-01 00:00:00,2024-12-31 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-7A2C610E,sha1WithRSAEncryption,rsaEncryption,Technicolor,1112WT0YK,,,,,,,,,,,Technicolor,1112WT0YK,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,,keep-alive,,xAuth_SESSION_ID=bm90aGluZyBoZXJlCg==; path=/;,,0,,"Mon, 23 Apr 2018 14:25:37 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:36,198.51.100.3,443,,ssl-freak,TLSv1.2,8445,AT,SALZBURG,HINTERGLEMM,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,2048,uag2100_04BF6D22A5A9,uag2100_04BF6D22A5A9,2016-03-08 20:27:08,2026-03-06 20:27:08,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B0F07D300BDB4FC4,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:39 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:38,198.51.100.198,443,198-51-100-198.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,2048,198-51-100-198.example.net,Go Daddy Secure Certificate Authority - G2,2016-12-29 08:51:00,2019-12-29 08:51:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,AEA6D3637023B56B,sha256WithRSAEncryption,rsaEncryption,,Domain Control Validated,,,,,,,,,,,198-51-100-198.example.net," Inc.""",http://certs.godaddy.com/repository/,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden ( The server,text/html,close,,,,2024,,,Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-23 13:25:38,198.51.100.98,443,198-51-100-98.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_FCF528743754,usg50_FCF528743754,2013-04-29 00:00:26,2033-04-24 00:00:26,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,517DB81A,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:41,198.51.100.156,443,198-51-100-156.example.net,ssl-freak,TLSv1.0,8339,AT,NIEDEROSTERREICH,SCHWECHAT,TLS_RSA_WITH_AES_128_CBC_SHA,1024,usg200_404A036775FC,usg200_404A036775FC,2010-05-01 00:04:04,2030-04-26 00:04:04,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4BDB6FF4,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:43 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:53,198.51.100.200,443,,ssl-freak,TLSv1.2,8447,AT,NIEDEROSTERREICH,KREMS AN DER DONAU,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB67FC6F,usg20_5CF4AB67FC6F,2015-12-02 00:00:47,2035-11-27 00:00:47,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,565E34AF,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:56 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:02,198.51.100.83,443,198-51-100-83.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_FCF5286F5972,usg20w_FCF5286F5972,2013-03-23 00:00:43,2033-03-18 00:00:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,514CF0AB,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:05 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:03,198.51.100.155,443,198-51-100-155.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-155.example.net,198-51-100-155.example.net,2018-03-19 19:47:07,2023-03-19 19:47:07,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2DF52AA905C7A2B44C2B9F0012FD5745,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden,text/html,,,,Microsoft-IIS/6.0,1939,,"Mon, 23 Apr 2018 13:11:52 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:03,198.51.100.129,443,198-51-100-129.example.net,ssl-freak,TLSv1.0,29654,AT,SALZBURG,SALZBURG,TLS_RSA_WITH_RC4_128_SHA,1024,localhost,localhost,2007-01-31 19:00:29,2008-01-31 19:00:29,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2,sha1WithRSAEncryption,rsaEncryption,Apache HTTP Server,Test Certificate,,,,,,,,,,,Apache HTTP Server,For testing purposes only,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,318,,"Mon, 23 Apr 2018 17:42:37 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:12,198.51.100.7,443,198-51-100-7.example.net,ssl-freak,TLSv1.0,8445,AT,SALZBURG,ALTENMARKT IM PONGAU,TLS_RSA_WITH_RC4_128_SHA,2048,IMM2-5cf3fcaf3abd,IMM2-5cf3fcaf3abd,2013-03-22 14:32:06,2023-03-20 14:32:06,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,D8C631398B585F10,sha1WithRSAEncryption,rsaEncryption,System X,,US,SomeState,SomeCity,,,,,,,,System X,,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,301,Moved Permanently,application/x-appweb-php,keep-alive,,,Mbedthis-Appweb/2.4.2,0,,"Mon, 23 Apr 2018 13:37:08 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:15,198.51.100.93,443,,ssl-freak,TLSv1.2,8447,AT,KARNTEN,SPITTAL AN DER DRAU,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC3308EF,usg50_B0B2DC3308EF,2012-05-25 00:00:39,2032-05-20 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FBECBA7,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:17 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:16,198.51.100.81,443,198-51-100-81.example.net,ssl-freak,TLSv1.0,5385,AT,VORARLBERG,FELDKIRCH,TLS_RSA_WITH_RC4_128_SHA,1024,usg100_5067F03642A5,usg100_5067F03642A5,2010-10-01 00:04:48,2030-09-26 00:04:48,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4CA525A0,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:19 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:17,198.51.100.162,443,198-51-100-162.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,rc1,Peppercon CA,2003-05-08 16:30:05,2008-05-06 16:30:05,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,18,md5WithRSAEncryption,rsaEncryption,,R&D,DE,SomeState,,,,,,198-51-100-162.example.net,,,,Security Department,DE,SomeState,SomeCity,,,,,198-51-100-162.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Redirect,,,,,,,,,N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:22,198.51.100.57,443,,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,GLEISDORF,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB661192,usg20_5CF4AB661192,2015-09-22 00:00:46,2035-09-17 00:00:46,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,56009A2E,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:25 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:28,198.51.100.146,443,198-51-100-146.example.net,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_RC4_128_SHA,1024,zywall_110_A0E4CB7CE5AF,zywall_110_A0E4CB7CE5AF,2015-01-26 17:19:56,2025-01-23 17:19:56,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,54C6773C,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:31 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:34,198.51.100.233,443,,ssl-freak,TLSv1.0,8447,AT,KARNTEN,KLAGENFURT AM WORTHERSEE,TLS_RSA_WITH_RC4_128_SHA,1024,198.51.100.174,198-51-100-174.example.net,2009-04-14 07:26:09,2025-04-15 07:26:09,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,A0571920C03C9EE0DA1168E586E0E8D440E42EA69D898AC829,sha1WithRSAEncryption,rsaEncryption,,General,DE,SomeState,SomeCity,,,,,198-51-100-174.example.net,,,,General,DE,SomeState,SomeCity,,,,,198-51-100-174.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,Close,,,LANCOM 1781A 8.50.0161 / 09.08.2011,,,,Y,N,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:35,198.51.100.106,443,198-51-100-106.example.net,ssl-freak,TLSv1.0,12793,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-106.example.net,SHT-Gruppe CA,2004-07-20 07:28:10,2006-07-20 07:38:10,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,190DBE75000000000007,sha1WithRSAEncryption,rsaEncryption,,,AT,SomeState,SomeCity,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/6.0,1508,,"Mon, 23 Apr 2018 13:26:37 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:37,198.51.100.191,443,,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,LEBRING,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB669448,usg20_5CF4AB669448,2015-10-01 00:00:38,2035-09-26 00:00:38,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,560C77A6,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:42,198.51.100.235,443,198-51-100-235.example.net,ssl-freak,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_107BEF33651A,usg50_107BEF33651A,2014-04-24 00:00:27,2034-04-19 00:00:27,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,5358541B,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:45 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:43,198.51.100.167,443,198-51-100-167.example.net,ssl-freak,TLSv1.0,8412,AT,BURGENLAND,ELTENDORF,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-167.example.net,198-51-100-167.example.net,2008-08-19 06:57:11,2010-08-19 06:57:11,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2,sha1WithRSAEncryption,rsaEncryption,SuSE Linux Web Server,web server,XY,unknown,unknown,,,,,198-51-100-167.example.net,,,SuSE Linux Web Server,CA,XY,SomeState,unknown,,,,,198-51-100-167.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache/2.2.3 (Linux/SUSE),80,,"Mon, 23 Apr 2018 13:26:45 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:47,198.51.100.42,443,198-51-100-42.example.net,ssl-freak,TLSv1.0,8437,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-42.example.net,iLO Default Issuer (Do not trust),2013-11-05 00:00:00,2028-11-04 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,72FD09EF,sha1WithRSAEncryption,rsaEncryption,,,US,SomeState,SomeCity,,,,,,,,,,US,SomeState,Houston,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,,,,,,,,,N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.177,443,198-51-100-177.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB625772,usg20_5CF4AB625772,2015-03-04 00:00:39,2035-02-27 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,54F64B27,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:50 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.66,443,198-51-100-66.example.net,ssl-freak,TLSv1.0,5385,AT,VORARLBERG,DORNBIRN,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-66.example.net,198-51-100-66.example.net,2009-10-06 11:23:48,2015-03-29 11:23:48,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,98B18BCD61B0CD5D,sha1WithRSAEncryption,rsaEncryption,,??,??,??,??,,,,,??,,,,??,??,??,??,,,,,??,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,close,,DSSignInURL=/; path=/; secure,,,,,Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.29,443,198-51-100-29.example.net,ssl-freak,TLSv1.0,6830,AT,NIEDEROSTERREICH,GUNTRAMSDORF,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_FCF52878354B,usg20_FCF52878354B,2013-05-20 00:00:39,2033-05-15 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,519967A7,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:50 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:49,198.51.100.235,443,,ssl-freak,TLSv1.0,8447,AT,TIROL,KITZBUHEL,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC3AEFE7,usg50_B0B2DC3AEFE7,2012-10-30 00:02:36,2032-10-25 00:02:36,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,508F191C,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:52 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:50,198.51.100.159,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-159.example.net,198-51-100-159.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-159.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-159.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:26:52 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:51,198.51.100.138,443,198-51-100-138.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_B0B2DC34A1F6,usg20_B0B2DC34A1F6,2012-06-16 00:00:58,2032-06-11 00:00:58,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FDBCCBA,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:54 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:52,198.51.100.64,443,,ssl-freak,TLSv1.0,1853,AT,OBEROSTERREICH,WILHERING,TLS_RSA_WITH_RC4_128_SHA,1024,198.51.100.171,198.51.100.117,2017-08-10 10:48:40,2020-08-09 10:48:40,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,598C3A08,sha1WithRSAEncryption,rsaEncryption,,,,,SomeCity,,,,,,,,,,,,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,5597,,"Mon, 23 Apr 2018 13:26:54 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:55,198.51.100.189,443,198-51-100-62.example.net,ssl-freak,TLSv1.0,25255,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_107BEF3A4C9E,usg20w_107BEF3A4C9E,2014-07-04 00:00:43,2034-06-29 00:00:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,53B5EEAB,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:57 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.17,443,198-51-100-17.example.net,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,SOEDING,TLS_RSA_WITH_AES_256_CBC_SHA,1024,Vimar By-Web,Vimar By-Web,2011-10-27 09:19:55,2016-10-25 09:19:55,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B82B13ED1FB0FD71,sha1WithRSAEncryption,rsaEncryption,,R&D,IT,SomeState,SomeCity,,,,,,,,,R&D,IT,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,text/html,keep-alive,,,nginx/0.6.32,,chunked,"Mon, 23 Apr 2018 13:26:56 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.111,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-111.example.net,198-51-100-111.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-111.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-111.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:26:58 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.179,443,198-51-100-179.example.net,ssl-freak,TLSv1.0,12605,AT,OBEROSTERREICH,LINZ,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB665FB9,usg20_5CF4AB665FB9,2015-09-25 00:00:42,2035-09-20 00:00:42,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,56048EAA,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:58 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:58,198.51.100.143,443,,ssl-freak,TLSv1.0,8447,AT,KARNTEN,GLAN,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_FCF5285DEDC4,usg20_FCF5285DEDC4,2012-11-09 00:00:44,2032-11-04 00:00:44,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,509C47AC,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:00 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:58,198.51.100.111,443,198-51-100-111.example.net,ssl-freak,TLSv1.0,1901,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,*.*,198-51-100-111.example.net,2009-01-16 12:51:43,2010-01-16 12:51:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6,md5WithRSAEncryption,rsaEncryption,,,IL,SomeState,,,,,,,,,,Visonic CA,IL,SomeState,,,,,,198-51-100-111.example.net,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html,close,,PowerLink=226002836046b4bddcd2d16b809f76d9; path=/,Apache/1.3.31 (Unix) PHP/4.3.9 mod_ssl/2.8.20 Open,,chunked,"Wed, 23 Jan 2002 10:17:09 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:59,198.51.100.79,443,,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB65A17C,usg20_5CF4AB65A17C,2015-09-01 00:00:51,2035-08-27 00:00:51,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,55E4EAB3,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:02 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:59,198.51.100.90,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-90.example.net,198-51-100-90.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-90.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-90.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:27:02 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.186,443,198-51-100-186.example.net,ssl-freak,TLSv1.0,31125,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-186.example.net,198-51-100-186.example.net,2013-07-11 12:20:19,2021-07-09 12:20:19,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,D947ED19BEAB28E6,sha1WithRSAEncryption,rsaEncryption,,IT,PL,SomeState,SomeCity,,,,,198-51-100-186.example.net,,,,IT,PL,SomeState,SomeCity,,,,,198-51-100-186.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,401,Unauthorized,text/plain,close,"Basic realm=""example.com""",,Microsoft-IIS/7.5,0,,"Mon, 23 Apr 2018 14:03:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.150,443,198-51-100-150.example.net,ssl-freak,TLSv1.0,8559,AT,BURGENLAND,NEUSIEDL AM SEE,TLS_ECDHE_RSA_WITH_RC4_128_SHA,2048,198-51-100-150.example.net,COMODO RSA Domain Validation Secure Server CA,2017-02-08 00:00:00,2019-05-09 23:59:59,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B6EF6CF436532F0252627393BD7311FD,sha256WithRSAEncryption,rsaEncryption,,Domain Control Validated,,,,,,,,,,,,,GB,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:06 GMT",N,N,N,N,DV,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.141,443,198-51-100-141.example.net,ssl-freak,TLSv1.0,39372,AT,OBEROSTERREICH,HINTERSTODER,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-141.example.net,iLO Default Issuer (Do not trust),2014-01-14 00:00:00,2029-01-13 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7852761B,sha1WithRSAEncryption,rsaEncryption,,,US,SomeState,SomeCity,,,,,,,,,,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,,,,,,,,,N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:27:04,198.51.100.194,443,198-51-100-194.example.net,ssl-freak,TLSv1.0,8447,AT,KARNTEN,GLAN,TLS_RSA_WITH_RC4_128_SHA,1024,iDRAC6 default certificate,iDRAC6 default certificate,2009-09-17 22:47:28,2019-09-15 22:47:28,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,1,sha1WithRSAEncryption,rsaEncryption,,Remote Access Group,US,SomeState,SomeCity,,,,,,,,,Remote Access Group,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,,keep-alive,,,Mbedthis-Appweb/2.4.2,0,,"Mon, 23 Apr 2018 13:25:57 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -"2022-02-07 00:01:41","2.136.0.0",10443,"2-136-0-0.example.com","ssl,ssl-freak,ssl-poodle,vpn","TLSv1.0",12345,"ES","MADRID","MADRID","TLS_RSA_WITH_RC4_128_SHA",1024,"usg50_107BEF336340","usg50_107BEF336340","2014-04-24 00:00:32","2034-04-19 00:00:32","F5:04:98:CD:D4:67:13:E1:77:B7:38:D4:B9:43:C0:72:50:6C:0D:58",53585420,"sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,517311,,"Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5","Communications, Service Provider, and Hosting Service","AF:3A:71:B7:1B:A2:62:4E:87:22:FF:19:3F:84:1F:7F:CC:DC:06:E0:AF:80:E2:5D:33:A5:68:9A:E3:81:25:45","14:92:CC:6B:C7:B3:09:31:50:8C:1C:8D:5B:FD:D1:BE:41:78:80:97:E0:10:11:48:1F:EE:D6:CB:4F:F0:13:D5:05:56:AC:BA:12:12:02:F7:0F:03:40:95:17:8A:5F:79:98:E1:44:EF:E6:5A:44:E3:AC:3A:F8:49:F7:AC:B6:52","E8:5F:96:16:3F:76:35:F0:07:4F:4C:2C:38:FC:27:6B","HTTP/1.1",200,"OK","text/html",,,,"",,"chunked","Mon, 07 Feb 2022 00:01:43 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,"Zyxel","firewall","ZyWALL USG 50",,"enterprise", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv deleted file mode 100644 index 4bcc6758a..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv +++ /dev/null @@ -1,32 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","ssl_poodle","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","tlsv13_support","tlsv13_cipher","device_model","device_sector","device_type","device_vendor","device_version","jarm","page_sha256fp","raw_cert","raw_cert_chain" -"2018-08-08 00:51:42","203.0.113.85",8443,"example.com","ssl-poodle","TLSv1.0",65540,"AT","WIEN","VIENNA","TLS_RSA_WITH_RC4_128_SHA","Y",1024,"usg20_107BEF394BA5","usg20_107BEF394BA5","2014-06-25 00:00:42","2034-06-20 00:00:42","04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3","53AA112A",2,"sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,0,0,,"16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E","0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE","33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC","HTTP/1.1",200,"OK","text/html",,,,,,"chunked","Wed, 08 Aug 2018 00:51:44 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -2018-04-19 13:32:27,198.51.100.147,443,,ssl-poodle,TLSv1.0,8445,AT,SALZBURG,SALZBURG,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-147.example.net,some_issuer,2017-09-18 08:22:17,2019-09-18 08:22:17,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,746481F100000000000C,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,Tirol,Ehrwald,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:32 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:32,198.51.100.207,443,198-51-100-94.example.net,ssl-poodle,TLSv1.0,25255,AT,SALZBURG,SALZBURG,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2004-06-03 11:11:43,2024-05-29 11:11:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,2,md5WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,"rg_cookie_session_id=1654544029; path=/; expires=Fri, 01 Jan 2038",,,,"Thu, 19 Apr 2018 13:32:34 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:32,198.51.100.200,443,198-51-100-200.example.net,ssl-poodle,TLSv1.2,8445,AT,SALZBURG,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-200.example.net,198-51-100-200.example.net,2016-10-01 14:09:12,2020-10-02 14:09:12,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2E8C9E4A2C7D3EDC,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,,AT,,,,,,,,,,some_org_name,,AT,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=utf-8,close,,,,,,,N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:33,198.51.100.239,443,198-51-100-239.example.net,ssl-poodle,TLSv1.0,8437,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198-51-100-239.example.net,198-51-100-239.example.net,2011-07-27 13:30:18,2012-07-26 13:30:18,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7C91,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,--,SomeState,SomeCity,,,,,198-51-100-239.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-239.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden,text/html; charset=UTF-8,close,,,Apache/2.2.3 (CentOS),4958,,"Thu, 19 Apr 2018 13:32:35 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:35,198.51.100.156,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2010-01-01 00:00:52,2029-12-27 00:00:52,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4B3D3B34,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Thu, 19 Apr 2018 13:32:37 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:36,198.51.100.122,443,198-51-100-122.example.net,ssl-poodle,TLSv1.2,36351,AT,AUSTRIA,?,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,2048,198-51-100-122.example.net,COMODO RSA Organization Validation Secure Server CA,2017-04-06 00:00:00,2019-04-06 23:59:59,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,CAB81F32F3FF4766BC545A2C14DF34B5,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,Wien,Wien,,1130,,,,,,COMODO CA Limited,,GB,Greater Manchester,Salford,,,,,,,,518210,737401,Information Technology,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,0,,"Thu, 19 Apr 2018 13:32:20 GMT",Y,N,N,Y,OV,,,,,,,,,,,, -2018-04-19 13:32:37,198.51.100.58,443,198-51-100-58.example.net,ssl-poodle,TLSv1.2,12605,AT,OBEROSTERREICH,LINZ,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,example,some_issuer,2015-01-17 16:11:24,2020-01-17 16:11:24,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6D9E2D4443F1D69E4A8865CC1C5B6963,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/8.5,701,,"Thu, 19 Apr 2018 13:34:53 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:38,198.51.100.18,443,198-51-100-18.example.net,ssl-poodle,TLSv1.2,6830,AT,OBEROSTERREICH,LINZ,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,2048,198-51-100-18.example.net,TERENA SSL CA 3,2017-07-14 00:00:00,2020-07-22 12:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0386AD387BEC13878473D23C8C786ECE,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,,Linz,,,,,,,,TERENA,,NL,Noord-Holland,Amsterdam,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,,Close,,BNIS_ChallengeState=Bqyd+IQebjQwiiYNKBJkA5Ta0spL1gX5; Path=/; Exp,,61,,,Y,N,N,Y,OV,,,,,,,,,,,, -2018-04-19 13:32:38,198.51.100.246,443,,ssl-poodle,TLSv1.2,8447,AT,SALZBURG,SALZBURG,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,example,some_issuer,2014-09-01 16:18:46,2054-08-24 16:18:46,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,129FA64A4BE039B54E850F1AA65AD835,2,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,,,ASP.NET_SessionId=e3qfk1dfz2mtqwzoym3gul3r; path=/; HttpOnly,Microsoft-IIS/8.5,145,,"Thu, 19 Apr 2018 13:32:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.35,443,198-51-100-35.example.net,ssl-poodle,TLSv1.0,12605,AT,OBEROSTERREICH,LINZ,TLS_RSA_WITH_AES_128_CBC_SHA,Y,2048,198-51-100-35.example.net,Go Daddy Secure Certificate Authority - G2,2017-08-28 13:29:01,2018-09-10 06:28:49,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,90B22B4CEF57C0FC,2,sha256WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,198-51-100-35.example.net,,,,,US,Arizona,Scottsdale,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,266,,"Thu, 19 Apr 2018 13:35:03 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.142,443,,ssl-poodle,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,Y,2048,198.51.100.19,198-51-100-19.example.net,2014-12-11 09:57:33,2024-12-08 09:57:33,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,A0571DCBE5E1A2C062D8FB7001271581B5F69824157E385563FA23527E0B,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,DE,,,,,,,198-51-100-19.example.net,,,some_org_name,Engineering,DE,NRW,Wuerselen,,,,,198-51-100-19.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,Close,,,LANCOM,,,"Thur, 19 Apr 2018 13:32:41 GMT",Y,N,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.178,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2012-05-30 00:00:44,2032-05-25 00:00:44,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FC5632C,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Thu, 19 Apr 2018 13:32:41 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:40,198.51.100.99,443,198-51-100-99.example.net,ssl-poodle,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-99.example.net,RapidSSL RSA CA 2018,2018-03-30 00:00:00,2019-04-29 12:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0BDCB5D6D4C22BD2A1CF55584B6DE09C,2,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,DigiCert Inc,198-51-100-99.example.net,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,404,Not Found,text/html; charset=us-ascii,close,,,Microsoft-HTTPAPI/2.0,315,,"Thu, 19 Apr 2018 13:32:43 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:40,198.51.100.235,443,198-51-100-235.example.net,ssl-poodle,TLSv1.0,25255,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,1024,Nextcloud,Nextcloud,2016-12-13 20:28:39,2017-01-12 20:28:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,CDE5769D28C80B6B,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,,AU,Some-State,,,,,,,,,Internet Widgits Pty Ltd,,AU,Some-State,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,400,Bad Request,text/html; charset=UTF-8,close,,nc_sameSiteCookiestrict=true; path=/; httponly;secure; expires=Fr,Apache/2.4.10 (FreeBSD) OpenSSL/0.9.8zd-freebsd PH,6939,,"Thu, 19 Apr 2018 13:32:42 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:41,198.51.100.187,443,198-51-100-187.example.net,ssl-poodle,TLSv1.2,28760,AT,OBEROSTERREICH,,TLS_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-187.example.net,Go Daddy Secure Certificate Authority - G2,2018-02-12 17:56:01,2020-02-12 17:56:01,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,03BA30FF4972177C,2,sha256WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,198-51-100-187.example.net,,,,,US,Arizona,Scottsdale,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,400,No parameters passed t,text/html,,,,Microsoft-IIS/10.0,11,,"Thu, 19 Apr 2018 13:32:42 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:42,198.51.100.213,443,198-51-100-213.example.net,ssl-poodle,TLSv1.2,8447,AT,OBEROSTERREICH,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-213.example.net,some_issuer,2016-09-22 08:12:17,2018-09-22 08:12:17,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,770000000EBB9429663601BAB700000000000E,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,301,Moved Permanently,,close,,,Microsoft-IIS/8.5,0,,"Thu, 19 Apr 2018 13:32:44 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:42,198.51.100.74,443,198-51-100-74.example.net,ssl-poodle,TLSv1.0,62363,AT,STEIERMARK,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,DC,DC,2016-12-30 17:15:38,2021-12-30 17:15:38,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7753CCEB55990A834E15DAC5707D403A,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:44 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:43,198.51.100.145,443,198-51-100-145.example.net,ssl-poodle,TLSv1.0,8447,AT,KARNTEN,KLAGENFURT AM WORTHERSEE,TLS_RSA_WITH_RC4_128_SHA,Y,1024,localdomain,localdomain,2008-10-07 20:12:54,2018-10-07 20:12:54,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,91B04FFCF174CCFF,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,,CA,,,,,,,198-51-100-145.example.net,,,some_org_name,,CA,Quebec,Gatineau,,,,,198-51-100-145.example.net,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,302,Found,text/html; charset=UTF-8,close,,"HOMEBASEID=658512b32961b9b6f8df7a3d4de7fa01; expires=Tue, 19-Jan-",Apache/2.2.3 (Red Hat),0,,"Thu, 19 Apr 2018 12:52:32 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:44,198.51.100.48,443,198-51-100-48.example.net,ssl-poodle,TLSv1.0,1901,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198-51-100-48.example.net,198-51-100-48.example.net,2013-06-15 20:10:49,2023-06-15 20:10:49,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,013F49762DAE,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,198-51-100-48.example.net,,,Western Digital,Branded Products,US,CS,Mountain View,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,225,,"Thu, 19 Apr 2018 03:08:06 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:45,198.51.100.94,443,198-51-100-94.example.net,ssl-poodle,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-94.example.net,RapidSSL CA,2013-04-03 17:02:33,2014-04-07 03:32:33,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0B697D,2,sha1WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,,,KtAjvog6HgAsml0cyxE4hpc9kv8dhgWZ,"GeoTrust, Inc.",,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,,,ASP.NET_SessionId=z5lph4ufefkvg1xzmd4q2m33; path=/; HttpOnly,Microsoft-IIS/8.0,144,,"Thu, 19 Apr 2018 13:32:48 GMT",Y,N,Y,N,unknown,x509: certificate has expired or is not yet valid,,,,,,,,,,, -2018-04-19 13:32:45,198.51.100.53,443,198-51-100-53.example.net,ssl-poodle,TLSv1.0,8447,AT,TIROL,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2008-11-13 13:47:18,2028-11-08 13:47:18,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,BE2B43544C0AFF2E,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,DE,,,,,,,198-51-100-53.example.net,,,some_org_name,some_org_name,DE,Niedersachsen,38162 Cremlingen (OT Schandelah),,,,,198-51-100-53.example.net,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=iso-8859-1;,,,,GoAhead-Webs,,,,Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:46,198.51.100.56,443,198-51-100-56.example.net,ssl-poodle,TLSv1.0,8445,AT,TIROL,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-56.example.net,some_issuer,2016-11-28 08:05:12,2018-11-28 08:05:12,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,637D34F100010000000E,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:49 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:46,198.51.100.82,443,198-51-100-82.example.net,ssl-poodle,TLSv1.0,6830,AT,OBEROSTERREICH,,TLS_RSA_WITH_AES_128_CBC_SHA,Y,1024,123AFG,7426AC8186F3,2011-01-01 00:00:06,2020-12-29 00:00:06,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,8186F3,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,,,,"Cisco Systems, Inc.",some_org_name,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html,close,,,Embedded HTTP Server.,107,,"Sat, 01 Jan 2011 00:00:45 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:48,198.51.100.29,443,198-51-100-29.example.net,ssl-poodle,TLSv1.0,6830,AT,STEIERMARK,GRAZ,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198.51.100.43,198.51.100.22,2018-04-18 13:32:09,2038-01-15 13:32:09,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,862D98F4B99D0042,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html; charset=utf-8,,,,,,,,Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.114,443,198-51-100-114.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_AES_256_CBC_SHA,Y,1024,198-51-100-114.example.net,198-51-100-114.example.net,2009-08-25 17:47:57,2019-05-25 17:47:57,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,FB09C7848A7F4D77,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,,AT,Vienna,Vienna,,,,,198-51-100-114.example.net,,,Digispectrum,,AT,Vienna,Vienna,,,,,198-51-100-114.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=utf-8,close,,b69223925949d45306d32f1a3d23c011=6a01vehilfpml41pl3pq3oth52; path,Apache/2.2.3 (CentOS),,chunked,"Thu, 19 Apr 2018 13:32:52 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.11,443,198-51-100-11.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,2048,FGT60C3G12019794,FGT60C3G12019794,2012-08-10 07:17:11,2022-08-11 07:17:11,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-6CD83A89,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,,,,,,,,,,,,Fortinet Ltd.,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,79,,"Thu, 19 Apr 2018 13:32:08 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.49,443,198-51-100-49.example.net,ssl-poodle,TLSv1.2,8447,AT,NIEDEROSTERREICH,,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,1024,localhost,localhost,2009-11-10 23:48:47,2019-11-08 23:48:47,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B5C752C98781B503,0,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache/2.4.10 (Win32) OpenSSL/1.0.1i PHP/5.5.15,2190,,"Thu, 19 Apr 2018 13:32:55 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:50,198.51.100.236,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_AES_128_CBC_SHA,Y,1024,example,some_issuer,2013-01-30 12:00:08,2023-01-28 12:00:08,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-462A1420,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,,,,Netgear Inc.,Netgear Prosafe,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html,close,,,Embedded HTTP Server.,107,,"Sat, 01 Jan 2011 00:00:21 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:50,198.51.100.224,443,198-51-100-224.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-224.example.net,some_issuer,2017-08-03 10:21:50,2019-08-03 10:21:50,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6126D181000300000041,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,401,Unauthorized,text/html,,NTLM,,Microsoft-IIS/7.5,1344,,"Thu, 19 Apr 2018 13:32:52 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -"2022-02-07 00:01:41","206.162.0.0",10443,,"ssl,ssl-poodle,vpn","TLSv1.2",12345,"CA","BRITISH COLUMBIA","BURNABY","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","Y",1024,"FWF60D4615000455","support","2015-01-28 18:14:33","2038-01-19 03:14:07","C9:B0:4E:B7:79:94:B4:DD:A7:15:21:86:43:F9:6E:4B:C9:A2:87:D9","1CA40F",2,"sha1WithRSAEncryption","rsaEncryption","Fortinet","FortiGate","US","California","Sunnyvale",,,,,"support@fortinet.com",,,"Fortinet","Certificate Authority","US","California","Sunnyvale",,,,,"support@fortinet.com",,,517311,,"Communications, Service Provider, and Hosting Service","38:F7:E0:92:24:8C:CB:28:43:93:0B:91:17:30:B1:41:8F:4E:2D:E5:A8:93:AE:4D:FE:53:00:D3:0E:53:02:16","0C:F0:37:3F:A8:93:AE:4D:FE:53:00:D3:2A:E6:6D:0B:02:9D:B9:46:58:A6:9E:5A:35:40:FB:62:9C:81:47:0A:4F:15:5D:53:D9:2F:36:4A:0B:3B:10:61:A9:07:EE:94:EC:00:B8:9C:F7:E0:92:24:8C:CB:28:2C:DD:E7:07:C6","8A:B3:08:20:34:79:94:B4:DD:A7:36:D7:14:6E:33:50","HTTP/1.1",200,"OK","text/html",,,,,131,,"Mon, 07 Feb 2022 00:01:43 GMT","Y","N","N","N","unknown","x509: unknown error",,,,,,"Fortinet","firewall","FortiGate",,"enterprise", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv deleted file mode 100644 index fd671ec90..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","transaction_id","magic_cookie","message_length","message_type","mapped_family","mapped_address","mapped_port","xor_mapped_family","xor_mapped_address","xor_mapped_port","software","fingerprint","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,3478,node01.example.com,stun,64512,ZZ,Region,City,0,0,,000000000000000000000000,2112a442,88,0101,01,192.168.0.1,3243,01,192.168.0.1,3243,"Coturn-4.5.1.1 'dan Eider'",0xfaedd06e,5.40,108 -"2010-02-10 00:00:01",192.168.0.2,udp,3478,node02.example.com,stun,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",000000000000000000000000,2112a442,88,0101,01,51.77.39.195,45877,01,192.168.0.2,45877,"Coturn-4.5.1.1 'dan Eider'",0x21128641,5.40,108 -"2010-02-10 00:00:02",192.168.0.3,udp,3478,node03.example.com,stun,64512,ZZ,Region,City,0,0,,000000000000000000000000,2112a442,76,0101,01,192.168.0.3,16321,01,188.68.240.32,16321,"ApolloProxy-1.20.1.28 'sunflower'",,4.80,96 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv deleted file mode 100644 index 8f6355491..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sequence_number","ack_number","window_size","urgent_pointer","tcp_flags","raw_packet","sector" -"2022-01-10 09:18:23","66.9.0.0","tcp",80,,"synfulknock",18885,"US","NEW JERSEY","JERSEY CITY",,,0,791102,8192,0,4608,"3cfdfec601e4700f6a9a2000080045000034c3780000f706789442099555b869f7ee0050b20800000000000c123e8012200002aa0000020405b40101040201030305", -"2022-01-10 09:19:17","213.131.0.0","tcp",80,"host-213-131-55-210-customer.wanex.net","synfulknock",35805,"GE","TBILISI","TBILISI",,,0,791102,8192,0,4608,"90e2baaf0b84700f6a9a200008004500003434100000f2064382d58337d2b8698b720050916200000000000c123e8012200059d50000020405b40101040201030305", -"2022-01-10 09:27:39","213.178.0.0","tcp",80,,"synfulknock",29256,"SY","DIMASHQ","DAMASCUS",,,0,791102,8192,0,4608,"90e2bab9cfd4700f6a9a20000800450000340f1d0000ea068bdad5b2e6914a522f360050eb5200000000000c123e801220001b4a0000020405b40101040201030305" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv deleted file mode 100644 index 3309e9a3d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" -"2019-09-04 12:27:34","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.5|Login:" -"2019-09-04 12:27:40","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.45.3 (stable)|Login:" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv deleted file mode 100644 index 3dde133d4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","opcode","errorcode","error","errormessage","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,35067,node01.example.com,tftp,64512,ZZ,Region,City,0,0,5,0,"Not defined","Get not supported",22,1.57 -"2010-02-10 00:00:01",192.168.0.2,udp,56709,node02.example.com,tftp,64512,ZZ,Region,City,0,0,5,1,"File not found","File not found",19,1.36 -"2010-02-10 00:00:02",192.168.0.3,udp,32785,node03.example.com,tftp,64512,ZZ,Region,City,0,0,5,2,"Access violation","Access violation",21,1.50 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv deleted file mode 100644 index efeab02c4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","mac","radioname","essid","modelshort","modelfull","firmware","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,10001,node01.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,00156db98c3a,kachine.meta.lidia.tereixa,Kachine-Meta-Lidia-Tereixa,NS5,,XS5.ar2313.v3.5.4494.091109.1459,148,37.00 -"2010-02-10 00:00:01",192.168.0.2,udp,10001,node02.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,00156d7c9188,adana.mason.lanikai.ozaner,Adana-Mason-Lanikai-Ozaner,LM5,"NanoStation Loco M5",XM.ar7240.v5.6.3.28591.151130.1749,156,39.00 -"2010-02-10 00:00:02",192.168.0.3,udp,10001,node03.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,0418d6000fd5,tailynn.kadija.noreen.dinkar,Tailynn-Kadija-Noreen-Dinkar,P2B-400,"PowerBeam M2 400",XW.ar934x.v5.6.5.29033.160515.2108,145,36.25 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv deleted file mode 100644 index 000f5ed42..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","asn","geo","region","city","naics","sic","product","banner","sector" -"2019-09-04 14:51:44","198.123.245.53",5678,,5678,"AA","LOCATION","LOCATION",0,0,"Apple remote desktop vnc","RFB 003.889", -"2019-09-04 14:51:44","198.123.245.112",5678,"localhost.localdomain",5678,"AA","LOCATION","LOCATION",517311,0,"RealVNC Enterprise v5.3 or later","RFB 005.000", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv deleted file mode 100644 index 7e279ca3e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","response_size","amplification","error","raw_response" -"2010-02-10 00:00:00",192.168.0.1,udp,3702,node01.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",989,164.83,"Validation constraint violation: SOAP message expected",c2FtcGxlIHJlc3BvbnNlIGRhdGEK -"2010-02-10 00:00:01",192.168.0.2,udp,3702,node02.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",918,183.60,"Validation constraint violation: missing root element",c2FtcGxlIHJlc3BvbnNlIGRhdGEK -"2010-02-10 00:00:02",192.168.0.3,udp,3702,node03.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",989,197.80,"Validation constraint violation: SOAP message expected",c2FtcGxlIHJlc3BvbnNlIGRhdGEK diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license deleted file mode 100644 index 9f58c89ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv deleted file mode 100644 index 7e83bbaf8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","opcode","reported_hostname","status","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,177,node01.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node01.example.com,"Linux 3.0.101-100-default",44,6.29 -"2010-02-10 00:00:01",192.168.0.2,udp,47074,node02.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node02.example.com,"Linux 2.6.9-103.ELsmp",48,6.86 -"2010-02-10 00:00:02",192.168.0.3,udp,177,node03.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node03.example.com,"1 user, load: 6,5, 6,6, 6,6",46,6.57 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv deleted file mode 100644 index 2e7b59158..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","protocol","asn","geo","region","city","hostname","naics","sector","tag","public_source","status","detail","method","device_vendor" -"2010-02-10 00:00:00",192.168.0.1,,,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",cyclops-blink,,"likely compromised",,, -"2010-02-10 00:00:01",192.168.0.2,,,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",cyclops-blink,,"likely compromised",,, -"2010-02-10 00:00:02",192.168.0.3,,,64512,ZZ,Region,City,node03.example.com,0,"Professional, Scientific, and Technical Services",cyclops-blink,,"likely compromised",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later From b5416c7ea1690304afae0a630dca2424baac3949 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:57:12 +0000 Subject: [PATCH 22/67] remove json parser - csv provides better performance --- .../shadowserver/collector_reports_api.py | 7 +- .../bots/parsers/shadowserver/parser_json.py | 171 ------------------ .../test_collector_reports_api.py | 7 +- 3 files changed, 7 insertions(+), 178 deletions(-) delete mode 100644 intelmq/bots/parsers/shadowserver/parser_json.py diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 5e0b045c8..dc8bd6b42 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv' or 'json'). The default is 'json' for compatibility. Using 'csv' is recommended for best performance. + file_format (str): File format to download ('csv'). The 'json' option is not longer supported. """ country = None @@ -67,11 +67,10 @@ def init(self): self._report_list.append(self.country) if self.file_format is not None: - if not (self.file_format == 'csv' or self.file_format == 'json'): + if not (self.file_format == 'csv'): raise ValueError('Invalid file_format') else: - self.file_format = 'json' - self.logger.info("For best performance, set 'file_format' to 'csv' and use intelmq.bots.parsers.shadowserver.parser.") + self.file_format = 'csv' self.preamble = f'{{ "apikey": "{self.api_key}" ' diff --git a/intelmq/bots/parsers/shadowserver/parser_json.py b/intelmq/bots/parsers/shadowserver/parser_json.py deleted file mode 100644 index 893ad877b..000000000 --- a/intelmq/bots/parsers/shadowserver/parser_json.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -Shadowserver JSON Parser - -SPDX-FileCopyrightText: 2020 Intelmq Team -SPDX-License-Identifier: AGPL-3.0-or-later -""" -import re -from typing import Any - -from intelmq.lib.bot import ParserBot -from intelmq.lib.exceptions import InvalidKey, InvalidValue -import intelmq.lib.message as libmessage -import intelmq.bots.parsers.shadowserver._config as config - - -class ShadowserverJSONParserBot(ParserBot): - """Parse all Shadowserver feeds in JSON format (data coming from the reports API) - Shadowserver JSON Parser - - Parameters: - feedname (str): The name of the feed - """ - __is_filename_regex = re.compile(r'^(?:\d{4}-\d{2}-\d{2}-)?(\w+)(-\w+)*\.json$') - feedname = None - _sparser_config = None - recover_line = ParserBot.recover_line_json - overwrite = True - - def init(self): - if self.feedname is not None: - feedname = self.feedname - self._sparser_config = config.get_feed_by_feedname(feedname) - if self._sparser_config: - self.logger.info('Using fixed feed name %r for parsing reports.', feedname) - else: - self.logger.info('Could not determine the feed by the feed name %r given by parameter. ' - 'Will determine the feed from the file names.', feedname) - - def parse(self, report): - report_name = report.get('extra.file_name') - if not report_name: - raise ValueError("No feedname given as parameter and the " - "processed report has no 'extra.file_name'. " - "Ensure that at least one is given. " - "Also have a look at the documentation of the bot.") - - filename_search = self.__is_filename_regex.search(report_name) - - if not filename_search: - raise ValueError(f"Report's 'extra.file_name' {report_name!r} is not valid.") - report_name = filename_search.group(1) - - self.logger.debug("Detected report's file name: %s.", report_name) - retval = config.get_feed_by_filename(report_name) - - if not retval: - raise ValueError('Could not get a config for {!r}, check the documentation.' - ''.format(report_name)) - self.feedname, self._sparser_config = retval - - return self.parse_json(report) - - def parse_line(self, line: Any, report: libmessage.Report): - conf = self._sparser_config - processedkeys = [] - - event = self.new_event(report) - event.add('feed.name', self.feedname, overwrite=self.overwrite) - - extra = {} - - for entry in conf.get('required_fields'): - intelmqkey, shadowserverkey = entry[0], entry[1] - value = self.get_value_from_config(line, entry) - - if value is not None: - event.add(intelmqkey, value) - processedkeys.append(shadowserverkey) - - # Now add optional fields. - # This action may fail, the value is added to - # extra if an add operation failed - for entry in conf.get('optional_fields'): - intelmqkey, shadowserverkey = entry[0], entry[1] - try: - value = self.get_value_from_config(line, entry) - except ValueError: - self.logger.warning('Optional key %s not found in feed %s. Possible change in data' - ' format or misconfiguration.', shadowserverkey, self.feedname) - continue - - intelmqkey, shadowserverkey = entry[0], entry[1] - if value is not None: - if intelmqkey == 'extra.': - extra[shadowserverkey] = value - processedkeys.append(shadowserverkey) - continue - elif intelmqkey and intelmqkey.startswith('extra.'): - extra[intelmqkey.replace('extra.', '', 1)] = value - processedkeys.append(shadowserverkey) - continue - elif intelmqkey is False: - # ignore it explicitly - processedkeys.append(shadowserverkey) - continue - try: - event.add(intelmqkey, value) - processedkeys.append(shadowserverkey) - except InvalidValue: - self.logger.debug('Could not add key %r in feed %r, adding it to extras.', - shadowserverkey, self.feedname) - except InvalidKey: - extra[intelmqkey] = value - processedkeys.append(shadowserverkey) - else: - processedkeys.append(shadowserverkey) - - # Now add additional constant fields. - event.update(conf.get('constant_fields', {})) - - event.add('raw', self.recover_line_json(line)) - - # Add everything which could not be resolved to extra. - for key in line: - if key not in processedkeys: - val = line[key] - if not val == "": - extra[key] = val - - if extra: - event.add('extra', extra) - - yield event - - def get_value_from_config(self, data, entry): - """ - Given a specific config, get the value for that data based on the entry - """ - conv_fun = None - - shadowserverkey = entry[1] - raw_value = data.get(shadowserverkey, None) - value = raw_value - - if raw_value is None: - raise ValueError('Key {!r} not found in feed {!r}. Possible change in data' - ' format or misconfiguration.'.format(shadowserverkey, self.feedname)) - if len(entry) > 2: - conv_fun = entry[2] - - if conv_fun is not None and raw_value is not None: - if len(entry) == 4 and entry[3]: - try: - value = conv_fun(raw_value, data) - except Exception: - self.logger.error('Could not convert shadowserverkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowserverkey, self.feedname, raw_value, conv_fun.__name__) - raise - else: - try: - value = conv_fun(raw_value) - except Exception: - self.logger.error('Could not convert shadowserverkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowserverkey, self.feedname, raw_value, conv_fun.__name__) - raise - return value - - -BOT = ShadowserverJSONParserBot diff --git a/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py b/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py index a625c9d34..2bf6e61e9 100644 --- a/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py +++ b/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py @@ -14,12 +14,13 @@ RANDSTR = secrets.token_urlsafe(50) ASSET_PATH = pathlib.Path(__file__).parent / 'reports-list.json' PARAMETERS = {'reports': 'anarres', 'api_key': RANDSTR, 'secret': RANDSTR, 'logging_level': 'DEBUG', 'types': ['scan_smb', 'cisco_smart_install', 'nonexistent'], 'name': 'shadowservercollector'} -REPORT = {'__type': 'Report', 'extra.file_name': '2020-08-02-scan_smb-anarres-geo.json', 'feed.accuracy': 100.0, 'feed.name': 'shadowservercollector', 'raw': 'e30='} +REPORT = {'__type': 'Report', 'extra.file_name': '2020-08-02-scan_smb-anarres-geo.csv', 'feed.accuracy': 100.0, 'feed.name': 'shadowservercollector', 'raw': 'e30='} def prepare_mocker(mocker): mocker.post('https://transform.shadowserver.org/api2/reports/list', content=ASSET_PATH.read_bytes()) - mocker.post('https://transform.shadowserver.org/api2/reports/download', text='{}') + mocker.get('https://dl.shadowserver.org/xNDSuwXrKnrLrDopU926rR75CAESMWesVCKsuyI8b8ncTv7GCX', text='{}') + mocker.get('https://dl.shadowserver.org/unnzVtn92tS9459rKIEz2J8qb7oJDv0Fa2feGUOiJLCDLqBXnN', text='{}') # Explicit skip_redis is required (although implicitly called by no_cache), otherwise fails in package build environments @@ -80,7 +81,7 @@ def test_report_sent(self, mocker): self.cache.flushdb() prepare_mocker(mocker) self.run_bot(iterations=1, parameters=PARAMETERS) - self.assertAnyLoglineEqual("Sent report: '2020-08-02-cisco_smart_install-anarres-geo.csv' (fixed: '2020-08-02-cisco_smart_install-anarres-geo.json', size: 0.00195 KiB).", 'DEBUG') + self.assertAnyLoglineEqual("Sent report: '2020-08-02-cisco_smart_install-anarres-geo.csv' (fixed: '2020-08-02-cisco_smart_install-anarres-geo.csv', size: 0.00195 KiB).", 'DEBUG') def test_report_content(self, mocker): self.cache.flushdb() From 876a41468db4d9d89a929b51abeb3df81644424d Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:59:42 +0000 Subject: [PATCH 23/67] dynamic configuration model --- intelmq/bots/parsers/shadowserver/README.md | 7 + intelmq/bots/parsers/shadowserver/_config.py | 4202 +---------------- intelmq/bots/parsers/shadowserver/parser.py | 46 +- .../parsers/shadowserver/schema.json.test | 180 + .../parsers/shadowserver/update_schema.py | 12 + 5 files changed, 303 insertions(+), 4144 deletions(-) create mode 100644 intelmq/bots/parsers/shadowserver/schema.json.test create mode 100644 intelmq/bots/parsers/shadowserver/update_schema.py diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index eb0ddfb4a..297930861 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,3 +7,10 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/_version_. + +For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. + +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory + +The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index bea3d0c0b..a7b80b7a6 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -77,20 +77,34 @@ feed_idx is not complete. """ +import os import re import base64 import binascii +import json +import urllib.request +import tempfile from typing import Optional, Dict, Tuple, Any import intelmq.lib.harmonization as harmonization +class __Container: + pass + +__config = __Container() +__config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') +__config.schema_mtime = 0.0 +__config.feedname_mapping = {} +__config.filename_mapping = {} def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: - return feedname_mapping.get(given_feedname, None) + reload() + return __config.feedname_mapping.get(given_feedname, None) def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, Any]]]: - return filename_mapping.get(given_filename, None) + reload() + return __config.filename_mapping.get(given_filename, None) def add_UTC_to_timestamp(value: str) -> str: @@ -165,11 +179,6 @@ def invalidate_zero(value: str) -> Optional[int]: return int(value) if value and int(value) != 0 else None -# TODO this function is a wild guess... -def set_tor_node(value: str) -> Optional[bool]: - return True if value else None - - def validate_ip(value: str) -> Optional[str]: """Remove "invalid" IP.""" # FIX: https://github.com/certtools/intelmq/issues/1720 # TODO: Find better fix @@ -240,4126 +249,63 @@ def scan_exchange_identifier(field): return 'exchange-server-webshell' return 'vulnerable-exchange-server' +functions = { + 'add_UTC_to_timestamp': add_UTC_to_timestamp, + 'convert_bool': convert_bool, + 'validate_to_none': validate_to_none, + 'convert_int': convert_int, + 'convert_float': convert_float, + 'convert_http_host_and_url': convert_http_host_and_url, + 'invalidate_zero': invalidate_zero, + 'validate_ip': validate_ip, + 'validate_network': validate_network, + 'validate_fqdn': validate_fqdn, + 'convert_date': convert_date, + 'convert_date_utc': convert_date_utc, + 'force_base64': force_base64, + 'scan_exchange_taxonomy': scan_exchange_taxonomy, + 'scan_exchange_type': scan_exchange_type, + 'scan_exchange_identifier': scan_exchange_identifier, + } + + +def reload (): + """ reload the configuration if it has changed """ + mtime = 0.0 + + if (os.path.isfile(__config.schema_file)): + mtime = os.path.getmtime(__config.schema_file) + if __config.schema_mtime == mtime: + return + schema_file = __config.schema_file + else: + # load a test schema if one has not been downloaded yet + schema_file = __config.schema_file + schema_file += '.test' + + __config.feedname_mapping.clear() + __config.filename_mapping.clear() + with open(schema_file) as fh: + schema = json.load(fh) + for report in schema: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + __config.schema_mtime = mtime + +def update_schema (version): + """ download the latest configuration """ + (th, tmp) = tempfile.mkstemp() + url = 'https://interchange.shadowserver.org/intelmq/'+version + try: + urllib.request.urlretrieve(url, tmp) + except: + raise ValueError("Failed to download %r" % url) -# BEGIN CONFGEN - -# https://www.shadowserver.org/what-we-do/network-reporting/blocklist-report/ -blocklist = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.network', 'ip', validate_network), - ('extra.', 'tag', validate_to_none), - ('source.reverse_dns', 'hostname'), - ('extra.', 'source', validate_to_none), - ('extra.', 'reason', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'blacklisted-ip', - 'classification.taxonomy': 'other', - 'classification.type': 'blacklist', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/compromised-website-report/ -compromised_website = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'application', validate_to_none), - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'http_host', validate_fqdn), - ('source.reverse_dns', 'hostname'), - ('malware.name', 'tag'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('event_description.text', 'category', validate_to_none), - ('extra.', 'system', validate_to_none), - ('extra.', 'detected_since', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'redirect_target', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'cc_url', validate_to_none), - ('extra.', 'family', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'classification.identifier': 'compromised-website', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/device-identification-report/ -device_id = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'undetermined', - 'classification.identifier': 'device-id', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ddos-participant-report/ -event_ddos_participant = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ('extra.', 'http_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'ddos-participant', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-brute-force-events-report/ -event_honeypot_brute_force = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'application'), - ('destination.account', 'username', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'service', validate_to_none), - ('extra.', 'start_time', convert_date_utc), - ('extra.', 'end_time', convert_date_utc), - ('extra.', 'client_version', validate_to_none), - ('extra.', 'password', validate_to_none), - ('extra.', 'payload_url', validate_to_none), - ('extra.', 'payload_md5', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'intrusion-attempts', - 'classification.type': 'brute-force', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-darknet-events-report/ -event_honeypot_darknet = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'tag', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'count', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ddos-events/ -event_honeypot_ddos = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ('extra.', 'http_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'honeypot-ddos', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-amplification-ddos-events-report/ -event_honeypot_ddos_amp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'avg_pps', convert_float), - ('extra.', 'max_pps', convert_float), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'count', convert_int), - ('extra.', 'bytes', convert_int), - ('extra.', 'end_time', convert_date_utc), - ('extra.', 'duration', convert_int), - ], - 'constant_fields': { - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ddos-target-events-report/ -event_honeypot_ddos_target = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'honeypot-ddos-target', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-http-scanner-events/ -event_honeypot_http_scan = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('user_agent', 'http_agent', validate_to_none), - ('extra.method', 'http_request_method', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'pattern', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('extra.', 'url_scheme', validate_to_none), - ('extra.', 'session_tags', validate_to_none), - ('extra.', 'vulnerability_enum', validate_to_none), - ('extra.', 'vulnerability_id', validate_to_none), - ('extra.', 'vulnerability_class', validate_to_none), - ('extra.', 'vulnerability_score', validate_to_none), - ('extra.', 'vulnerability_severity', validate_to_none), - ('extra.', 'vulnerability_version', validate_to_none), - ('extra.', 'threat_framework', validate_to_none), - ('extra.', 'threat_tactic_id', validate_to_none), - ('extra.', 'threat_technique_id', validate_to_none), - ('extra.', 'target_vendor', validate_to_none), - ('extra.', 'target_product', validate_to_none), - ('extra.', 'target_class', validate_to_none), - ('extra.', 'file_md5', validate_to_none), - ('extra.', 'file_sha256', validate_to_none), - ('extra.', 'request_raw', force_base64), - ('extra.', 'body_raw', force_base64), - ], - 'constant_fields': { - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'protocol.application': 'http', - 'classification.identifier': 'honeypot-http-scan', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ics-scanner-events-report/ -event_honeypot_ics_scan = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'state', validate_to_none), - ('extra.', 'sensor_id', validate_to_none), - ('extra.', 'slave_id', validate_to_none), - ('extra.', 'function_code', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'response', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'ics', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ip-spoofer-events-report/ -event_ip_spoofer = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'infection', validate_to_none), - ('source.network', 'network', validate_network), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'routedspoof', validate_to_none), - ('extra.', 'session', validate_to_none), - ('extra.', 'nat', convert_bool), - ], - 'constant_fields': { - 'classification.taxonomy': 'fraud', - 'classification.type': 'masquerade', - 'classification.identifier': 'ip-spoofer', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-events-report/ -event_sinkhole = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'infection', validate_to_none), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-dns-events-report/ -event_sinkhole_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.naics', 'src_naics', invalidate_zero), - ('extra.sector', 'src_sector', validate_to_none), - ('extra.dns_query_type', 'query_type'), - ('extra.dns_query', 'query'), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'count', convert_int), - ], - 'constant_fields': { - 'classification.identifier': 'sinkholedns', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-events-report/ -event_sinkhole_http = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'tag'), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_agent', validate_to_none), - ('extra.', 'forwarded_by', validate_to_none), - ('extra.', 'ssl_cipher', validate_to_none), - ('extra.', 'http_referer', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-referer-events-report/ -event_sinkhole_http_referer = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ], - 'optional_fields': [ - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('extra.', 'http_referer_ip', validate_ip), - ('extra.', 'http_referer_port', convert_int), - ('extra.', 'http_referer_asn', invalidate_zero), - ('extra.', 'http_referer_geo', validate_to_none), - ('extra.', 'http_referer_region', validate_to_none), - ('extra.', 'http_referer_city', validate_to_none), - ('extra.', 'http_referer_hostname', validate_to_none), - ('extra.', 'http_referer_naics', invalidate_zero), - ('extra.', 'http_referer_sector', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_referer', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'sinkhole-http-referer', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/malware-url-report/ -malware_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'source', validate_to_none), - ('malware.hash.sha256', 'sha256', validate_to_none), - ('extra.', 'application', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'malware-url', - }, -} - -phish_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'source', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'fraud', - 'classification.type': 'phishing', - 'classification.identifier': 'phish-url', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-http-proxy-report/ -population_http_proxy = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('malware.name', 'tag'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'proxy_authenticate', validate_to_none), - ('extra.', 'via', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'http', - }, -} - -# http://www.shadowserver.org/wiki/pmwiki.php/Services/Sandbox-Connection -sandbox_conn = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('destination.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('malware.hash.md5', 'md5', validate_to_none), - ('protocol.transport', 'protocol'), - ('extra.', 'bytes_in', validate_to_none), - ('extra.', 'bytes_out', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'sandbox-conn', - }, -} - -sandbox_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ], - 'optional_fields': [ - ('extra.dns_query_type', 'type', validate_to_none), - ('malware.hash.md5', 'md5hash', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'response', validate_to_none), - ('extra.', 'family', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'source', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'dns', - 'classification.identifier': 'sandbox-dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sandbox-url-report/ -sandbox_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('destination.fqdn', 'host', validate_fqdn), - ('extra.http_request_method', 'method', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('malware.hash.md5', 'md5', validate_to_none), - ('destination.url', 'url', convert_http_host_and_url, True), - ('user_agent', 'user_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'sandbox-url', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-adb-report/ -scan_adb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'name', validate_to_none), - ('extra.', 'model', validate_to_none), - ('extra.', 'device', validate_to_none), - ('extra.', 'features', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-adb', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'adb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-afp-report/ -scan_afp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'machine_type', validate_to_none), - ('extra.', 'afp_versions', validate_to_none), - ('extra.', 'uams', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'signature', validate_to_none), - ('extra.', 'directory_service', validate_to_none), - ('extra.', 'utf8_servername', validate_to_none), - ('extra.', 'network_address', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-afp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'afp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-amqp-report/ -scan_amqp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'channel', validate_to_none), - ('extra.', 'message_length', convert_int), - ('extra.', 'class', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'version_major', validate_to_none), - ('extra.', 'version_minor', validate_to_none), - ('extra.', 'capabilities', validate_to_none), - ('extra.', 'cluster_name', validate_to_none), - ('extra.', 'platform', validate_to_none), - ('extra.', 'product', validate_to_none), - ('extra.', 'product_version', validate_to_none), - ('extra.', 'mechanisms', validate_to_none), - ('extra.', 'locales', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-amqp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'amqp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-apple-remote-desktop-ard-report/ -scan_ard = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-chargen-report/ -scan_chargen = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.response_size', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'chargen', - 'classification.identifier': 'open-chargen', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-cisco-smart-install-report/ -scan_cisco_smart_install = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'cisco-smart-install', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-coap-report/ -scan_coap = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'response', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'coap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-couchdb-report/ -scan_couchdb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'server_version', validate_to_none), - ('extra.', 'couchdb_message', validate_to_none), - ('extra.', 'couchdb_version', validate_to_none), - ('extra.', 'git_sha', validate_to_none), - ('extra.', 'features', validate_to_none), - ('extra.', 'vendor', validate_to_none), - ('extra.', 'visible_databases', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'error_reason', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'CouchDB', - 'classification.identifier': 'open-couchdb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-cwmp-report/ -scan_cwmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'cwmp', - 'classification.identifier': 'open-cwmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-db2-discovery-service-report/ -scan_db2 = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'db2_hostname', validate_to_none), - ('extra.', 'servername', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'db2', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-ddos-middlebox-report/ -scan_ddos_middlebox = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'source_port', validate_to_none), - ('extra.', 'bytes', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'method', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ddos-middlebox', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/dns-open-resolvers-report/ -scan_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'min_amplification', convert_float), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'dns_version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'dns-open-resolver', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-docker-service-report/ -scan_docker = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'experimental', validate_to_none), - ('extra.', 'api_version', validate_to_none), - ('extra.', 'arch', validate_to_none), - ('extra.', 'go_version', validate_to_none), - ('extra.os.name', 'os', validate_to_none), - ('extra.', 'kernel_version', validate_to_none), - ('extra.', 'git_commit', validate_to_none), - ('extra.', 'min_api_version', validate_to_none), - ('extra.', 'build_time', validate_to_none), - ('extra.', 'pkg_version', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'docker', - 'classification.identifier': 'open-docker', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-dvr-dhcpdiscover-report/ -scan_dvr_dhcpdiscover = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('extra.', 'video_input_channels', convert_int), - ('extra.', 'alarm_input_channels', convert_int), - ('extra.', 'video_output_channels', convert_int), - ('extra.', 'alarm_output_channels', convert_int), - ('extra.', 'remote_video_input_channels', convert_int), - ('extra.', 'ipv4_dhcp_enable', convert_bool), - ('extra.', 'ipv6_dhcp_enable', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_id', validate_to_none), - ('extra.', 'device_serial', validate_to_none), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'manufacturer', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'http_port', convert_int), - ('extra.', 'internal_port', convert_int), - ('extra.', 'mac_address', validate_to_none), - ('extra.', 'ipv4_address', validate_to_none), - ('extra.', 'ipv4_gateway', validate_to_none), - ('extra.', 'ipv4_subnet_mask', validate_to_none), - ('extra.', 'ipv6_address', validate_to_none), - ('extra.', 'ipv6_link_local', validate_to_none), - ('extra.', 'ipv6_gateway', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-dvr-dhcpdiscover', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-elasticsearch-report/ -scan_elasticsearch = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'build_snapshot', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'ok', convert_bool), - ('extra.', 'name', validate_to_none), - ('extra.', 'cluster_name', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'build_hash', validate_to_none), - ('extra.', 'build_timestamp', validate_to_none), - ('extra.', 'lucene_version', validate_to_none), - ('extra.', 'tagline', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'elasticsearch', - 'classification.identifier': 'open-elasticsearch', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-erlang-port-mapper-report-daemon/ -scan_epmd = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'nodes', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'Erlang Port Mapper Daemon', - 'classification.identifier': 'open-epmd', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-exchange-server-report/ -scan_exchange = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('classification.taxonomy', 'tag', scan_exchange_taxonomy), - ('classification.type', 'tag', scan_exchange_type), - ('classification.identifier', 'tag', scan_exchange_identifier), - ('extra.', 'tag', validate_to_none), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'servername', validate_to_none), - ('destination.url', 'url', convert_http_host_and_url, True), - ], - 'constant_fields': { - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ftp-report/ -scan_ftp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'auth_tls_response', validate_to_none), - ('extra.', 'auth_ssl_response', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ftp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-hadoop-report/ -scan_hadoop = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'total_disk', convert_int), - ('extra.', 'used_disk', convert_int), - ('extra.', 'free_disk', convert_int), - ('source.reverse_dns', 'hostname'), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'clusterid', validate_to_none), - ('extra.', 'livenodes', validate_to_none), - ('extra.', 'namenodeaddress', validate_to_none), - ('extra.', 'volumeinfo', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-http-report/ -scan_http = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-http-proxy-report/ -scan_http_proxy = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'proxy_authenticate', validate_to_none), - ('extra.', 'via', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-http-report/ -scan_http_vulnerable = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'version', validate_to_none), - ('extra.', 'build_date', validate_to_none), - ('extra.', 'detail', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ics-report/ -scan_ics = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_id', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ics', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ipmi-report/ -scan_ipmi = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'none_auth', convert_bool), - ('extra.', 'md2_auth', convert_bool), - ('extra.', 'md5_auth', convert_bool), - ('extra.', 'passkey_auth', convert_bool), - ('extra.', 'oem_auth', convert_bool), - ('extra.', 'permessage_auth', convert_bool), - ('extra.', 'userlevel_auth', convert_bool), - ('extra.', 'usernames', convert_bool), - ('extra.', 'nulluser', convert_bool), - ('extra.', 'anon_login', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'ipmi_version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'defaultkg', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'deviceid', validate_to_none), - ('extra.', 'devicerev', validate_to_none), - ('extra.', 'firmwarerev', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'manufacturerid', validate_to_none), - ('extra.', 'manufacturername', validate_to_none), - ('extra.', 'productid', validate_to_none), - ('extra.', 'productname', validate_to_none), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipmi', - 'protocol.transport': 'udp', - 'classification.identifier': 'open-ipmi', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ipp-report/ -scan_ipp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'ipp_version', validate_to_none), - ('extra.', 'cups_version', validate_to_none), - ('extra.', 'printer_uris', validate_to_none), - ('extra.', 'printer_name', validate_to_none), - ('extra.', 'printer_info', validate_to_none), - ('extra.', 'printer_more_info', validate_to_none), - ('extra.', 'printer_make_and_model', validate_to_none), - ('extra.', 'printer_firmware_name', validate_to_none), - ('extra.', 'printer_firmware_string_version', validate_to_none), - ('extra.', 'printer_firmware_version', validate_to_none), - ('extra.', 'printer_organization', validate_to_none), - ('extra.', 'printer_organization_unit', validate_to_none), - ('extra.', 'printer_uuid', validate_to_none), - ('extra.', 'printer_wifi_ssid', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipp', - 'classification.identifier': 'open-ipp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-isakmp-report/ -scan_isakmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'spi_size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'initiator_spi', validate_to_none), - ('extra.', 'responder_spi', validate_to_none), - ('extra.', 'next_payload', validate_to_none), - ('extra.', 'exchange_type', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'message_id', validate_to_none), - ('extra.', 'next_payload2', validate_to_none), - ('extra.', 'domain_of_interpretation', validate_to_none), - ('extra.', 'protocol_id', validate_to_none), - ('extra.', 'notify_message_type', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'open-ike', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipsec', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-kubernetes-api-server-report/ -scan_kubernetes = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'major', validate_to_none), - ('extra.', 'minor', validate_to_none), - ('extra.', 'git_version', validate_to_none), - ('extra.', 'git_commit', validate_to_none), - ('extra.', 'git_tree_state', validate_to_none), - ('extra.', 'build_date', validate_to_none), - ('extra.', 'go_version', validate_to_none), - ('extra.', 'compiler', validate_to_none), - ('extra.', 'platform', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'kubernetes', - 'classification.identifier': 'open-kubernetes', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ldap-tcp-report/ -scan_ldap_tcp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'dns_host_name', validate_to_none), - ('extra.', 'domain_controller_functionality', convert_int), - ('extra.', 'domain_functionality', convert_int), - ('extra.', 'forest_functionality', convert_int), - ('extra.', 'highest_committed_usn', convert_int), - ('extra.', 'is_global_catalog_ready', convert_bool), - ('extra.', 'is_synchronized', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'size', convert_int), - ('extra.', 'configuration_naming_context', validate_to_none), - ('extra.', 'current_time', validate_to_none), - ('extra.', 'default_naming_context', validate_to_none), - ('extra.', 'ds_service_name', validate_to_none), - ('extra.', 'ldap_service_name', validate_to_none), - ('extra.', 'naming_contexts', validate_to_none), - ('extra.', 'root_domain_naming_context', validate_to_none), - ('extra.', 'schema_naming_context', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'subschema_subentry', validate_to_none), - ('extra.', 'supported_capabilities', validate_to_none), - ('extra.', 'supported_control', validate_to_none), - ('extra.', 'supported_ldap_policies', validate_to_none), - ('extra.', 'supported_ldap_version', validate_to_none), - ('extra.', 'supported_sasl_mechanisms', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ldap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ldap-report/ -scan_ldap_udp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'dns_host_name', validate_to_none), - ('extra.', 'domain_controller_functionality', convert_int), - ('extra.', 'domain_functionality', convert_int), - ('extra.', 'forest_functionality', convert_int), - ('extra.', 'highest_committed_usn', convert_int), - ('extra.', 'is_global_catalog_ready', convert_bool), - ('extra.', 'is_synchronized', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'size', convert_int), - ('extra.', 'configuration_naming_context', validate_to_none), - ('extra.', 'current_time', validate_to_none), - ('extra.', 'default_naming_context', validate_to_none), - ('extra.', 'ds_service_name', validate_to_none), - ('extra.', 'ldap_service_name', validate_to_none), - ('extra.', 'naming_contexts', validate_to_none), - ('extra.', 'root_domain_naming_context', validate_to_none), - ('extra.', 'schema_naming_context', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'subschema_subentry', validate_to_none), - ('extra.', 'supported_capabilities', validate_to_none), - ('extra.', 'supported_control', validate_to_none), - ('extra.', 'supported_ldap_policies', validate_to_none), - ('extra.', 'supported_ldap_version', validate_to_none), - ('extra.', 'supported_sasl_mechanisms', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ldap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mdns-report/ -scan_mdns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'mdns_name', validate_to_none), - ('extra.', 'mdns_ipv4', validate_to_none), - ('extra.', 'mdns_ipv6', validate_to_none), - ('extra.', 'services', validate_to_none), - ('extra.', 'workstation_name', validate_to_none), - ('extra.', 'workstation_ipv4', validate_to_none), - ('extra.', 'workstation_ipv6', validate_to_none), - ('extra.', 'workstation_info', validate_to_none), - ('extra.', 'http_name', validate_to_none), - ('extra.', 'http_ipv4', validate_to_none), - ('extra.', 'http_ipv6', validate_to_none), - ('extra.', 'http_ptr', validate_to_none), - ('extra.', 'http_info', validate_to_none), - ('extra.', 'http_target', validate_to_none), - ('extra.', 'http_port', convert_int), - ('extra.', 'spotify_name', validate_to_none), - ('extra.', 'spotify_ipv4', validate_to_none), - ('extra.', 'spotify_ipv6', validate_to_none), - ('extra.', 'opc_ua_discovery', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mdns', - 'classification.identifier': 'open-mdns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-memcached-report/ -scan_memcached = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'pid', convert_int), - ('extra.', 'pointer_size', convert_int), - ('extra.', 'uptime', convert_int), - ('extra.', 'curr_connections', convert_int), - ('extra.', 'total_connections', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'time', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'memcached', - 'classification.identifier': 'open-memcached', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mongodb-report/ -scan_mongodb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'gitversion', validate_to_none), - ('extra.', 'sysinfo', validate_to_none), - ('extra.', 'opensslversion', validate_to_none), - ('extra.', 'allocator', validate_to_none), - ('extra.', 'javascriptengine', validate_to_none), - ('extra.', 'bits', validate_to_none), - ('extra.', 'maxbsonobjectsize', validate_to_none), - ('extra.', 'ok', convert_bool), - ('extra.', 'visible_databases', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mongodb', - 'classification.identifier': 'open-mongodb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mqtt-report/ -scan_mqtt = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'anonymous_access', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'raw_response', validate_to_none), - ('extra.', 'hex_code', validate_to_none), - ('extra.', 'code', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serialNumber', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mqtt', - 'classification.identifier': 'open-mqtt', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mqtt-report/ -scan_mqtt_anon = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'raw_response', validate_to_none), - ('extra.', 'hex_code', validate_to_none), - ('extra.', 'code', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serialNumber', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mqtt', - 'classification.identifier': 'open-mqtt-anon', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-MSSQL -scan_mssql = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'server_name', validate_to_none), - ('extra.', 'tcp_port', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'instance_name', validate_to_none), - ('extra.', 'named_pipe', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mssql', - 'classification.identifier': 'open-mssql', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-mysql-server-report/ -scan_mysql = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'client_can_handle_expired_passwords', convert_bool), - ('extra.', 'client_compress', convert_bool), - ('extra.', 'client_connect_attrs', convert_bool), - ('extra.', 'client_connect_with_db', convert_bool), - ('extra.', 'client_deprecated_eof', convert_bool), - ('extra.', 'client_found_rows', convert_bool), - ('extra.', 'client_ignore_sigpipe', convert_bool), - ('extra.', 'client_ignore_space', convert_bool), - ('extra.', 'client_interactive', convert_bool), - ('extra.', 'client_local_files', convert_bool), - ('extra.', 'client_long_flag', convert_bool), - ('extra.', 'client_long_password', convert_bool), - ('extra.', 'client_multi_results', convert_bool), - ('extra.', 'client_multi_statements', convert_bool), - ('extra.', 'client_no_schema', convert_bool), - ('extra.', 'client_odbc', convert_bool), - ('extra.', 'client_plugin_auth', convert_bool), - ('extra.', 'client_plugin_auth_len_enc_client_data', convert_bool), - ('extra.', 'client_protocol_41', convert_bool), - ('extra.', 'client_ps_multi_results', convert_bool), - ('extra.', 'client_reserved', convert_bool), - ('extra.', 'client_secure_connection', convert_bool), - ('extra.', 'client_session_track', convert_bool), - ('extra.', 'client_ssl', convert_bool), - ('extra.', 'client_transactions', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'mysql_protocol_version', validate_to_none), - ('extra.', 'server_version', validate_to_none), - ('extra.', 'error_code', validate_to_none), - ('extra.', 'error_id', validate_to_none), - ('extra.', 'error_message', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'mysql', - 'classification.identifier': 'open-mysql', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-NATPMP -scan_nat_pmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'uptime', convert_int), - ('extra.', 'external_ip', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'natpmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-netbios-report/ -scan_netbios = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.account', 'username'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'mac_address', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'workgroup', validate_to_none), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'netbios-nameservice', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/netcore-netis-router-vulnerability-scan-report/ -scan_netis_router = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'response', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.transport': 'udp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ntp-version-report/ -scan_ntp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'clk_wander', convert_float), - ('extra.', 'frequency', convert_float), - ('extra.', 'jitter', convert_float), - ('extra.', 'leap', convert_float), - ('extra.', 'offset', convert_float), - ('extra.', 'peer', convert_int), - ('extra.', 'poll', convert_int), - ('extra.', 'precision', convert_int), - ('extra.', 'rootdelay', convert_float), - ('extra.', 'rootdispersion', convert_float), - ('extra.', 'stratum', convert_int), - ('extra.', 'tc', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'version', validate_to_none), - ('extra.', 'clock', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'mintc', validate_to_none), - ('extra.', 'noise', validate_to_none), - ('extra.', 'phase', validate_to_none), - ('extra.', 'processor', validate_to_none), - ('extra.', 'refid', validate_to_none), - ('extra.', 'reftime', validate_to_none), - ('extra.', 'stability', validate_to_none), - ('extra.', 'state', validate_to_none), - ('extra.', 'system', validate_to_none), - ('extra.', 'tai', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ntp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ntp-monitor-report/ -scan_ntpmonitor = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'packets', convert_int), - ('extra.', 'size', convert_int), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ntp', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Portmapper -scan_portmapper = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'programs', validate_to_none), - ('extra.', 'mountd_port', validate_to_none), - ('extra.', 'exports', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'portmapper', - 'classification.identifier': 'open-portmapper', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-postgresql-server-report/ -scan_postgres = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'startup_error_line', convert_int), - ('extra.', 'client_ssl', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'supported_protocols', validate_to_none), - ('extra.', 'protocol_error_code', validate_to_none), - ('extra.', 'protocol_error_file', validate_to_none), - ('extra.', 'protocol_error_line', validate_to_none), - ('extra.', 'protocol_error_message', validate_to_none), - ('extra.', 'protocol_error_routine', validate_to_none), - ('extra.', 'protocol_error_severity', validate_to_none), - ('extra.', 'protocol_error_severity_v', validate_to_none), - ('extra.', 'startup_error_code', validate_to_none), - ('extra.', 'startup_error_file', validate_to_none), - ('extra.', 'startup_error_message', validate_to_none), - ('extra.', 'startup_error_routine', validate_to_none), - ('extra.', 'startup_error_severity', validate_to_none), - ('extra.', 'startup_error_severity_v', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'postgres', - 'classification.identifier': 'open-postgres', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-QOTD -scan_qotd = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'quote', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'qotd', - 'classification.identifier': 'open-qotd', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-quic-report/ -scan_quic = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'version_field_1', validate_to_none), - ('extra.', 'version_field_2', validate_to_none), - ('extra.', 'version_field_3', validate_to_none), - ('extra.', 'version_field_4', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'open-quic', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-radmin-report/ -scan_radmin = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-radmin', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-rdp-report/ -scan_rdp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'cve20190708_vulnerable', convert_bool), - ('extra.', 'bluekeep_vulnerable', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'rdp_protocol', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'rdp', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-rdp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ms-rdpeudp/ -scan_rdpeudp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sessionid', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Redis -scan_redis = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'git_sha1', validate_to_none), - ('extra.', 'git_dirty_flag', validate_to_none), - ('extra.', 'build_id', validate_to_none), - ('extra.', 'mode', validate_to_none), - ('extra.os.name', 'os', validate_to_none), - ('extra.', 'architecture', validate_to_none), - ('extra.', 'multiplexing_api', validate_to_none), - ('extra.', 'gcc_version', validate_to_none), - ('extra.', 'process_id', validate_to_none), - ('extra.', 'run_id', validate_to_none), - ('extra.', 'uptime', convert_int), - ('extra.', 'connected_clients', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'redis', - 'classification.identifier': 'open-redis', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-rsync-report/ -scan_rsync = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'module', validate_to_none), - ('extra.', 'motd', validate_to_none), - ('extra.', 'has_password', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'rsync', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-sip-report/ -scan_sip = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'sip', validate_to_none), - ('extra.', 'sip_code', validate_to_none), - ('extra.', 'sip_reason', validate_to_none), - ('user_agent', 'user_agent', validate_to_none), - ('extra.', 'sip_via', validate_to_none), - ('extra.', 'sip_to', validate_to_none), - ('extra.', 'sip_from', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'content_type', validate_to_none), - ('extra.sip_server', 'server', validate_to_none), - ('extra.sip_contact', 'contact', validate_to_none), - ('extra.sip_cseq', 'cseq', validate_to_none), - ('extra.sip_call_id', 'call_id', validate_to_none), - ('extra.sip_allow', 'allow', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'sip', - 'classification.identifier': 'open-sip', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-slp-service-report/ -scan_slp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'function', validate_to_none), - ('extra.', 'function_text', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'next_extension_offset', validate_to_none), - ('extra.', 'xid', validate_to_none), - ('extra.', 'language_tag_length', validate_to_none), - ('extra.', 'language_tag', validate_to_none), - ('extra.', 'error_code', validate_to_none), - ('extra.', 'error_code_text', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'slp', - 'classification.identifier': 'open-slp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-smb-report/ -scan_smb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'smb_implant', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'arch', validate_to_none), - ('extra.', 'key', validate_to_none), - ('extra.', 'smbv1_support', validate_to_none), - ('extra.', 'smb_major_number', validate_to_none), - ('extra.', 'smb_minor_number', validate_to_none), - ('extra.', 'smb_revision', validate_to_none), - ('extra.', 'smb_version_string', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'smb', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-smb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-smtp-report/ -scan_smtp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'smtp', - 'classification.identifier': 'open-smtp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-smtp-report/ -scan_smtp_vulnerable = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'smtp', - 'classification.identifier': 'vulnerable-smtp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-snmp-report/ -scan_snmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'sysdesc', validate_to_none), - ('extra.', 'sysname', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'version', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'community', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'snmp', - 'classification.identifier': 'open-snmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-socks4-5-proxy-report/ -scan_socks = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-socks', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-SSDP -scan_ssdp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'header', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'systime', validate_to_none), - ('extra.', 'cache_control', validate_to_none), - ('extra.', 'location', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'search_target', validate_to_none), - ('extra.', 'unique_service_name', validate_to_none), - ('extra.', 'host', validate_to_none), - ('extra.', 'nts', validate_to_none), - ('extra.', 'nt', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'server_port', validate_to_none), - ('extra.', 'instance', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'updated_at', validate_to_none), - ('extra.', 'resource_identifier', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ssdp', - 'classification.identifier': 'open-ssdp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ssh-report/ -scan_ssh = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'serverid_raw', validate_to_none), - ('extra.', 'serverid_version', validate_to_none), - ('extra.', 'serverid_software', validate_to_none), - ('extra.', 'serverid_comment', validate_to_none), - ('extra.', 'server_cookie', validate_to_none), - ('extra.', 'available_kex', validate_to_none), - ('extra.', 'available_ciphers', validate_to_none), - ('extra.', 'available_mac', validate_to_none), - ('extra.', 'available_compression', validate_to_none), - ('extra.', 'selected_kex', validate_to_none), - ('extra.', 'algorithm', validate_to_none), - ('extra.', 'selected_cipher', validate_to_none), - ('extra.', 'selected_mac', validate_to_none), - ('extra.', 'selected_compression', validate_to_none), - ('extra.', 'server_signature_value', validate_to_none), - ('extra.', 'server_signature_raw', validate_to_none), - ('extra.', 'server_host_key', validate_to_none), - ('extra.', 'server_host_key_sha256', validate_to_none), - ('extra.', 'rsa_prime', validate_to_none), - ('extra.', 'rsa_prime_length', validate_to_none), - ('extra.', 'rsa_generator', validate_to_none), - ('extra.', 'rsa_generator_length', validate_to_none), - ('extra.', 'rsa_public_key', validate_to_none), - ('extra.', 'rsa_public_key_length', validate_to_none), - ('extra.', 'rsa_exponent', validate_to_none), - ('extra.', 'rsa_modulus', validate_to_none), - ('extra.', 'rsa_length', validate_to_none), - ('extra.', 'dss_prime', validate_to_none), - ('extra.', 'dss_prime_length', validate_to_none), - ('extra.', 'dss_generator', validate_to_none), - ('extra.', 'dss_generator_length', validate_to_none), - ('extra.', 'dss_public_key', validate_to_none), - ('extra.', 'dss_public_key_length', validate_to_none), - ('extra.', 'dss_dsa_public_g', validate_to_none), - ('extra.', 'dss_dsa_public_p', validate_to_none), - ('extra.', 'dss_dsa_public_q', validate_to_none), - ('extra.', 'dss_dsa_public_y', validate_to_none), - ('extra.', 'ecdsa_curve25519', validate_to_none), - ('extra.', 'ecdsa_curve', validate_to_none), - ('extra.', 'ecdsa_public_key_length', validate_to_none), - ('extra.', 'ecdsa_public_key_b', validate_to_none), - ('extra.', 'ecdsa_public_key_gx', validate_to_none), - ('extra.', 'ecdsa_public_key_gy', validate_to_none), - ('extra.', 'ecdsa_public_key_n', validate_to_none), - ('extra.', 'ecdsa_public_key_p', validate_to_none), - ('extra.', 'ecdsa_public_key_x', validate_to_none), - ('extra.', 'ecdsa_public_key_y', validate_to_none), - ('extra.', 'ed25519_curve25519', validate_to_none), - ('extra.', 'ed25519_cert_public_key_nonce', validate_to_none), - ('extra.', 'ed25519_cert_public_key_bytes', validate_to_none), - ('extra.', 'ed25519_cert_public_key_raw', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sha256', validate_to_none), - ('extra.', 'ed25519_cert_public_key_serial', validate_to_none), - ('extra.', 'ed25519_cert_public_key_type_id', validate_to_none), - ('extra.', 'ed25519_cert_public_key_type_name', validate_to_none), - ('extra.', 'ed25519_cert_public_key_keyid', validate_to_none), - ('extra.', 'ed25519_cert_public_key_principles', validate_to_none), - ('extra.', 'ed25519_cert_public_key_valid_after', validate_to_none), - ('extra.', 'ed25519_cert_public_key_valid_before', validate_to_none), - ('extra.', 'ed25519_cert_public_key_duration', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_bytes', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_raw', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_sha256', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_value', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sig_raw', validate_to_none), - ('extra.', 'banner', validate_to_none), - ('extra.', 'userauth_methods', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'open-ssh', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ssl-report/ -scan_ssl = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'ssl_poodle', convert_bool), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'freak_vulnerable', convert_bool), - ('extra.', 'freak_cipher_suite', validate_to_none), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'https', - 'classification.identifier': 'open-ssl', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Freak-Scan -scan_ssl_freak = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'freak_vulnerable', convert_bool), - ('extra.', 'freak_cipher_suite', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'page_sha256fp', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'https', - 'classification.identifier': 'ssl-freak', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Scan -scan_ssl_poodle = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'ssl_poodle', convert_bool), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'page_sha256fp', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'https', - 'classification.identifier': 'ssl-poodle', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-stun-service-report/ -scan_stun = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'mapped_port', convert_int), - ('extra.', 'xor_mapped_port', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'transaction_id', validate_to_none), - ('extra.', 'magic_cookie', validate_to_none), - ('extra.', 'message_length', convert_int), - ('extra.', 'message_type', validate_to_none), - ('extra.', 'mapped_family', validate_to_none), - ('extra.', 'mapped_address', validate_to_none), - ('extra.', 'xor_mapped_family', validate_to_none), - ('extra.', 'xor_mapped_address', validate_to_none), - ('extra.', 'software', validate_to_none), - ('extra.', 'fingerprint', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'Session Traversal Utilities for NAT', - 'classification.identifier': 'open-stun', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/synful-scan-report/ -scan_synfulknock = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'ack_number', convert_int), - ('extra.', 'window_size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'sequence_number', validate_to_none), - ('extra.', 'urgent_pointer', validate_to_none), - ('extra.', 'tcp_flags', validate_to_none), - ('extra.', 'raw_packet', validate_to_none), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-synfulknock', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-telnet-report/ -scan_telnet = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'telnet', - 'classification.identifier': 'open-telnet', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-TFTP -scan_tftp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'errorcode', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'errormessage', validate_to_none), - ('extra.', 'size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'tftp', - 'classification.identifier': 'open-tftp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ubiquiti-report/ -scan_ubiquiti = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.mac_address', 'mac', validate_to_none), - ('extra.radio_name', 'radioname', validate_to_none), - ('extra.model', 'modelshort', validate_to_none), - ('extra.model_full', 'modelfull', validate_to_none), - ('extra.firmwarerev', 'firmware', validate_to_none), - ('extra.response_size', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'essid', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-vnc-report/ -scan_vnc = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'product', validate_to_none), - ('extra.', 'banner', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'vnc', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-vnc', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ws-discovery-service-report/ -scan_ws_discovery = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'error', validate_to_none), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ws-discovery', - 'classification.identifier': 'open-ws-discovery', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-xdmcp-service-report/ -scan_xdmcp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'reported_hostname', validate_to_none), - ('extra.', 'status', validate_to_none), - ('extra.', 'size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'xdmcp', - 'classification.identifier': 'open-xdmcp', - }, -} - -# http://www.shadowserver.org/wiki/pmwiki.php/Services/Spam-URL -spam_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'source', validate_to_none), - ('extra.', 'sender', validate_to_none), - ('extra.', 'subject', validate_to_none), - ('malware.hash.md5', 'md5', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'abusive-content', - 'classification.type': 'spam', - 'classification.identifier': 'spam-url', - }, -} - -special = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('source.reverse_dns', 'hostname'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'status', validate_to_none), - ('extra.', 'detail', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'special', - }, -} - -mapping = ( - # feed name, file name, function - ('Blocklist', 'blocklist', blocklist), - ('Compromised-Website', 'compromised_website', compromised_website), - ('Device-Identification IPv4', 'device_id', device_id), - ('Device-Identification IPv6', 'device_id6', device_id), - ('DDoS-Participant', 'event4_ddos_participant', event_ddos_participant), - ('Honeypot-Brute-Force-Events', 'event4_honeypot_brute_force', event_honeypot_brute_force), - ('Honeypot-Darknet', 'event4_honeypot_darknet', event_honeypot_darknet), - ('Honeypot-DDoS', 'event4_honeypot_ddos', event_honeypot_ddos), - ('Honeypot-Amplification-DDoS-Events', 'event4_honeypot_ddos_amp', event_honeypot_ddos_amp), - ('Honeypot-DDoS-Target', 'event4_honeypot_ddos_target', event_honeypot_ddos_target), - ('Honeypot-HTTP-Scan', 'event4_honeypot_http_scan', event_honeypot_http_scan), - ('Honeypot-ICS-Scanner', 'event4_honeypot_ics_scan', event_honeypot_ics_scan), - ('IP-Spoofer-Events', 'event4_ip_spoofer', event_ip_spoofer), - ('Microsoft-Sinkhole-Events IPv4', 'event4_microsoft_sinkhole', event_sinkhole), - ('Microsoft-Sinkhole-Events-HTTP IPv4', 'event4_microsoft_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events IPv4', 'event4_sinkhole', event_sinkhole), - ('Sinkhole-DNS', 'event4_sinkhole_dns', event_sinkhole_dns), - ('Sinkhole-Events-HTTP IPv4', 'event4_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer IPv4', 'event4_sinkhole_http_referer', event_sinkhole_http_referer), - ('Sinkhole-Events IPv6', 'event6_sinkhole', event_sinkhole), - ('Sinkhole-Events-HTTP IPv6', 'event6_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer IPv6', 'event6_sinkhole_http_referer', event_sinkhole_http_referer), - ('Malware-URL', 'malware_url', malware_url), - ('Phish-URL', 'phish_url', phish_url), - ('IPv6-Accessible-HTTP-Proxy', 'population6_http_proxy', population_http_proxy), - ('Accessible-HTTP-Proxy', 'population_http_proxy', population_http_proxy), - ('Sandbox-Connections', 'sandbox_conn', sandbox_conn), - ('Sandbox-DNS', 'sandbox_dns', sandbox_dns), - ('Sandbox-URL', 'sandbox_url', sandbox_url), - ('IPv6-Accessible-CWMP', 'scan6_cwmp', scan_cwmp), - ('IPv6-DNS-Open-Resolvers', 'scan6_dns', scan_dns), - ('IPv6-Vulnerable-Exchange', 'scan6_exchange', scan_exchange), - ('IPv6-Accessible-FTP', 'scan6_ftp', scan_ftp), - ('IPv6-Accessible-HTTP', 'scan6_http', scan_http), - ('IPv6-Open-HTTP-Proxy', 'scan6_http_proxy', scan_http_proxy), - ('IPv6-Vulnerable-HTTP', 'scan6_http_vulnerable', scan_http_vulnerable), - ('IPv6-Open-IPP', 'scan6_ipp', scan_ipp), - ('IPv6-Open-LDAP-TCP', 'scan6_ldap_tcp', scan_ldap_tcp), - ('IPv6-Open-MQTT', 'scan6_mqtt', scan_mqtt), - ('IPv6-Open-Anonymous-MQTT', 'scan6_mqtt_anon', scan_mqtt_anon), - ('IPv6-Accessible-MySQL', 'scan6_mysql', scan_mysql), - ('IPv6-NTP-Version', 'scan6_ntp', scan_ntp), - ('IPv6-NTP-Monitor', 'scan6_ntpmonitor', scan_ntpmonitor), - ('IPv6-Accessible-PostgreSQL', 'scan6_postgres', scan_postgres), - ('IPv6-Accessible-RDP', 'scan6_rdp', scan_rdp), - ('IPv6-Accessible-SLP', 'scan6_slp', scan_slp), - ('IPv6-Accessible-SMB', 'scan6_smb', scan_smb), - ('IPv6-Accessible-SMTP', 'scan6_smtp', scan_smtp), - ('IPv6-Vulnerable-SMTP', 'scan6_smtp_vulnerable', scan_smtp_vulnerable), - ('IPv6-Open-SNMP', 'scan6_snmp', scan_snmp), - ('IPv6-Accessible-SSH', 'scan6_ssh', scan_ssh), - ('IPv6-Accessible-SSL', 'scan6_ssl', scan_ssl), - ('SSL-FREAK-Vulnerable-Servers IPv6', 'scan6_ssl_freak', scan_ssl_freak), - ('SSL-POODLE-Vulnerable-Servers IPv6', 'scan6_ssl_poodle', scan_ssl_poodle), - ('IPv6-Accessible-Session-Traversal-Utilities-for-NAT', 'scan6_stun', scan_stun), - ('IPv6-Accessible-Telnet', 'scan6_telnet', scan_telnet), - ('IPv6-Accessible-VNC', 'scan6_vnc', scan_vnc), - ('Accessible-ADB', 'scan_adb', scan_adb), - ('Accessible-AFP', 'scan_afp', scan_afp), - ('Accessible-AMQP', 'scan_amqp', scan_amqp), - ('Accessible-ARD', 'scan_ard', scan_ard), - ('Open-Chargen', 'scan_chargen', scan_chargen), - ('Accessible-Cisco-Smart-Install', 'scan_cisco_smart_install', scan_cisco_smart_install), - ('Accessible-CoAP', 'scan_coap', scan_coap), - ('Accessible-CouchDB', 'scan_couchdb', scan_couchdb), - ('Accessible-CWMP', 'scan_cwmp', scan_cwmp), - ('Open-DB2-Discovery-Service', 'scan_db2', scan_db2), - ('Vulnerable-DDoS-Middlebox', 'scan_ddos_middlebox', scan_ddos_middlebox), - ('DNS-Open-Resolvers', 'scan_dns', scan_dns), - ('Accessible-Docker', 'scan_docker', scan_docker), - ('Accessible-DVR-DHCPDiscover', 'scan_dvr_dhcpdiscover', scan_dvr_dhcpdiscover), - ('Open-Elasticsearch', 'scan_elasticsearch', scan_elasticsearch), - ('Accessible-Erlang-Port-Mapper-Daemon', 'scan_epmd', scan_epmd), - ('Vulnerable-Exchange-Server', 'scan_exchange', scan_exchange), - ('Accessible-FTP', 'scan_ftp', scan_ftp), - ('Accessible-Hadoop', 'scan_hadoop', scan_hadoop), - ('Accessible-HTTP', 'scan_http', scan_http), - ('Open-HTTP-Proxy', 'scan_http_proxy', scan_http_proxy), - ('Vulnerable-HTTP', 'scan_http_vulnerable', scan_http_vulnerable), - ('Accessible-ICS', 'scan_ics', scan_ics), - ('Open-IPMI', 'scan_ipmi', scan_ipmi), - ('Open-IPP', 'scan_ipp', scan_ipp), - ('Vulnerable-ISAKMP', 'scan_isakmp', scan_isakmp), - ('Accessible-Kubernetes-API', 'scan_kubernetes', scan_kubernetes), - ('Open-LDAP-TCP', 'scan_ldap_tcp', scan_ldap_tcp), - ('Open-LDAP', 'scan_ldap_udp', scan_ldap_udp), - ('Open-mDNS', 'scan_mdns', scan_mdns), - ('Open-Memcached', 'scan_memcached', scan_memcached), - ('Open-MongoDB', 'scan_mongodb', scan_mongodb), - ('Open-MQTT', 'scan_mqtt', scan_mqtt), - ('Open-Anonymous-MQTT', 'scan_mqtt_anon', scan_mqtt_anon), - ('Open-MSSQL', 'scan_mssql', scan_mssql), - ('Accessible-MySQL', 'scan_mysql', scan_mysql), - ('Open-NATPMP', 'scan_nat_pmp', scan_nat_pmp), - ('Open-NetBIOS-Nameservice', 'scan_netbios', scan_netbios), - ('Open-Netis', 'scan_netis_router', scan_netis_router), - ('NTP-Version', 'scan_ntp', scan_ntp), - ('NTP-Monitor', 'scan_ntpmonitor', scan_ntpmonitor), - ('Open-Portmapper', 'scan_portmapper', scan_portmapper), - ('Accessible-PostgreSQL', 'scan_postgres', scan_postgres), - ('Open-QOTD', 'scan_qotd', scan_qotd), - ('Accessible-QUIC', 'scan_quic', scan_quic), - ('Accessible-Radmin', 'scan_radmin', scan_radmin), - ('Accessible-RDP', 'scan_rdp', scan_rdp), - ('Accessible-MS-RDPEUDP', 'scan_rdpeudp', scan_rdpeudp), - ('Open-Redis', 'scan_redis', scan_redis), - ('Accessible-Rsync', 'scan_rsync', scan_rsync), - ('Accessible-SIP', 'scan_sip', scan_sip), - ('Accessible-SLP', 'scan_slp', scan_slp), - ('Accessible-SMB', 'scan_smb', scan_smb), - ('Accessible-SMTP', 'scan_smtp', scan_smtp), - ('Vulnerable-SMTP', 'scan_smtp_vulnerable', scan_smtp_vulnerable), - ('Open-SNMP', 'scan_snmp', scan_snmp), - ('Accessible-SOCKS4/5-Proxy', 'scan_socks', scan_socks), - ('Open-SSDP', 'scan_ssdp', scan_ssdp), - ('Accessible-SSH', 'scan_ssh', scan_ssh), - ('Accessible-SSL', 'scan_ssl', scan_ssl), - ('SSL-FREAK-Vulnerable-Servers', 'scan_ssl_freak', scan_ssl_freak), - ('SSL-POODLE-Vulnerable-Servers IPv4', 'scan_ssl_poodle', scan_ssl_poodle), - ('Accessible-Session-Traversal-Utilities-for-NAT', 'scan_stun', scan_stun), - ('SYNful-Knock', 'scan_synfulknock', scan_synfulknock), - ('Accessible-Telnet', 'scan_telnet', scan_telnet), - ('Open-TFTP', 'scan_tftp', scan_tftp), - ('Accessible-Ubiquiti-Discovery-Service', 'scan_ubiquiti', scan_ubiquiti), - ('Accessible-VNC', 'scan_vnc', scan_vnc), - ('Accessible-WS-Discovery-Service', 'scan_ws_discovery', scan_ws_discovery), - ('Open-XDMCP', 'scan_xdmcp', scan_xdmcp), - ('Spam-URL', 'spam_url', spam_url), - ('Special', 'special', special), - ('Accessible-RDPEUDP', 'scan_rdpeudp', scan_rdpeudp), - ('Sinkhole-Events', 'event4_sinkhole', event_sinkhole), - ('Sinkhole-Events-HTTP', 'event4_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer', 'event4_sinkhole_http_referer', event_sinkhole_http_referer), -) -# END CONFGEN + try: + with open(tmp) as fh: + schema = json.load(fh) + except: + # leave tempfile behind for diagnosis + raise ValueError("Failed to validate %r" % tmp) -feedname_mapping = {feedname: function for feedname, filename, function in mapping} -filename_mapping = {filename: (feedname, function) for feedname, filename, function in mapping} + os.replace(tmp, __config.schema_file) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 70ba3b4bb..f14549141 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -22,6 +22,7 @@ """ import copy import re +import os from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue @@ -29,7 +30,13 @@ class ShadowserverParserBot(ParserBot): - """Parse all ShadowServer feeds""" + """ + Parse all ShadowServer feeds + + Parameters: + schema_file (str): Path to the report schema file + + """ recover_line = ParserBot.recover_line_csv_dict _csv_params = {'dialect': 'unix'} @@ -124,10 +131,17 @@ def parse_line(self, row, report): value = raw_value if conv_func is not None and raw_value is not None: - if len(item) == 4 and item[3]: - value = conv_func(raw_value, row) - else: - value = conv_func(raw_value) + try: + if len(item) == 4 and item[3]: + value = config.functions[conv_func](raw_value, row) + else: + value = config.functions[conv_func](raw_value) + except Exception: + """ fail early and often in this case. We want to be able to convert everything """ + self.logger.error('Could not convert shadowkey: %r in feed %r, ' + 'value: %r via conversion function %r.', + shadowkey, self.feedname, raw_value, conv_func) + raise if value is not None: event.add(intelmqkey, value) @@ -153,17 +167,17 @@ def parse_line(self, row, report): value = raw_value if conv_func is not None and raw_value is not None: - if len(item) == 4 and item[3]: - value = conv_func(raw_value, row) - else: - try: - value = conv_func(raw_value) - except Exception: - """ fail early and often in this case. We want to be able to convert everything """ - self.logger.error('Could not convert shadowkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowkey, self.feedname, raw_value, conv_func.__name__) - raise + try: + if len(item) == 4 and item[3]: + value = config.functions[conv_func](raw_value, row) + else: + value = config.functions[conv_func](raw_value) + except Exception: + """ fail early and often in this case. We want to be able to convert everything """ + self.logger.error('Could not convert shadowkey: %r in feed %r, ' + 'value: %r via conversion function %r.', + shadowkey, self.feedname, raw_value, conv_func) + raise if value is not None: if intelmqkey == 'extra.': diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test b/intelmq/bots/parsers/shadowserver/schema.json.test new file mode 100644 index 000000000..2cfb8bb1d --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/schema.json.test @@ -0,0 +1,180 @@ +{ + "test_smb" : { + "constant_fields" : { + "classification.identifier" : "test-smb", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "smb", + "protocol.transport" : "tcp" + }, + "feed_name" : "Test-Accessible-SMB", + "file_name" : "test_smb", + "optional_fields" : [ + [ + "extra.", + "smb_implant", + "convert_bool" + ], + [ + "source.reverse_dns", + "hostname" + ], + [ + "extra.", + "tag" + ], + [ + "source.asn", + "asn", + "invalidate_zero" + ], + [ + "source.geolocation.cc", + "geo" + ], + [ + "source.geolocation.region", + "region" + ], + [ + "source.geolocation.city", + "city" + ], + [ + "extra.source.naics", + "naics", + "invalidate_zero" + ], + [ + "extra.source.sic", + "sic", + "invalidate_zero" + ], + [ + "extra.", + "arch", + "validate_to_none" + ], + [ + "extra.", + "key", + "validate_to_none" + ], + [ + "extra.", + "smbv1_support", + "validate_to_none" + ], + [ + "extra.", + "smb_major_number", + "validate_to_none" + ], + [ + "extra.", + "smb_minor_number", + "validate_to_none" + ], + [ + "extra.", + "smb_revision", + "validate_to_none" + ], + [ + "extra.", + "smb_version_string", + "validate_to_none" + ] + ], + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ] + ] + }, + "test_telnet" : { + "constant_fields" : { + "classification.identifier" : "test-telnet", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "telnet" + }, + "feed_name" : "Test-Accessible-Telnet", + "file_name" : "test_telnet", + "optional_fields" : [ + [ + "protocol.transport", + "protocol" + ], + [ + "source.reverse_dns", + "hostname" + ], + [ + "extra.", + "tag", + "validate_to_none" + ], + [ + "source.asn", + "asn", + "invalidate_zero" + ], + [ + "source.geolocation.cc", + "geo" + ], + [ + "source.geolocation.region", + "region" + ], + [ + "source.geolocation.city", + "city" + ], + [ + "extra.", + "naics", + "invalidate_zero" + ], + [ "extra.", + "sic", + "invalidate_zero" + ], + [ + "extra.", + "banner", + "validate_to_none" + ] + ], + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ] + ] + } +} diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py new file mode 100644 index 000000000..040f67259 --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/update_schema.py @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import intelmq.bots.parsers.shadowserver._config as config + +if __name__ == '__main__': # pragma: no cover + exec(open(os.path.join(os.path.dirname(__file__), '../../../version.py')).read()) # defines __version__ + config.update_schema(__version__) From b917a9484776cba1cc472b598748067a4821f52d Mon Sep 17 00:00:00 2001 From: elsif2 Date: Wed, 12 Apr 2023 00:01:32 +0000 Subject: [PATCH 24/67] revised tests --- .../bots/parsers/shadowserver/test_broken.py | 12 +- .../bots/parsers/shadowserver/test_mapping.py | 8 +- .../parsers/shadowserver/test_parameters.py | 37 +++--- .../parsers/shadowserver/test_report_smb.py | 124 ++++++++++++++++++ .../shadowserver/test_report_switch.py | 16 +-- .../shadowserver/test_report_telnet.py | 87 ++++++++++++ .../shadowserver/testdata/test_smb.csv | 4 + .../testdata/test_smb.csv.license | 2 + .../shadowserver/testdata/test_telnet.csv | 3 + .../testdata/test_telnet.csv.license | 2 + 10 files changed, 260 insertions(+), 35 deletions(-) create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_report_smb.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 472dd0b90..2b803142e 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -13,12 +13,12 @@ REPORT1 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_http-test-test.csv", + "extra.file_name": "2019-01-01-test_smb-test-test.csv", } REPORT2 = {"raw": utils.base64_encode('timestamp,ip,port\n2018-08-01T00:00:00+00,127.0.0.1,80'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ftp-test-test.csv", + "extra.file_name": "2019-01-01-test_telnet-test-test.csv", } REPORT3 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", @@ -48,10 +48,10 @@ def test_broken(self): """ self.input_message = REPORT1 self.run_bot(allowed_error_count=1) - self.assertLogMatches(pattern="Detected report's file name: 'scan_http'.", + self.assertLogMatches(pattern="Detected report's file name: 'test_smb'.", levelname="DEBUG") self.assertLogMatches(pattern="Failed to parse line.") - self.assertLogMatches(pattern="ValueError: Required column 'timestamp' not found in feed 'Accessible-HTTP'. Possible change in data format or misconfiguration.") + self.assertLogMatches(pattern="ValueError: Required column 'timestamp' not found in feed 'Test-Accessible-SMB'. Possible change in data format or misconfiguration.") self.assertLogMatches(pattern=r"Sent 0 events and found 1 problem\(s\)\.", levelname="INFO") @@ -61,9 +61,9 @@ def test_half_broken(self): """ self.input_message = REPORT2 self.run_bot(allowed_warning_count=63) - self.assertLogMatches(pattern="Detected report's file name: 'scan_ftp'.", + self.assertLogMatches(pattern="Detected report's file name: 'test_telnet'.", levelname="DEBUG") - self.assertLogMatches(pattern="Optional key 'jarm' not found in feed 'Accessible-FTP'.", + self.assertLogMatches(pattern="Optional key 'banner' not found in feed 'Test-Accessible-Telnet'.", levelname="WARNING") self.assertLogMatches(pattern=r"Sent 1 events and found 0 problem\(s\)\.", levelname="INFO") diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index f58aed66e..6a2af9447 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -11,22 +11,22 @@ with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_telnet.csv')) as handle: + 'testdata/test_telnet.csv')) as handle: TELNET_FILE = handle.read() EXAMPLE_TELNET = { "raw": utils.base64_encode(TELNET_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_telnet.csv", + "extra.file_name": "2019-01-01-test_telnet.csv", } with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_vnc.csv')) as handle: + 'testdata/test_smb.csv')) as handle: TELNET_FILE = handle.read() EXAMPLE_VNC = { "raw": utils.base64_encode(TELNET_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_vnc.csv", + "extra.file_name": "2019-01-01-test_smb.csv", } diff --git a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py index a5ea81f19..677cd0319 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py @@ -12,38 +12,41 @@ from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dns.csv')) as handle: + 'testdata/test_smb.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() EXAMPLE_REPORT = {"raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_dns-test-test.csv", + "extra.file_name": "2019-01-01-test_smb-test-test.csv", 'feed.name': 'report feedname', } EVENTS = [{ '__type': 'Event', 'feed.name': 'report feedname', - "classification.identifier": "dns-open-resolver", + "classification.identifier": 'test-smb', "classification.taxonomy": "vulnerable", "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.66", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", + "extra.smb_implant": False, + "extra.smb_major_number": '2', + "extra.smb_minor_number": '1', + "extra.smb_version_string": 'SMB 2.1', + "extra.smbv1_support": 'N', + "extra.tag": "smb", + "protocol.application": "smb", + "protocol.transport": "tcp", 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.179", - "source.port": 53, - "source.reverse_dns": "198-51-100-189.example.net", + "source.asn": 64512, + "source.geolocation.cc": "ZZ", + "source.geolocation.city": "City", + "source.geolocation.region": "Region", + "source.ip": "192.168.0.1", + "source.port": 445, + "source.reverse_dns": "node01.example.com", "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:34+00:00" + "time.source": "2010-02-10T00:00:00+00:00" }, ] @@ -70,7 +73,7 @@ def test_overwrite_feed_name(self): self.run_bot(prepare=False) for i, EVENT in enumerate(EVENTS): event = EVENT.copy() - event['feed.name'] = 'DNS-Open-Resolvers' + event['feed.name'] = 'Test-Accessible-SMB' self.assertMessageEqual(i, event) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py new file mode 100644 index 000000000..c7eefdf0a --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import unittest + +import intelmq.lib.test as test +import intelmq.lib.utils as utils +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot + +with open(os.path.join(os.path.dirname(__file__), + 'testdata/test_smb.csv')) as handle: + EXAMPLE_FILE = handle.read() +EXAMPLE_LINES = EXAMPLE_FILE.splitlines() + +EXAMPLE_REPORT = {'feed.name': 'Test-Accessible-SMB', + "raw": utils.base64_encode(EXAMPLE_FILE), + "__type": "Report", + "time.observation": "2015-01-01T00:00:00+00:00", + "extra.file_name": "2019-01-01-test_smb-test-geo.csv", + } +EVENTS = [ +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.1', + 'source.port' : 445, + 'source.reverse_dns' : 'node01.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:00+00:00' +}, + +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.2', + 'source.port' : 445, + 'source.reverse_dns' : 'node02.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:01+00:00' +}, + +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.3', + 'source.port' : 445, + 'source.reverse_dns' : 'node03.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:02+00:00' +} + ] + + +class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for a ShadowserverParserBot. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.default_input_message = EXAMPLE_REPORT + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() + for i, EVENT in enumerate(EVENTS): + self.assertMessageEqual(i, EVENT) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index 0a34a69f0..570d612fb 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -12,24 +12,24 @@ from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ftp.csv')) as handle: + 'testdata/test_smb.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] -FIRST_REPORT = {'feed.name': 'Accessible FTP', +FIRST_REPORT = {'feed.name': 'Test-Accessible-SMB', "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), "__type": "Report", "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ftp-test-test.csv", + "extra.file_name": "2019-03-25-test_smb-test-test.csv", } -with open(os.path.join(os.path.dirname(__file__), 'testdata/blocklist.csv')) as handle: +with open(os.path.join(os.path.dirname(__file__), 'testdata/test_telnet.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] SECOND_REPORT = { - 'feed.name': 'Blocklist', + 'feed.name': 'Test-Accessible-Telnet', "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-blocklist-test-geo.csv", + "extra.file_name": "2019-01-01-test_telnet-test-geo.csv", } @@ -48,9 +48,9 @@ def test_event(self): """ Test if the parser correctly detects and handles different report types. """ self.input_message = [FIRST_REPORT, SECOND_REPORT] self.run_bot(iterations=2) - self.assertLogMatches("Detected report's file name: 'scan_ftp'", + self.assertLogMatches("Detected report's file name: 'test_smb'", levelname='DEBUG') - self.assertLogMatches("Detected report's file name: 'blocklist'", + self.assertLogMatches("Detected report's file name: 'test_telnet'", levelname='DEBUG') diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py new file mode 100644 index 000000000..6d539ac4a --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import unittest + +import intelmq.lib.test as test +import intelmq.lib.utils as utils +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot + +with open(os.path.join(os.path.dirname(__file__), + 'testdata/test_telnet.csv')) as handle: + EXAMPLE_FILE = handle.read() +EXAMPLE_LINES = EXAMPLE_FILE.splitlines() + +EXAMPLE_REPORT = {'feed.name': 'Test-Accessible-Telnet', + "raw": utils.base64_encode(EXAMPLE_FILE), + "__type": "Report", + "time.observation": "2015-01-01T00:00:00+00:00", + "extra.file_name": "2019-01-01-test_telnet-test-geo.csv", + } +EVENTS = [{'__type': 'Event', + 'feed.name': 'Test-Accessible-Telnet', + "classification.identifier": "test-telnet", + "classification.taxonomy": "vulnerable", + "classification.type": "vulnerable-system", + "extra.banner": "|MikroTik v6.5|Login:", + "extra.tag": "telnet-alt", + "protocol.application": "telnet", + "protocol.transport": "tcp", + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], + EXAMPLE_LINES[1]])), + "source.asn": 20255, + "source.geolocation.cc": "AA", + "source.geolocation.city": "LOCATION", + "source.geolocation.region": "LOCATION", + "source.ip": "198.123.245.145", + "source.port": 5678, + "source.reverse_dns": "example.local", + "time.observation": "2015-01-01T00:00:00+00:00", + "time.source": "2019-09-04T12:27:34+00:00" + }, + {'__type': 'Event', + 'feed.name': 'Test-Accessible-Telnet', + "classification.identifier": "test-telnet", + "classification.taxonomy": "vulnerable", + "classification.type": "vulnerable-system", + "extra.banner": "|MikroTik v6.45.3 (stable)|Login:", + "extra.tag": "telnet-alt", + "protocol.application": "telnet", + "protocol.transport": "tcp", + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], + EXAMPLE_LINES[2]])), + "source.asn": 20255, + "source.geolocation.cc": "AA", + "source.geolocation.city": "LOCATION", + "source.geolocation.region": "LOCATION", + "source.ip": "198.123.245.145", + "source.port": 5678, + "source.reverse_dns": "example.local", + "time.observation": "2015-01-01T00:00:00+00:00", + "time.source": "2019-09-04T12:27:40+00:00" + }] + + +class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for a ShadowserverParserBot. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.default_input_message = EXAMPLE_REPORT + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() + for i, EVENT in enumerate(EVENTS): + self.assertMessageEqual(i, EVENT) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv new file mode 100644 index 000000000..fc7fe2fff --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv @@ -0,0 +1,4 @@ +"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","smb_implant","arch","key","smbv1_support","smb_major_number","smb_minor_number","smb_revision","smb_version_string" +"2010-02-10 00:00:00",192.168.0.1,445,node01.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" +"2010-02-10 00:00:01",192.168.0.2,445,node02.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" +"2010-02-10 00:00:02",192.168.0.3,445,node03.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license new file mode 100644 index 000000000..f512a890e --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2022 The Shadowserver Foundation +SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv new file mode 100644 index 000000000..3309e9a3d --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv @@ -0,0 +1,3 @@ +"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" +"2019-09-04 12:27:34","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.5|Login:" +"2019-09-04 12:27:40","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.45.3 (stable)|Login:" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license new file mode 100644 index 000000000..942a94035 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +SPDX-License-Identifier: AGPL-3.0-or-later From eafa15bc8ea8ac214db9cf349d971dbd450aa149 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 8 May 2023 15:05:12 +0000 Subject: [PATCH 25/67] Updated to reset report type on reload #2361 --- intelmq/bots/parsers/shadowserver/README.md | 2 +- intelmq/bots/parsers/shadowserver/_config.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index 297930861..bb6216b9a 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -11,6 +11,6 @@ The report configuration is now stored in a _schema.json_ file downloaded from h For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index a7b80b7a6..29382d278 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -272,15 +272,14 @@ def scan_exchange_identifier(field): def reload (): """ reload the configuration if it has changed """ mtime = 0.0 + schema_file = __config.schema_file - if (os.path.isfile(__config.schema_file)): + if os.path.isfile(__config.schema_file): mtime = os.path.getmtime(__config.schema_file) if __config.schema_mtime == mtime: return - schema_file = __config.schema_file else: # load a test schema if one has not been downloaded yet - schema_file = __config.schema_file schema_file += '.test' __config.feedname_mapping.clear() From b2753cb9fe6ae15eb569b6d718f54333e476c62d Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 23 May 2023 01:12:47 +0000 Subject: [PATCH 26/67] Added schema download on startup and additional logging --- intelmq/bots/parsers/shadowserver/_config.py | 33 +++++++++++++------ intelmq/bots/parsers/shadowserver/parser.py | 1 + .../parsers/shadowserver/update_schema.py | 3 +- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 29382d278..f766be322 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -106,6 +106,8 @@ def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, A reload() return __config.filename_mapping.get(given_filename, None) +def set_logger(logger): + __config.logger = logger def add_UTC_to_timestamp(value: str) -> str: return value + ' UTC' @@ -272,29 +274,38 @@ def scan_exchange_identifier(field): def reload (): """ reload the configuration if it has changed """ mtime = 0.0 - schema_file = __config.schema_file if os.path.isfile(__config.schema_file): mtime = os.path.getmtime(__config.schema_file) if __config.schema_mtime == mtime: return else: - # load a test schema if one has not been downloaded yet - schema_file += '.test' + __config.logger.info("The schema file does not exist.") + + if __config.schema_mtime == 0.0 and mtime == 0.0 and not os.environ.get('INTELMQ_SKIP_INTERNET'): + __config.logger.info("Attempting to download schema.") + update_schema() __config.feedname_mapping.clear() __config.filename_mapping.clear() - with open(schema_file) as fh: - schema = json.load(fh) - for report in schema: - __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) - __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + for schema_file in [ __config.schema_file, ".".join([__config.schema_file, 'test']) ]: + if os.path.isfile(schema_file): + with open(schema_file) as fh: + schema = json.load(fh) + for report in schema: + if report == "_meta": + __config.logger.info("Loading schema %s." % schema[report]['date_created']) + for msg in schema[report]['change_log']: + __config.logger.info(msg) + else: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime -def update_schema (version): +def update_schema (): """ download the latest configuration """ (th, tmp) = tempfile.mkstemp() - url = 'https://interchange.shadowserver.org/intelmq/'+version + url = 'https://interchange.shadowserver.org/intelmq/v1' try: urllib.request.urlretrieve(url, tmp) except: @@ -307,4 +318,6 @@ def update_schema (version): # leave tempfile behind for diagnosis raise ValueError("Failed to validate %r" % tmp) + if os.path.exists(__config.schema_file): + os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) os.replace(tmp, __config.schema_file) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index f14549141..2f20262bf 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -47,6 +47,7 @@ class ShadowserverParserBot(ParserBot): overwrite = False def init(self): + config.set_logger(self.logger) if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py index 040f67259..a7975147e 100644 --- a/intelmq/bots/parsers/shadowserver/update_schema.py +++ b/intelmq/bots/parsers/shadowserver/update_schema.py @@ -8,5 +8,4 @@ import intelmq.bots.parsers.shadowserver._config as config if __name__ == '__main__': # pragma: no cover - exec(open(os.path.join(os.path.dirname(__file__), '../../../version.py')).read()) # defines __version__ - config.update_schema(__version__) + config.update_schema() From fd0a8fd44c39a5dba2684846b9c03262ccf9307a Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 23 May 2023 23:32:53 +0000 Subject: [PATCH 27/67] Added version support to the schema update function. --- intelmq/bots/parsers/shadowserver/README.md | 6 ++-- intelmq/bots/parsers/shadowserver/_config.py | 32 +++++++++++++++++--- intelmq/bots/parsers/shadowserver/parser.py | 4 +++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index bb6216b9a..c757020e9 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,10 +7,10 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/_version_. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1. -For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. +For environments that have internet connectivity the `update_schema.py` script can be called from a cron job to obtain the latest revision. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index f766be322..bb67db525 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -97,6 +97,11 @@ class __Container: __config.feedname_mapping = {} __config.filename_mapping = {} +def set_logger(logger): + """ Sets the logger instance. """ + __config.logger = logger + + def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: reload() return __config.feedname_mapping.get(given_feedname, None) @@ -106,8 +111,6 @@ def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, A reload() return __config.filename_mapping.get(given_filename, None) -def set_logger(logger): - __config.logger = logger def add_UTC_to_timestamp(value: str) -> str: return value + ' UTC' @@ -304,20 +307,39 @@ def reload (): def update_schema (): """ download the latest configuration """ - (th, tmp) = tempfile.mkstemp() + if os.environ.get('INTELMQ_SKIP_INTERNET'): + return None + + (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) url = 'https://interchange.shadowserver.org/intelmq/v1' try: urllib.request.urlretrieve(url, tmp) except: raise ValueError("Failed to download %r" % url) + new_version = '' + old_version = '' + try: with open(tmp) as fh: schema = json.load(fh) + new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis raise ValueError("Failed to validate %r" % tmp) if os.path.exists(__config.schema_file): - os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) - os.replace(tmp, __config.schema_file) + old_version = '' + try: + with open(__config.schema_file) as fh: + schema = json.load(fh) + old_version = schema['_meta']['date_created'] + if new_version != old_version: + os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) + except: + pass + + if new_version != old_version: + os.replace(tmp, __config.schema_file) + else: + os.unlink(tmp) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 2f20262bf..71489e2ec 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -48,6 +48,10 @@ class ShadowserverParserBot(ParserBot): def init(self): config.set_logger(self.logger) + try: + config.update_schema() + except Exception as e: + logger.warning(f"Schema update failed: {e}.") if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: From 357aad523c5a875121a38f26164cfff9fbacd24b Mon Sep 17 00:00:00 2001 From: elsif2 Date: Sun, 28 May 2023 23:13:54 +0000 Subject: [PATCH 28/67] Documentation and style updates. --- CHANGELOG.md | 6 + .../shadowserver/collector_reports_api.py | 2 +- intelmq/bots/parsers/shadowserver/README.md | 39 ++++- intelmq/bots/parsers/shadowserver/_config.py | 52 +++--- intelmq/bots/parsers/shadowserver/parser.py | 2 +- .../bots/parsers/shadowserver/test_broken.py | 4 +- .../bots/parsers/shadowserver/test_mapping.py | 1 - .../parsers/shadowserver/test_report_smb.py | 151 +++++++++--------- .../shadowserver/test_report_switch.py | 10 +- .../shadowserver/test_report_telnet.py | 4 +- 10 files changed, 154 insertions(+), 117 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72d950193..ea36275bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,15 +118,21 @@ CHANGELOG ### Bots #### Collectors +<<<<<<< HEAD - `intelmq.bots.collector.rt`: - restrict `python-rt` to be below version 3.0 due to introduced breaking changes, - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). +======= +- `intelmq.bots.collectors.shadowserver.collector_reports_api`: + - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) +>>>>>>> Documentation and style updates. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: - Reset detected `feedname` at shutdown to re-detect the feedname on reloads (PR#2361 by @elsif2, fixes #2360). + - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. Please see intelmq/bots/parsers/shadowserver/README.md for details. (PR#2372) - `intelmq.bots.parsers.shadowserver._config`: - Added 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Service' report. (PR#2338) - Removed unused `p0f_genre` and `p0f_detail` from the 'DNS-Open-Resolvers' report. (PR#2338) diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index dc8bd6b42..5e7117bd2 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv'). The 'json' option is not longer supported. + file_format (str): File format to download ('csv'). The 'json' option is no longer supported. """ country = None diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index c757020e9..ae38dcb8c 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,10 +7,45 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. -For environments that have internet connectivity the `update_schema.py` script can be called from a cron job to obtain the latest revision. +For environments that have internet connectivity the `update_schema.py` script should be called from a cron job to obtain the latest revision. +The parser will attempt to download a schema update on startup unless INTELMQ_SKIP_INTERNET is set. For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. + + +## Sample configuration: + +``` +shadowserver-collector: + description: Our bot responsible for getting reports from Shadowserver + enabled: true + group: Collector + module: intelmq.bots.collectors.shadowserver.collector_reports_api + name: Shadowserver_Collector + parameters: + destination_queues: + _default: [shadowserver-parser-queue] + file_format: csv + api_key: "$API_KEY_received_from_the_shadowserver_foundation" + secret: "$SECRET_received_from_the_shadowserver_foundation" + run_mode: continuous +``` + +``` +shadowserver-parser: + bot_id: shadowserver-parser + name: Shadowserver Parser + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.shadowserver.parser + parameters: + destination_queues: + _default: [file-output-queue] + run_mode: continuous +``` + diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index bb67db525..5219fdb34 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -88,15 +88,18 @@ import intelmq.lib.harmonization as harmonization + class __Container: pass + __config = __Container() __config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') __config.schema_mtime = 0.0 __config.feedname_mapping = {} __config.filename_mapping = {} + def set_logger(logger): """ Sets the logger instance. """ __config.logger = logger @@ -254,27 +257,28 @@ def scan_exchange_identifier(field): return 'exchange-server-webshell' return 'vulnerable-exchange-server' + functions = { - 'add_UTC_to_timestamp': add_UTC_to_timestamp, - 'convert_bool': convert_bool, - 'validate_to_none': validate_to_none, - 'convert_int': convert_int, - 'convert_float': convert_float, - 'convert_http_host_and_url': convert_http_host_and_url, - 'invalidate_zero': invalidate_zero, - 'validate_ip': validate_ip, - 'validate_network': validate_network, - 'validate_fqdn': validate_fqdn, - 'convert_date': convert_date, - 'convert_date_utc': convert_date_utc, - 'force_base64': force_base64, - 'scan_exchange_taxonomy': scan_exchange_taxonomy, - 'scan_exchange_type': scan_exchange_type, - 'scan_exchange_identifier': scan_exchange_identifier, - } - - -def reload (): + 'add_UTC_to_timestamp': add_UTC_to_timestamp, + 'convert_bool': convert_bool, + 'validate_to_none': validate_to_none, + 'convert_int': convert_int, + 'convert_float': convert_float, + 'convert_http_host_and_url': convert_http_host_and_url, + 'invalidate_zero': invalidate_zero, + 'validate_ip': validate_ip, + 'validate_network': validate_network, + 'validate_fqdn': validate_fqdn, + 'convert_date': convert_date, + 'convert_date_utc': convert_date_utc, + 'force_base64': force_base64, + 'scan_exchange_taxonomy': scan_exchange_taxonomy, + 'scan_exchange_type': scan_exchange_type, + 'scan_exchange_identifier': scan_exchange_identifier, +} + + +def reload(): """ reload the configuration if it has changed """ mtime = 0.0 @@ -291,7 +295,7 @@ def reload (): __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [ __config.schema_file, ".".join([__config.schema_file, 'test']) ]: + for schema_file in [__config.schema_file, ".".join([__config.schema_file, 'test'])]: if os.path.isfile(schema_file): with open(schema_file) as fh: schema = json.load(fh) @@ -305,13 +309,14 @@ def reload (): __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime -def update_schema (): + +def update_schema(): """ download the latest configuration """ if os.environ.get('INTELMQ_SKIP_INTERNET'): return None (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) - url = 'https://interchange.shadowserver.org/intelmq/v1' + url = 'https://interchange.shadowserver.org/intelmq/v1/schema' try: urllib.request.urlretrieve(url, tmp) except: @@ -329,7 +334,6 @@ def update_schema (): raise ValueError("Failed to validate %r" % tmp) if os.path.exists(__config.schema_file): - old_version = '' try: with open(__config.schema_file) as fh: schema = json.load(fh) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 71489e2ec..668a81534 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -51,7 +51,7 @@ def init(self): try: config.update_schema() except Exception as e: - logger.warning(f"Schema update failed: {e}.") + self.logger.warning("Schema update failed: %s." % e) if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 2b803142e..3797f03cd 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -24,12 +24,12 @@ "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2019-01-01-some_string-test-test.csv", -} + } REPORT4 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2020.wrong-filename.csv", -} + } class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index 6a2af9447..d296dfdc2 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -52,6 +52,5 @@ def test_changed_feed(self): self.run_bot(iterations=2) - if __name__ == '__main__': # pragma: no cover unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py index c7eefdf0a..93d592d15 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -22,85 +22,78 @@ "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2019-01-01-test_smb-test-geo.csv", } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] +EVENTS = [{'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.1', + 'source.port': 445, + 'source.reverse_dns': 'node01.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:00+00:00' + }, + {'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.2', + 'source.port': 445, + 'source.reverse_dns': 'node02.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:01+00:00' + }, + {'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.3', + 'source.port': 445, + 'source.reverse_dns': 'node03.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:02+00:00' + }] class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index 570d612fb..a9be8a0a1 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -16,11 +16,11 @@ EXAMPLE_LINES = handle.read().splitlines()[:2] FIRST_REPORT = {'feed.name': 'Test-Accessible-SMB', - "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-test_smb-test-test.csv", - } + "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), + "__type": "Report", + "time.observation": "2019-03-25T00:00:00+00:00", + "extra.file_name": "2019-03-25-test_smb-test-test.csv", + } with open(os.path.join(os.path.dirname(__file__), 'testdata/test_telnet.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py index 6d539ac4a..df9cf25dc 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -42,7 +42,7 @@ "source.reverse_dns": "example.local", "time.observation": "2015-01-01T00:00:00+00:00", "time.source": "2019-09-04T12:27:34+00:00" - }, + }, {'__type': 'Event', 'feed.name': 'Test-Accessible-Telnet', "classification.identifier": "test-telnet", @@ -63,7 +63,7 @@ "source.reverse_dns": "example.local", "time.observation": "2015-01-01T00:00:00+00:00", "time.source": "2019-09-04T12:27:40+00:00" - }] + }] class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): From 37c67459f7ea791c31cd36b74456be27d079f9fe Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 30 May 2023 16:05:26 +0000 Subject: [PATCH 29/67] Added schema.json.test.license. --- intelmq/bots/parsers/shadowserver/schema.json.test.license | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 intelmq/bots/parsers/shadowserver/schema.json.test.license diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test.license b/intelmq/bots/parsers/shadowserver/schema.json.test.license new file mode 100644 index 000000000..9f58c89ef --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/schema.json.test.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +SPDX-License-Identifier: AGPL-3.0-or-later From ee8ce873977d3de18ebddddaac2c38c3ed5ca257 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 27 Jul 2023 20:19:25 +0000 Subject: [PATCH 30/67] Updates in response to feedback. --- .../shadowserver/collector_reports_api.py | 9 +++- intelmq/bots/parsers/shadowserver/README.md | 21 ++++++-- intelmq/bots/parsers/shadowserver/_config.py | 53 +++++++++++++------ intelmq/bots/parsers/shadowserver/parser.py | 45 +++++++++++++--- .../parsers/shadowserver/update_schema.py | 11 ---- .../shadowserver/test_download_schema.py | 28 ++++++++++ 6 files changed, 130 insertions(+), 37 deletions(-) delete mode 100644 intelmq/bots/parsers/shadowserver/update_schema.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_download_schema.py diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 5e7117bd2..05bffa898 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -68,12 +68,19 @@ def init(self): if self.file_format is not None: if not (self.file_format == 'csv'): - raise ValueError('Invalid file_format') + raise ValueError("Invalid file_format '%s'. Must be 'csv'." % self.file_format) else: self.file_format = 'csv' self.preamble = f'{{ "apikey": "{self.api_key}" ' + def check(parameters: dict): + for key in parameters: + if key == 'file_format' and parameters[key] != 'csv': + return [["error", "Invalid file_format '%s'. Must be 'csv'." % parameters[key]]] + elif key == 'country': + return [["warning", "Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0."]] + def _headers(self, data): return {'HMAC2': hmac.new(self.secret.encode(), data.encode('utf-8'), digestmod=hashlib.sha256).hexdigest()} diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index ae38dcb8c..cd750d00b 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,16 +7,28 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. +The report configuration is now stored in a _shadowserver-schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. -For environments that have internet connectivity the `update_schema.py` script should be called from a cron job to obtain the latest revision. -The parser will attempt to download a schema update on startup unless INTELMQ_SKIP_INTERNET is set. +The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. +Schema downloads can also be scheduled as a cron job: + +``` +02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema +``` + +For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. The parser will automatically reload the configuration when the file changes. +## Schema contract + +Once set the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static. + +Once set report fields will not be deleted. + + ## Sample configuration: ``` @@ -46,6 +58,7 @@ shadowserver-parser: parameters: destination_queues: _default: [file-output-queue] + auto_update: true run_mode: continuous ``` diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 5219fdb34..afe3a6b11 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -82,11 +82,12 @@ import base64 import binascii import json -import urllib.request import tempfile from typing import Optional, Dict, Tuple, Any import intelmq.lib.harmonization as harmonization +from intelmq.lib.utils import create_request_session +from intelmq import VAR_STATE_PATH class __Container: @@ -94,8 +95,10 @@ class __Container: __config = __Container() -__config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') +__config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') +__config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') __config.schema_mtime = 0.0 +__config.auto_update = False __config.feedname_mapping = {} __config.filename_mapping = {} @@ -105,13 +108,16 @@ def set_logger(logger): __config.logger = logger +def enable_auto_update(enable): + """ Enable automatic schema update. """ + __config.auto_update = enable + + def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: - reload() return __config.feedname_mapping.get(given_feedname, None) def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, Any]]]: - reload() return __config.filename_mapping.get(given_filename, None) @@ -289,19 +295,18 @@ def reload(): else: __config.logger.info("The schema file does not exist.") - if __config.schema_mtime == 0.0 and mtime == 0.0 and not os.environ.get('INTELMQ_SKIP_INTERNET'): - __config.logger.info("Attempting to download schema.") + if __config.schema_mtime == 0.0 and mtime == 0.0 and __config.auto_update: update_schema() __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [__config.schema_file, ".".join([__config.schema_file, 'test'])]: + for schema_file in [__config.schema_file, __config.schema_base]: if os.path.isfile(schema_file): with open(schema_file) as fh: schema = json.load(fh) for report in schema: if report == "_meta": - __config.logger.info("Loading schema %s." % schema[report]['date_created']) + __config.logger.info("Loading schema %r." % schema[report]['date_created']) for msg in schema[report]['change_log']: __config.logger.info(msg) else: @@ -313,37 +318,55 @@ def reload(): def update_schema(): """ download the latest configuration """ if os.environ.get('INTELMQ_SKIP_INTERNET'): - return None + return False - (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) + # download the schema to a temp file + (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) url = 'https://interchange.shadowserver.org/intelmq/v1/schema' + __config.logger.info("Attempting to download schema from %r" % url) + __config.logger.debug("Using temp file %r for the download." % tmp) try: - urllib.request.urlretrieve(url, tmp) + with create_request_session() as session: + with session.get(url, stream=True) as r: + r.raise_for_status() + with open(tmp, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) except: - raise ValueError("Failed to download %r" % url) + __config.logger.error("Failed to download %r" % url) + return False + __config.logger.info("Download successful.") new_version = '' old_version = '' try: + # validate the downloaded file with open(tmp) as fh: schema = json.load(fh) new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis - raise ValueError("Failed to validate %r" % tmp) + __config.logger.error("Failed to validate %r" % tmp) + return False if os.path.exists(__config.schema_file): + # compare the new version against the old; rename the existing file try: with open(__config.schema_file) as fh: schema = json.load(fh) old_version = schema['_meta']['date_created'] if new_version != old_version: os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) - except: - pass + except Exception as e: + __config.logger.error("Unable to replace schema file: %s" % str(e)) + return False if new_version != old_version: os.replace(tmp, __config.schema_file) + __config.logger.info("New schema version is %r." % new_version) + return True else: os.unlink(tmp) + + return False diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 668a81534..2e383a004 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -26,6 +26,8 @@ from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue +from intelmq.bin.intelmqctl import IntelMQController +import intelmq.lib.utils as utils import intelmq.bots.parsers.shadowserver._config as config @@ -34,8 +36,7 @@ class ShadowserverParserBot(ParserBot): Parse all ShadowServer feeds Parameters: - schema_file (str): Path to the report schema file - + auto_update (boolean): Enable automatic schema download """ recover_line = ParserBot.recover_line_csv_dict @@ -45,13 +46,15 @@ class ShadowserverParserBot(ParserBot): feedname = None _mode = None overwrite = False + auto_update = False def init(self): config.set_logger(self.logger) - try: - config.update_schema() - except Exception as e: - self.logger.warning("Schema update failed: %s." % e) + if self.auto_update: + config.enable_auto_update(True) + self.logger.debug("Feature 'auto_update' is enabled.") + config.reload() + if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: @@ -228,5 +231,35 @@ def parse_line(self, row, report): def shutdown(self): self.feedname = None + @classmethod + def _create_argparser(cls): + argparser = super()._create_argparser() + argparser.add_argument("--update-schema", action='store_true', help='downloads latest report schema') + argparser.add_argument("--verbose", action='store_true', help='be verbose') + return argparser + + @classmethod + def run(cls, parsed_args=None): + if not parsed_args: + parsed_args = cls._create_argparser().parse_args() + if parsed_args.update_schema: + logger = utils.log(__name__, log_path=None) + if parsed_args.verbose: + logger.setLevel('INFO') + else: + logger.setLevel('ERROR') + config.set_logger(logger) + if config.update_schema(): + runtime_conf = utils.get_bots_settings() + try: + ctl = IntelMQController() + for bot in runtime_conf: + if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('auto_update', True): + ctl.bot_reload(bot) + except Exception as e: + logger.error("Failed to signal bot: %r" % str(e)) + else: + super().run(parsed_args=parsed_args) + BOT = ShadowserverParserBot diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py deleted file mode 100644 index a7975147e..000000000 --- a/intelmq/bots/parsers/shadowserver/update_schema.py +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import intelmq.bots.parsers.shadowserver._config as config - -if __name__ == '__main__': # pragma: no cover - config.update_schema() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py new file mode 100644 index 000000000..e68587682 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- +""" +Created on Thu Jul 27 19:44:44 2023 + +""" + +import unittest +import os +import logging +from intelmq import VAR_STATE_PATH +import intelmq.bots.parsers.shadowserver._config as config +import intelmq.lib.utils as utils +import intelmq.lib.test as test + +@test.skip_internet() +class TestShadowserverSchemaDownload(unittest.TestCase): + + def test_download(self): + schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + config.set_logger(utils.log('test-bot', log_path=None)) + if os.path.exists(schema_file): + os.unlink(schema_file) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From 4a73f0b9af80d126b1e19de43097700e24ad7f63 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 28 Jul 2023 14:17:41 +0000 Subject: [PATCH 31/67] Removed file_format parameter --- .../shadowserver/collector_reports_api.py | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 05bffa898..66169d96f 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,6 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv'). The 'json' option is no longer supported. """ country = None @@ -42,7 +41,6 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): secret = None types = None reports = None - file_format = None rate_limit: int = 86400 redis_cache_db: int = 12 redis_cache_host: str = "127.0.0.1" # TODO: type could be ipadress @@ -66,18 +64,12 @@ def init(self): self.logger.warn("Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0.") self._report_list.append(self.country) - if self.file_format is not None: - if not (self.file_format == 'csv'): - raise ValueError("Invalid file_format '%s'. Must be 'csv'." % self.file_format) - else: - self.file_format = 'csv' - self.preamble = f'{{ "apikey": "{self.api_key}" ' def check(parameters: dict): for key in parameters: - if key == 'file_format' and parameters[key] != 'csv': - return [["error", "Invalid file_format '%s'. Must be 'csv'." % parameters[key]]] + if key == 'file_format': + return [["error", "The file_format parameter is no longer supported. All reports are CSV."]] elif key == 'country': return [["warning", "Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0."]] @@ -129,11 +121,7 @@ def _report_download(self, reportid: str): data = self.preamble data += f',"id": "{reportid}"}}' self.logger.debug('Downloading report with data: %s.', data) - - if (self.file_format == 'json'): - response = self.http_session().post(APIROOT + 'reports/download', data=data, headers=self._headers(data)) - else: - response = self.http_session().get(DLROOT + reportid) + response = self.http_session().get(DLROOT + reportid) response.raise_for_status() return response.text @@ -150,7 +138,7 @@ def process(self): for item in reportslist: filename = item['file'] - filename_fixed = FILENAME_PATTERN.sub('.' + self.file_format, filename, count=1) + filename_fixed = FILENAME_PATTERN.sub('.csv', filename, count=1) if self.cache_get(filename): self.logger.debug('Processed file %r (fixed: %r) already.', filename, filename_fixed) continue From e413fb50513f900a146dbd4c1c45667ae8e04541 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 24 Aug 2023 16:04:21 +0000 Subject: [PATCH 32/67] Minor changes based on feedback 2023-08-24 --- CHANGELOG.md | 2 - intelmq/bots/parsers/shadowserver/README.md | 2 + intelmq/bots/parsers/shadowserver/_config.py | 49 ++++++++++--------- intelmq/bots/parsers/shadowserver/parser.py | 6 ++- .../bots/parsers/shadowserver/test_broken.py | 5 ++ .../bots/parsers/shadowserver/test_mapping.py | 1 + .../parsers/shadowserver/test_parameters.py | 3 +- .../parsers/shadowserver/test_report_smb.py | 1 + .../shadowserver/test_report_switch.py | 1 + .../shadowserver/test_report_telnet.py | 1 + 10 files changed, 45 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea36275bc..8cee9e520 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,10 +124,8 @@ CHANGELOG - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). -======= - `intelmq.bots.collectors.shadowserver.collector_reports_api`: - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) ->>>>>>> Documentation and style updates. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index cd750d00b..4969acb6d 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -28,6 +28,8 @@ Once set the `classification.identifier`, `classification.taxonomy`, and `classi Once set report fields will not be deleted. +The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. + ## Sample configuration: diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index afe3a6b11..4bfadb9d9 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -95,8 +95,10 @@ class __Container: __config = __Container() +__config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema' __config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') __config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') +__config.schema_active = __config.schema_file __config.schema_mtime = 0.0 __config.auto_update = False __config.feedname_mapping = {} @@ -108,6 +110,13 @@ def set_logger(logger): __config.logger = logger +def enable_test_mode(enable): + """ Set which schema to load. """ + if enable: + __config.schema_active = __config.schema_base + else: + __config.schema_active = __config.schema_file + def enable_auto_update(enable): """ Enable automatic schema update. """ __config.auto_update = enable @@ -300,40 +309,36 @@ def reload(): __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [__config.schema_file, __config.schema_base]: - if os.path.isfile(schema_file): - with open(schema_file) as fh: - schema = json.load(fh) - for report in schema: - if report == "_meta": - __config.logger.info("Loading schema %r." % schema[report]['date_created']) - for msg in schema[report]['change_log']: - __config.logger.info(msg) - else: - __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) - __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + if os.path.isfile(__config.schema_active): + with open(__config.schema_active) as fh: + schema = json.load(fh) + for report in schema: + if report == "_meta": + __config.logger.info("Loading schema %r.", schema[report]['date_created']) + for msg in schema[report]['change_log']: + __config.logger.info(msg) + else: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime def update_schema(): """ download the latest configuration """ - if os.environ.get('INTELMQ_SKIP_INTERNET'): - return False # download the schema to a temp file (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) - url = 'https://interchange.shadowserver.org/intelmq/v1/schema' - __config.logger.info("Attempting to download schema from %r" % url) - __config.logger.debug("Using temp file %r for the download." % tmp) + __config.logger.info("Attempting to download schema from %r", __config.schema_url) + __config.logger.debug("Using temp file %r for the download.", tmp) try: with create_request_session() as session: - with session.get(url, stream=True) as r: + with session.get(__config.schema_url, stream=True) as r: r.raise_for_status() with open(tmp, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) except: - __config.logger.error("Failed to download %r" % url) + __config.logger.error("Failed to download %r", __config.schema_url) return False __config.logger.info("Download successful.") @@ -347,7 +352,7 @@ def update_schema(): new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis - __config.logger.error("Failed to validate %r" % tmp) + __config.logger.error("Failed to validate %r", tmp) return False if os.path.exists(__config.schema_file): @@ -359,12 +364,12 @@ def update_schema(): if new_version != old_version: os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) except Exception as e: - __config.logger.error("Unable to replace schema file: %s" % str(e)) + __config.logger.error("Unable to replace schema file: %s", str(e)) return False if new_version != old_version: os.replace(tmp, __config.schema_file) - __config.logger.info("New schema version is %r." % new_version) + __config.logger.info("New schema version is %r.", new_version) return True else: os.unlink(tmp) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 2e383a004..fd9fa6b2c 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -37,6 +37,7 @@ class ShadowserverParserBot(ParserBot): Parameters: auto_update (boolean): Enable automatic schema download + test_mode (boolean): Use test schema """ recover_line = ParserBot.recover_line_csv_dict @@ -47,9 +48,12 @@ class ShadowserverParserBot(ParserBot): _mode = None overwrite = False auto_update = False + test_mode = False def init(self): config.set_logger(self.logger) + if self.test_mode: + config.enable_test_mode(True) if self.auto_update: config.enable_auto_update(True) self.logger.debug("Feature 'auto_update' is enabled.") @@ -254,7 +258,7 @@ def run(cls, parsed_args=None): try: ctl = IntelMQController() for bot in runtime_conf: - if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('auto_update', True): + if runtime_conf[bot]["module"] == __name__: ctl.bot_reload(bot) except Exception as e: logger.error("Failed to signal bot: %r" % str(e)) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 3797f03cd..54a85e780 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -46,6 +46,7 @@ def test_broken(self): """ Test a report which does not have valid fields """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT1 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="Detected report's file name: 'test_smb'.", @@ -59,6 +60,7 @@ def test_half_broken(self): """ Test a report which does not have an optional field. """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT2 self.run_bot(allowed_warning_count=63) self.assertLogMatches(pattern="Detected report's file name: 'test_telnet'.", @@ -72,6 +74,7 @@ def test_no_config(self): """ Test a report which does not have a valid extra.file_name """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT3 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: Could not get a config for 'some_string', check the documentation.") @@ -80,6 +83,7 @@ def test_invalid_filename(self): """ Test a report which does not have a valid extra.file_name """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT4 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: Report's 'extra.file_name' '2020.wrong-filename.csv' is not valid.") @@ -89,6 +93,7 @@ def test_no_report_name(self): Test a report without file_name and no given feedname as parameter. Error message should be verbose. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: No feedname given as parameter and the " "processed report has no 'extra.file_name'. " diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index d296dfdc2..b764de827 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -48,6 +48,7 @@ def test_changed_feed(self): Tests if the parser correctly re-detects the feed for the second received report #1493 """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = (EXAMPLE_TELNET, EXAMPLE_VNC) self.run_bot(iterations=2) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py index 677cd0319..45a4a8735 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py @@ -63,13 +63,14 @@ def set_bot(cls): def test_default(self): """ Test if feed name is not overwritten has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) def test_overwrite_feed_name(self): """ Test if feed name is overwritten if asked to do so. """ - self.prepare_bot(parameters={'overwrite': True}) + self.prepare_bot(parameters={'test_mode': True, 'overwrite': True}) self.run_bot(prepare=False) for i, EVENT in enumerate(EVENTS): event = EVENT.copy() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py index 93d592d15..aa6940061 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -108,6 +108,7 @@ def set_bot(cls): def test_event(self): """ Test if correct Event has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index a9be8a0a1..488f5a51a 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -46,6 +46,7 @@ def set_bot(cls): def test_event(self): """ Test if the parser correctly detects and handles different report types. """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = [FIRST_REPORT, SECOND_REPORT] self.run_bot(iterations=2) self.assertLogMatches("Detected report's file name: 'test_smb'", diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py index df9cf25dc..b2499c589 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -78,6 +78,7 @@ def set_bot(cls): def test_event(self): """ Test if correct Event has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) From df6e62235001d64d23fb8f667f14962b0beb14e9 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 24 Aug 2023 16:26:59 +0000 Subject: [PATCH 33/67] Added VAR_STATE_PATH check. --- intelmq/bots/parsers/shadowserver/_config.py | 1 + .../parsers/shadowserver/test_download_schema.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 4bfadb9d9..6ffffdae8 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -117,6 +117,7 @@ def enable_test_mode(enable): else: __config.schema_active = __config.schema_file + def enable_auto_update(enable): """ Enable automatic schema update. """ __config.auto_update = enable diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index e68587682..f9512ca98 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -20,9 +20,10 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') - config.set_logger(utils.log('test-bot', log_path=None)) - if os.path.exists(schema_file): - os.unlink(schema_file) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + if os.path.isdir(VAR_STATE_PATH): + schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + config.set_logger(utils.log('test-bot', log_path=None)) + if os.path.exists(schema_file): + os.unlink(schema_file) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From 9195213959d2e0e2e464cb1359a0b69bb9d14f94 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 15:37:51 +0000 Subject: [PATCH 34/67] Changes based on feedback 2023-08-25. --- CHANGELOG.md | 6 +- docs/user/bots.rst | 171 ++++++------------ intelmq/bots/parsers/shadowserver/README.md | 57 ------ intelmq/bots/parsers/shadowserver/_config.py | 10 +- .../shadowserver/test_download_schema.py | 8 +- 5 files changed, 72 insertions(+), 180 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cee9e520..9fdc10225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,20 +118,18 @@ CHANGELOG ### Bots #### Collectors -<<<<<<< HEAD - `intelmq.bots.collector.rt`: - restrict `python-rt` to be below version 3.0 due to introduced breaking changes, - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). - `intelmq.bots.collectors.shadowserver.collector_reports_api`: - - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) + - The 'json' option is no longer supported as the 'csv' option provides better performance. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: - Reset detected `feedname` at shutdown to re-detect the feedname on reloads (PR#2361 by @elsif2, fixes #2360). - - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. Please see intelmq/bots/parsers/shadowserver/README.md for details. (PR#2372) -- `intelmq.bots.parsers.shadowserver._config`: + - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. - Added 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Service' report. (PR#2338) - Removed unused `p0f_genre` and `p0f_detail` from the 'DNS-Open-Resolvers' report. (PR#2338) - Added 'Accessible-SIP' report. (PR#2348) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index 2fbe27df8..a758ff8ad 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -673,6 +673,23 @@ The resulting reports contain the following special field: * `extra.file_name`: The name of the downloaded file, with fixed filename extension. The API returns file names with the extension `.csv`, although the files are JSON, not CSV. Therefore, for clarity and better error detection in the parser, the file name in `extra.file_name` uses `.json` as extension. +**Sample configuration** + +.. code-block:: yaml + + shadowserver-collector: + description: Our bot responsible for getting reports from Shadowserver + enabled: true + group: Collector + module: intelmq.bots.collectors.shadowserver.collector_reports_api + name: Shadowserver_Collector + parameters: + destination_queues: + _default: [shadowserver-parser-queue] + file_format: csv + api_key: "$API_KEY_received_from_the_shadowserver_foundation" + secret: "$SECRET_received_from_the_shadowserver_foundation" + run_mode: continuous .. _intelmq.bots.collectors.shodan.collector_stream: @@ -1557,17 +1574,15 @@ This does not affect URLs which already include the scheme. .. _intelmq.bots.parsers.shadowserver.parser: -.. _intelmq.bots.parsers.shadowserver.parser_json: Shadowserver ^^^^^^^^^^^^ -There are two Shadowserver parsers, one for data in ``CSV`` format (``intelmq.bots.parsers.shadowserver.parser``) and one for data in ``JSON`` format (``intelmq.bots.parsers.shadowserver.parser_json``). -The latter was added in IntelMQ 2.3 and is meant to be used together with the Shadowserver API collector. +The Shadowserver parser operates on ``CSV`` formatted data. **Information** -* `name:` `intelmq.bots.parsers.shadowserver.parser` (for CSV data) or `intelmq.bots.parsers.shadowserver.parser_json` (for JSON data) +* `name:` `intelmq.bots.parsers.shadowserver.parser` * `public:` yes * `description:` Parses different reports from Shadowserver. @@ -1603,107 +1618,45 @@ A list of possible feeds can be found in the table below in the column "feed nam **Supported reports** -These are the supported feed name and their corresponding file name for automatic detection: - - ======================================= ========================= - feed name file name - ======================================= ========================= - Accessible-ADB `scan_adb` - Accessible-AFP `scan_afp` - Accessible-AMQP `scan_amqp` - Accessible-ARD `scan_ard` - Accessible-Cisco-Smart-Install `cisco_smart_install` - Accessible-CoAP `scan_coap` - Accessible-CWMP `scan_cwmp` - Accessible-MS-RDPEUDP `scan_msrdpeudp` - Accessible-FTP `scan_ftp` - Accessible-Hadoop `scan_hadoop` - Accessible-HTTP `scan_http` - Accessible-Radmin `scan_radmin` - Accessible-RDP `scan_rdp` - Accessible-Rsync `scan_rsync` - Accessible-SMB `scan_smb` - Accessible-Telnet `scan_telnet` - Accessible-Ubiquiti-Discovery-Service `scan_ubiquiti` - Accessible-VNC `scan_vnc` - Blacklisted-IP (deprecated) `blacklist` - Blocklist `blocklist` - Compromised-Website `compromised_website` - Device-Identification IPv4 / IPv6 `device_id`/`device_id6` - DNS-Open-Resolvers `scan_dns` - Honeypot-Amplification-DDoS-Events `event4_honeypot_ddos_amp` - Honeypot-Brute-Force-Events `event4_honeypot_brute_force` - Honeypot-Darknet `event4_honeypot_darknet` - Honeypot-HTTP-Scan `event4_honeypot_http_scan` - HTTP-Scanners `hp_http_scan` - ICS-Scanners `hp_ics_scan` - IP-Spoofer-Events `event4_ip_spoofer` - Microsoft-Sinkhole-Events IPv4 `event4_microsoft_sinkhole` - Microsoft-Sinkhole-Events-HTTP IPv4 `event4_microsoft_sinkhole_http` - NTP-Monitor `scan_ntpmonitor` - NTP-Version `scan_ntp` - Open-Chargen `scan_chargen` - Open-DB2-Discovery-Service `scan_db2` - Open-Elasticsearch `scan_elasticsearch` - Open-IPMI `scan_ipmi` - Open-IPP `scan_ipp` - Open-LDAP `scan_ldap` - Open-LDAP-TCP `scan_ldap_tcp` - Open-mDNS `scan_mdns` - Open-Memcached `scan_memcached` - Open-MongoDB `scan_mongodb` - Open-MQTT `scan_mqtt` - Open-MSSQL `scan_mssql` - Open-NATPMP `scan_nat_pmp` - Open-NetBIOS-Nameservice `scan_netbios` - Open-Netis `netis_router` - Open-Portmapper `scan_portmapper` - Open-QOTD `scan_qotd` - Open-Redis `scan_redis` - Open-SNMP `scan_snmp` - Open-SSDP `scan_ssdp` - Open-TFTP `scan_tftp` - Open-XDMCP `scan_xdmcp` - Outdated-DNSSEC-Key `outdated_dnssec_key` - Outdated-DNSSEC-Key-IPv6 `outdated_dnssec_key_v6` - Sandbox-URL `cwsandbox_url` - Sinkhole-DNS `sinkhole_dns` - Sinkhole-Events `event4_sinkhole`/`event6_sinkhole` - Sinkhole-Events IPv4 `event4_sinkhole` - Sinkhole-Events IPv6 `event6_sinkhole` - Sinkhole-HTTP-Events `event4_sinkhole_http`/`event6_sinkhole_http` - Sinkhole-HTTP-Events IPv4 `event4_sinkhole_http` - Sinkhole-HTTP-Events IPv6 `event6_sinkhole_http` - Sinkhole-Events-HTTP-Referer `event4_sinkhole_http_referer`/`event6_sinkhole_http_referer` - Sinkhole-Events-HTTP-Referer IPv4 `event4_sinkhole_http_referer` - Sinkhole-Events-HTTP-Referer IPv6 `event6_sinkhole_http_referer` - Spam-URL `spam_url` - SSL-FREAK-Vulnerable-Servers `scan_ssl_freak` - SSL-POODLE-Vulnerable-Servers `scan_ssl_poodle`/`scan6_ssl_poodle` - Vulnerable-Exchange-Server `*` `scan_exchange` - Vulnerable-ISAKMP `scan_isakmp` - Vulnerable-HTTP `scan_http` - Vulnerable-SMTP `scan_smtp_vulnerable` - ======================================= ========================= - -`*` This report can also contain data on active webshells (column `tag` is `exchange;webshell`), and are therefore not only vulnerable but also actively infected. - -In addition, the following legacy reports are supported: - - =========================== =================================================== ======================== - feed name successor feed name file name - =========================== =================================================== ======================== - Amplification-DDoS-Victim Honeypot-Amplification-DDoS-Events ``ddos_amplification`` - CAIDA-IP-Spoofer IP-Spoofer-Events ``caida_ip_spoofer`` - Darknet Honeypot-Darknet ``darknet`` - Drone Sinkhole-Events ``botnet_drone`` - Drone-Brute-Force Honeypot-Brute-Force-Events, Sinkhole-HTTP-Events ``drone_brute_force`` - Microsoft-Sinkhole Sinkhole-HTTP-Events ``microsoft_sinkhole`` - Sinkhole-HTTP-Drone Sinkhole-HTTP-Events ``sinkhole_http_drone`` - IPv6-Sinkhole-HTTP-Drone Sinkhole-HTTP-Events ``sinkhole6_http`` - =========================== =================================================== ======================== - -More information on these legacy reports can be found in `Changes in Sinkhole and Honeypot Report Types and Formats `_. +The report configuration is stored in a `shadowserver-schema.json` file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. + +The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. + +Schema downloads can also be scheduled as a cron job: + +.. code-block:: bash + + 02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema + + +For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. + +The parser will automatically reload the configuration when the file changes. + +**Schema contract** + +Once set in the schema, the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static for a specific report. + +Report fields will not be removed from a report. + +The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. + +**Sample configuration** + +.. code-block:: yaml + + shadowserver-parser: + bot_id: shadowserver-parser + name: Shadowserver Parser + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.shadowserver.parser + parameters: + destination_queues: + _default: [file-output-queue] + auto_update: true + run_mode: continuous **Development** @@ -1715,14 +1668,6 @@ The parser consists of two files: Both files are required for the parser to work properly. -**Add new Feedformats** - -Add a new feed format and conversions if required to the file -``_config.py``. Don't forget to update the ``mapping`` dict. -It is required to look up the correct configuration. - -Look at the documentation in the bot's ``_config.py`` file for more information. - .. _intelmq.bots.parsers.shodan.parser: diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index 4969acb6d..eb0ddfb4a 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,60 +7,3 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _shadowserver-schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. - -The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. - -Schema downloads can also be scheduled as a cron job: - -``` -02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema -``` - -For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. - -The parser will automatically reload the configuration when the file changes. - - -## Schema contract - -Once set the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static. - -Once set report fields will not be deleted. - -The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. - - -## Sample configuration: - -``` -shadowserver-collector: - description: Our bot responsible for getting reports from Shadowserver - enabled: true - group: Collector - module: intelmq.bots.collectors.shadowserver.collector_reports_api - name: Shadowserver_Collector - parameters: - destination_queues: - _default: [shadowserver-parser-queue] - file_format: csv - api_key: "$API_KEY_received_from_the_shadowserver_foundation" - secret: "$SECRET_received_from_the_shadowserver_foundation" - run_mode: continuous -``` - -``` -shadowserver-parser: - bot_id: shadowserver-parser - name: Shadowserver Parser - enabled: true - group: Parser - groupname: parsers - module: intelmq.bots.parsers.shadowserver.parser - parameters: - destination_queues: - _default: [file-output-queue] - auto_update: true - run_mode: continuous -``` - diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 6ffffdae8..279093dfe 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -95,6 +95,7 @@ class __Container: __config = __Container() +__config.var_state_path = VAR_STATE_PATH __config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema' __config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') __config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') @@ -328,7 +329,7 @@ def update_schema(): """ download the latest configuration """ # download the schema to a temp file - (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) + (th, tmp) = tempfile.mkstemp(dir=__config.var_state_path) __config.logger.info("Attempting to download schema from %r", __config.schema_url) __config.logger.debug("Using temp file %r for the download.", tmp) try: @@ -376,3 +377,10 @@ def update_schema(): os.unlink(tmp) return False + + +def prepare_update_schema_test(path): + """ Reconfigure internal settings to perform a schema update test. """ + __config.var_state_path = path + __config.schema_file = os.path.join(path, 'shadowserver-schema.json') + return __config.schema_file diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index f9512ca98..5246e6bb6 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -10,8 +10,8 @@ import unittest import os +import tempfile import logging -from intelmq import VAR_STATE_PATH import intelmq.bots.parsers.shadowserver._config as config import intelmq.lib.utils as utils import intelmq.lib.test as test @@ -20,10 +20,8 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - if os.path.isdir(VAR_STATE_PATH): - schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) config.set_logger(utils.log('test-bot', log_path=None)) - if os.path.exists(schema_file): - os.unlink(schema_file) self.assertEqual(True, config.update_schema()) self.assertEqual(True, os.path.exists(schema_file)) From cc48565bb325c26a7e92690185474525f48c04e5 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 15:51:38 +0000 Subject: [PATCH 35/67] Added INTELMQ_SKIP_INTERNET check --- .../bots/parsers/shadowserver/test_download_schema.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index 5246e6bb6..203a3c0b1 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -20,8 +20,9 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - with tempfile.TemporaryDirectory() as tmp_dir: - schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None)) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + if not os.environ.get('INTELMQ_SKIP_INTERNET'): + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) + config.set_logger(utils.log('test-bot', log_path=None)) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From 16daee468f62209459647f242d938dac56fc40de Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 16:11:21 +0000 Subject: [PATCH 36/67] Added debug logging for CI test. --- intelmq/bots/parsers/shadowserver/_config.py | 3 ++- .../tests/bots/parsers/shadowserver/test_download_schema.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 279093dfe..d573d12c6 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -339,8 +339,9 @@ def update_schema(): with open(tmp, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) - except: + except Exception as e: __config.logger.error("Failed to download %r", __config.schema_url) + __config.logger.debug(str(e)) return False __config.logger.info("Download successful.") diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index 203a3c0b1..abcd0ca2a 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -23,6 +23,6 @@ def test_download(self): if not os.environ.get('INTELMQ_SKIP_INTERNET'): with tempfile.TemporaryDirectory() as tmp_dir: schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None)) + config.set_logger(utils.log('test-bot', log_path=None, log_level=logging.DEBUG)) self.assertEqual(True, config.update_schema()) self.assertEqual(True, os.path.exists(schema_file)) From f102f2c0b7eef245db04c39aba28517090a93129 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 18:47:54 +0000 Subject: [PATCH 37/67] Refactored test_download_schema to utilize mocking. --- intelmq/bots/parsers/shadowserver/parser.py | 6 ++++ .../shadowserver/test_download_schema.py | 30 ++++++++++++------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index fd9fa6b2c..48cbba901 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -23,6 +23,7 @@ import copy import re import os +import tempfile from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue @@ -265,5 +266,10 @@ def run(cls, parsed_args=None): else: super().run(parsed_args=parsed_args) + def test_update_schema(cls): + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) + return config.update_schema() + BOT = ShadowserverParserBot diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index abcd0ca2a..abf27a5bd 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -8,21 +8,29 @@ """ -import unittest -import os -import tempfile import logging -import intelmq.bots.parsers.shadowserver._config as config +import unittest +import unittest.mock as mock +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot import intelmq.lib.utils as utils import intelmq.lib.test as test + @test.skip_internet() -class TestShadowserverSchemaDownload(unittest.TestCase): +class TestShadowserverSchemaDownload(test.BotTestCase, unittest.TestCase): + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.sysconfig = {"logging_level": "DEBUG"} def test_download(self): - if not os.environ.get('INTELMQ_SKIP_INTERNET'): - with tempfile.TemporaryDirectory() as tmp_dir: - schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None, log_level=logging.DEBUG)) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + self.prepare_bot(prepare_source_queue=False, parameters={'test_mode': True}) + result = False + with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): + with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)): + self.log_stream.truncate(0) + result = self.bot.test_update_schema() + self.bot.stop(exitcode=0) + print(self.log_stream.getvalue()) + self.assertEqual(True, result) From b103282cb083ba586b40559606d47e33ac8c5b86 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 28 Aug 2023 14:18:22 +0000 Subject: [PATCH 38/67] Added docstring for test_update_schema(). --- intelmq/bots/parsers/shadowserver/parser.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 48cbba901..4485a2602 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -267,6 +267,13 @@ def run(cls, parsed_args=None): super().run(parsed_args=parsed_args) def test_update_schema(cls): + """ + Test schema download to a temporary directory. + + This is necessary as the request session requires mocking in order to function. + + Returns True on success. + """ with tempfile.TemporaryDirectory() as tmp_dir: schema_file = config.prepare_update_schema_test(tmp_dir) return config.update_schema() From 356b956a3ce79eaa723c774bc54eafa149a5b528 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 29 Aug 2023 14:09:33 +0000 Subject: [PATCH 39/67] Removed logging output. --- intelmq/tests/bots/parsers/shadowserver/test_download_schema.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index abf27a5bd..84922bf17 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -29,8 +29,6 @@ def test_download(self): result = False with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)): - self.log_stream.truncate(0) result = self.bot.test_update_schema() self.bot.stop(exitcode=0) - print(self.log_stream.getvalue()) self.assertEqual(True, result) From c72d553fdab8546dda0b669ab1557ace6745e644 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 31 Aug 2023 20:52:17 +0000 Subject: [PATCH 40/67] Removed the assertion regarding report fields. --- docs/user/bots.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index a758ff8ad..ae17cbf55 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -1637,8 +1637,6 @@ The parser will automatically reload the configuration when the file changes. Once set in the schema, the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static for a specific report. -Report fields will not be removed from a report. - The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. **Sample configuration** From 3b60c2f9699f576ecadb262fb2ad592112a9a69e Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 16 Oct 2023 17:57:46 +0000 Subject: [PATCH 41/67] Skip and log a warning message for fields not in the IDF. --- intelmq/bots/parsers/shadowserver/parser.py | 5 ++- .../parsers/shadowserver/schema.json.test | 37 +++++++++++++++++++ .../bots/parsers/shadowserver/test_broken.py | 15 ++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 4485a2602..cfa343138 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -157,7 +157,10 @@ def parse_line(self, row, report): raise if value is not None: - event.add(intelmqkey, value) + try: + event.add(intelmqkey, value) + except InvalidKey: + self.logger.warning('Key not found in IDF %r.', intelmqkey) fields.remove(shadowkey) # Now add optional fields. diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test b/intelmq/bots/parsers/shadowserver/schema.json.test index 2cfb8bb1d..932b8df03 100644 --- a/intelmq/bots/parsers/shadowserver/schema.json.test +++ b/intelmq/bots/parsers/shadowserver/schema.json.test @@ -176,5 +176,42 @@ "convert_int" ] ] + }, + "test_afs" : { + "constant_fields" : { + "classification.identifier" : "test-afs", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "afs" + }, + "feed_name" : "Test-Accessible-AFS", + "file_name" : "test_afs", + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ], + [ + "not_in_idf", + "severity" + ] + ], + "optional_fields" : [ + [ + "protocol.transport", + "protocol" + ] + ] } } diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 54a85e780..f1af08e58 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -30,6 +30,11 @@ "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2020.wrong-filename.csv", } +REPORT5 = {"raw": utils.base64_encode('timestamp,ip,protocol,port,severity\n2018-08-01T00:00:00+00,127.0.0.1,tcp,7000,critical'), + "__type": "Report", + "time.observation": "2023-10-16T00:00:00+00:00", + "extra.file_name": "2023-10-16-test_afs-test-test.csv", + } class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): @@ -100,6 +105,16 @@ def test_no_report_name(self): "Ensure that at least one is given. " "Also have a look at the documentation of the bot.") + def test_field_not_in_idf(self): + """ + Test a report that contains a field mapping not in the IDF. + Error message should be verbose. + """ + self.prepare_bot(parameters={'test_mode': True}) + self.input_message = REPORT5 + self.run_bot(allowed_error_count=0, allowed_warning_count=1) + self.assertLogMatches(pattern="Key not found in IDF", levelname="WARNING") + if __name__ == '__main__': # pragma: no cover unittest.main() From 473f6a64c671d8910ec427daf5ba791eee82887a Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:53:24 +0000 Subject: [PATCH 42/67] remove obsolete tests and data --- .../shadowserver/scan_rdpeudp.csv.license | 2 - .../parsers/shadowserver/test_blocklist.py | 103 ------- .../shadowserver/test_compromised_website.py | 88 ------ .../parsers/shadowserver/test_device_id.py | 116 -------- .../test_event4_ddos_participant.py | 131 --------- .../test_event4_honeypot_darknet.py | 106 ------- .../shadowserver/test_event4_honeypot_ddos.py | 148 ---------- .../test_event4_honeypot_ddos_target.py | 150 ---------- .../test_event4_honeypot_http_scan.py | 109 -------- .../shadowserver/test_event4_ip_spoofer.py | 182 ------------ .../test_event4_microsoft_sinkhole.py | 135 --------- .../test_event4_microsoft_sinkhole_http.py | 202 -------------- .../shadowserver/test_event4_sinkhole.py | 73 ----- .../shadowserver/test_event4_sinkhole_dns.py | 127 --------- .../shadowserver/test_event4_sinkhole_http.py | 189 ------------- .../test_event4_sinkhole_http_referer.py | 213 --------------- .../shadowserver/test_event6_sinkhole_http.py | 146 ---------- .../shadowserver/test_honeypot_brute_force.py | 72 ----- .../shadowserver/test_honeypot_ddos_amp.py | 91 ------ .../parsers/shadowserver/test_malware_url.py | 107 -------- .../parsers/shadowserver/test_phish_url.py | 106 ------- .../test_population_http_proxy.py | 130 --------- .../parsers/shadowserver/test_sandbox_conn.py | 99 ------- .../parsers/shadowserver/test_sandbox_dns.py | 95 ------- .../parsers/shadowserver/test_sandbox_url.py | 104 ------- .../parsers/shadowserver/test_scan_adb.py | 98 ------- .../parsers/shadowserver/test_scan_afp.py | 106 ------- .../parsers/shadowserver/test_scan_amqp.py | 144 ---------- .../parsers/shadowserver/test_scan_ard.py | 111 -------- .../parsers/shadowserver/test_scan_chargen.py | 110 -------- .../test_scan_cisco_smart_install.py | 82 ------ .../parsers/shadowserver/test_scan_coap.py | 121 -------- .../parsers/shadowserver/test_scan_couchdb.py | 128 --------- .../parsers/shadowserver/test_scan_cwmp.py | 103 ------- .../parsers/shadowserver/test_scan_db2.py | 91 ------ .../shadowserver/test_scan_ddos_middlebox.py | 119 -------- .../parsers/shadowserver/test_scan_dns.py | 91 ------ .../parsers/shadowserver/test_scan_docker.py | 159 ----------- .../test_scan_dvr_dhcpdiscover.py | 178 ------------ .../shadowserver/test_scan_elasticsearch.py | 126 --------- .../shadowserver/test_scan_exchange.py | 149 ---------- .../parsers/shadowserver/test_scan_ftp.py | 120 -------- .../parsers/shadowserver/test_scan_hadoop.py | 94 ------- .../parsers/shadowserver/test_scan_http.py | 100 ------- .../shadowserver/test_scan_http_proxy.py | 118 -------- .../shadowserver/test_scan_http_vulnerable.py | 125 --------- .../parsers/shadowserver/test_scan_ics.py | 125 --------- .../parsers/shadowserver/test_scan_ipmi.py | 106 ------- .../parsers/shadowserver/test_scan_ipp.py | 79 ------ .../parsers/shadowserver/test_scan_isakmp.py | 105 ------- .../shadowserver/test_scan_kubernetes.py | 214 --------------- .../shadowserver/test_scan_ldap_tcp.py | 154 ----------- .../shadowserver/test_scan_ldap_udp.py | 162 ----------- .../parsers/shadowserver/test_scan_mdns.py | 127 --------- .../shadowserver/test_scan_memcached.py | 130 --------- .../parsers/shadowserver/test_scan_mongodb.py | 103 ------- .../parsers/shadowserver/test_scan_mqtt.py | 89 ------ .../shadowserver/test_scan_mqtt_anon.py | 173 ------------ .../parsers/shadowserver/test_scan_mssql.py | 123 --------- .../parsers/shadowserver/test_scan_mysql.py | 258 ------------------ .../parsers/shadowserver/test_scan_nat_pmp.py | 116 -------- .../parsers/shadowserver/test_scan_netbios.py | 121 -------- .../shadowserver/test_scan_netis_router.py | 107 -------- .../parsers/shadowserver/test_scan_ntp.py | 161 ----------- .../shadowserver/test_scan_ntpmonitor.py | 108 -------- .../shadowserver/test_scan_portmapper.py | 120 -------- .../shadowserver/test_scan_postgres.py | 199 -------------- .../parsers/shadowserver/test_scan_qotd.py | 119 -------- .../parsers/shadowserver/test_scan_quic.py | 118 -------- .../parsers/shadowserver/test_scan_radmin.py | 236 ---------------- .../parsers/shadowserver/test_scan_rdp.py | 117 -------- .../parsers/shadowserver/test_scan_rdpeudp.py | 109 -------- .../parsers/shadowserver/test_scan_redis.py | 107 -------- .../parsers/shadowserver/test_scan_rsync.py | 116 -------- .../parsers/shadowserver/test_scan_sip.py | 124 --------- .../parsers/shadowserver/test_scan_slp.py | 137 ---------- .../parsers/shadowserver/test_scan_smb.py | 124 --------- .../shadowserver/test_scan_smb_json.py | 123 --------- .../shadowserver/test_scan_smtp_vulnerable.py | 92 ------- .../parsers/shadowserver/test_scan_snmp.py | 120 -------- .../parsers/shadowserver/test_scan_socks.py | 107 -------- .../parsers/shadowserver/test_scan_ssdp.py | 136 --------- .../parsers/shadowserver/test_scan_ssh.py | 182 ------------ .../parsers/shadowserver/test_scan_ssl.py | 218 --------------- .../shadowserver/test_scan_ssl_freak.py | 136 --------- .../shadowserver/test_scan_ssl_poodle.py | 91 ------ .../parsers/shadowserver/test_scan_stun.py | 146 ---------- .../shadowserver/test_scan_synfulknock.py | 117 -------- .../parsers/shadowserver/test_scan_telnet.py | 87 ------ .../parsers/shadowserver/test_scan_tftp.py | 121 -------- .../shadowserver/test_scan_ubiquiti.py | 124 --------- .../parsers/shadowserver/test_scan_vnc.py | 86 ------ .../shadowserver/test_scan_ws_discovery.py | 119 -------- .../parsers/shadowserver/test_scan_xdmcp.py | 117 -------- .../bots/parsers/shadowserver/test_special.py | 106 ------- .../parsers/shadowserver/test_testdata.py | 81 ------ .../shadowserver/testdata/blocklist.csv | 4 - .../testdata/blocklist.csv.license | 2 - .../testdata/botnet_drone.csv.license | 2 - .../testdata/caida_ip_spoofer.csv.license | 2 - .../testdata/compromised_website.csv | 4 - .../testdata/compromised_website.csv.license | 2 - .../shadowserver/testdata/darknet.csv.license | 2 - .../testdata/ddos_amplification.csv.license | 2 - .../shadowserver/testdata/device_id.csv | 4 - .../testdata/device_id.csv.license | 2 - .../testdata/drone_brute_force.csv.license | 2 - .../testdata/event4_ddos_participant.csv | 4 - .../event4_ddos_participant.csv.license | 2 - .../testdata/event4_honeypot_brute_force.csv | 7 - .../event4_honeypot_brute_force.csv.license | 2 - .../testdata/event4_honeypot_darknet.csv | 9 - .../event4_honeypot_darknet.csv.license | 2 - .../testdata/event4_honeypot_ddos.csv | 4 - .../testdata/event4_honeypot_ddos.csv.license | 2 - .../testdata/event4_honeypot_ddos_amp.csv | 6 - .../event4_honeypot_ddos_amp.csv.license | 2 - .../testdata/event4_honeypot_ddos_target.csv | 4 - .../event4_honeypot_ddos_target.csv.license | 2 - .../testdata/event4_honeypot_http_scan.csv | 3 - .../event4_honeypot_http_scan.csv.license | 2 - .../testdata/event4_ip_spoofer.csv | 7 - .../testdata/event4_ip_spoofer.csv.license | 2 - .../testdata/event4_microsoft_sinkhole.csv | 7 - .../event4_microsoft_sinkhole.csv.license | 2 - .../event4_microsoft_sinkhole_http.csv | 6 - ...event4_microsoft_sinkhole_http.csv.license | 2 - .../shadowserver/testdata/event4_sinkhole.csv | 4 - .../testdata/event4_sinkhole.csv.license | 2 - .../testdata/event4_sinkhole_dns.csv | 4 - .../testdata/event4_sinkhole_dns.csv.license | 2 - .../testdata/event4_sinkhole_http.csv | 6 - .../testdata/event4_sinkhole_http.csv.license | 2 - .../testdata/event4_sinkhole_http_referer.csv | 6 - .../event4_sinkhole_http_referer.csv.license | 2 - .../testdata/event6_sinkhole_http.csv | 4 - .../testdata/event6_sinkhole_http.csv.license | 2 - .../testdata/hp_http_scan.csv.license | 2 - .../testdata/hp_ics_scan.csv.license | 2 - .../shadowserver/testdata/malware_url.csv | 4 - .../testdata/malware_url.csv.license | 2 - .../testdata/outdated_dnssec_key.csv.license | 2 - .../shadowserver/testdata/phish_url.csv | 4 - .../testdata/phish_url.csv.license | 2 - .../testdata/population_http_proxy.csv | 4 - .../population_http_proxy.csv.license | 2 - .../shadowserver/testdata/sandbox_conn.csv | 4 - .../testdata/sandbox_conn.csv.license | 2 - .../shadowserver/testdata/sandbox_dns.csv | 4 - .../testdata/sandbox_dns.csv.license | 2 - .../shadowserver/testdata/sandbox_url.csv | 4 - .../testdata/sandbox_url.csv.license | 2 - .../shadowserver/testdata/scan_adb.csv | 3 - .../testdata/scan_adb.csv.license | 2 - .../shadowserver/testdata/scan_afp.csv | 3 - .../testdata/scan_afp.csv.license | 2 - .../shadowserver/testdata/scan_amqp.csv | 4 - .../testdata/scan_amqp.csv.license | 2 - .../shadowserver/testdata/scan_ard.csv | 4 - .../testdata/scan_ard.csv.license | 2 - .../shadowserver/testdata/scan_chargen.csv | 4 - .../testdata/scan_chargen.csv.license | 2 - .../testdata/scan_cisco_smart_install.csv | 3 - .../scan_cisco_smart_install.csv.license | 2 - .../shadowserver/testdata/scan_coap.csv | 4 - .../testdata/scan_coap.csv.license | 2 - .../shadowserver/testdata/scan_couchdb.csv | 4 - .../testdata/scan_couchdb.csv.license | 2 - .../shadowserver/testdata/scan_cwmp.csv | 3 - .../testdata/scan_cwmp.csv.license | 2 - .../shadowserver/testdata/scan_db2.csv | 3 - .../testdata/scan_db2.csv.license | 2 - .../testdata/scan_ddos_middlebox.csv | 4 - .../testdata/scan_ddos_middlebox.csv.license | 2 - .../shadowserver/testdata/scan_dns.csv | 101 ------- .../testdata/scan_dns.csv.license | 2 - .../shadowserver/testdata/scan_docker.csv | 4 - .../testdata/scan_docker.csv.license | 2 - .../testdata/scan_dvr_dhcpdiscover.csv | 4 - .../scan_dvr_dhcpdiscover.csv.license | 2 - .../testdata/scan_elasticsearch.csv | 4 - .../testdata/scan_elasticsearch.csv.license | 2 - .../shadowserver/testdata/scan_exchange.csv | 8 - .../testdata/scan_exchange.csv.license | 2 - .../shadowserver/testdata/scan_ftp.csv | 3 - .../testdata/scan_ftp.csv.license | 2 - .../shadowserver/testdata/scan_hadoop.csv | 3 - .../testdata/scan_hadoop.csv.license | 2 - .../shadowserver/testdata/scan_http.csv | 3 - .../testdata/scan_http.csv.license | 2 - .../shadowserver/testdata/scan_http_proxy.csv | 4 - .../testdata/scan_http_proxy.csv.license | 2 - .../testdata/scan_http_vulnerable.csv | 4 - .../testdata/scan_http_vulnerable.csv.license | 2 - .../shadowserver/testdata/scan_ics.csv | 4 - .../testdata/scan_ics.csv.license | 2 - .../shadowserver/testdata/scan_ipmi.csv | 96 ------- .../testdata/scan_ipmi.csv.license | 2 - .../shadowserver/testdata/scan_ipp.csv | 2 - .../testdata/scan_ipp.csv.license | 2 - .../shadowserver/testdata/scan_isakmp.csv | 3 - .../testdata/scan_isakmp.csv.license | 2 - .../shadowserver/testdata/scan_kubernetes.csv | 4 - .../testdata/scan_kubernetes.csv.license | 2 - .../shadowserver/testdata/scan_ldap_tcp.csv | 4 - .../testdata/scan_ldap_tcp.csv.license | 2 - .../shadowserver/testdata/scan_ldap_udp.csv | 4 - .../testdata/scan_ldap_udp.csv.license | 2 - .../shadowserver/testdata/scan_mdns.csv | 4 - .../testdata/scan_mdns.csv.license | 2 - .../shadowserver/testdata/scan_memcached.csv | 4 - .../testdata/scan_memcached.csv.license | 2 - .../shadowserver/testdata/scan_mongodb.csv | 11 - .../testdata/scan_mongodb.csv.license | 2 - .../shadowserver/testdata/scan_mqtt.csv | 2 - .../testdata/scan_mqtt.csv.license | 2 - .../shadowserver/testdata/scan_mqtt_anon.csv | 4 - .../testdata/scan_mqtt_anon.csv.license | 2 - .../shadowserver/testdata/scan_mssql.csv | 4 - .../testdata/scan_mssql.csv.license | 2 - .../shadowserver/testdata/scan_mysql.csv | 4 - .../testdata/scan_mysql.csv.license | 2 - .../shadowserver/testdata/scan_nat_pmp.csv | 4 - .../testdata/scan_nat_pmp.csv.license | 2 - .../shadowserver/testdata/scan_netbios.csv | 4 - .../testdata/scan_netbios.csv.license | 2 - .../testdata/scan_netis_router.csv | 4 - .../testdata/scan_netis_router.csv.license | 2 - .../shadowserver/testdata/scan_ntp.csv | 4 - .../testdata/scan_ntp.csv.license | 2 - .../shadowserver/testdata/scan_ntpmonitor.csv | 4 - .../testdata/scan_ntpmonitor.csv.license | 2 - .../shadowserver/testdata/scan_portmapper.csv | 4 - .../testdata/scan_portmapper.csv.license | 2 - .../shadowserver/testdata/scan_postgres.csv | 4 - .../testdata/scan_postgres.csv.license | 2 - .../shadowserver/testdata/scan_qotd.csv | 4 - .../testdata/scan_qotd.csv.license | 2 - .../shadowserver/testdata/scan_quic.csv | 4 - .../testdata/scan_quic.csv.license | 2 - .../shadowserver/testdata/scan_radmin.csv | 10 - .../testdata/scan_radmin.csv.license | 2 - .../shadowserver/testdata/scan_rdp.csv | 3 - .../testdata/scan_rdp.csv.license | 2 - .../shadowserver/testdata/scan_rdpeudp.csv | 4 - .../testdata/scan_rdpeudp.csv.license | 2 - .../shadowserver/testdata/scan_redis.csv | 94 ------- .../testdata/scan_redis.csv.license | 2 - .../shadowserver/testdata/scan_rsync.csv | 4 - .../testdata/scan_rsync.csv.license | 2 - .../shadowserver/testdata/scan_sip.csv | 4 - .../testdata/scan_sip.csv.license | 2 - .../shadowserver/testdata/scan_slp.csv | 4 - .../testdata/scan_slp.csv.license | 2 - .../shadowserver/testdata/scan_smb.csv | 4 - .../testdata/scan_smb.csv.license | 2 - .../testdata/scan_smtp_vulnerable.csv | 3 - .../testdata/scan_smtp_vulnerable.csv.license | 2 - .../shadowserver/testdata/scan_snmp.csv | 4 - .../testdata/scan_snmp.csv.license | 2 - .../shadowserver/testdata/scan_socks.csv | 4 - .../testdata/scan_socks.csv.license | 2 - .../shadowserver/testdata/scan_ssdp.csv | 4 - .../testdata/scan_ssdp.csv.license | 2 - .../shadowserver/testdata/scan_ssh.csv | 4 - .../testdata/scan_ssh.csv.license | 2 - .../shadowserver/testdata/scan_ssl.csv | 4 - .../testdata/scan_ssl.csv.license | 2 - .../shadowserver/testdata/scan_ssl_freak.csv | 46 ---- .../testdata/scan_ssl_freak.csv.license | 2 - .../shadowserver/testdata/scan_ssl_poodle.csv | 32 --- .../testdata/scan_ssl_poodle.csv.license | 2 - .../shadowserver/testdata/scan_stun.csv | 4 - .../testdata/scan_stun.csv.license | 2 - .../testdata/scan_synfulknock.csv | 4 - .../testdata/scan_synfulknock.csv.license | 2 - .../shadowserver/testdata/scan_telnet.csv | 3 - .../testdata/scan_telnet.csv.license | 2 - .../shadowserver/testdata/scan_tftp.csv | 4 - .../testdata/scan_tftp.csv.license | 2 - .../shadowserver/testdata/scan_ubiquiti.csv | 4 - .../testdata/scan_ubiquiti.csv.license | 2 - .../shadowserver/testdata/scan_vnc.csv | 3 - .../testdata/scan_vnc.csv.license | 2 - .../testdata/scan_ws_discovery.csv | 4 - .../testdata/scan_ws_discovery.csv.license | 2 - .../shadowserver/testdata/scan_xdmcp.csv | 4 - .../testdata/scan_xdmcp.csv.license | 2 - .../testdata/sinkhole_http_drone.csv.license | 2 - .../parsers/shadowserver/testdata/special.csv | 4 - .../shadowserver/testdata/special.csv.license | 2 - 291 files changed, 12939 deletions(-) delete mode 100644 intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_blocklist.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_device_id.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_malware_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_phish_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_special.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/test_testdata.py delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/special.csv delete mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license diff --git a/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license b/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license deleted file mode 100644 index 043ed079f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/scan_rdpeudp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py b/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py deleted file mode 100644 index 48509eea0..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_blocklist.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/blocklist.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - 'feed.name': 'Block Listed IP Addresses', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-blocklist-test-geo.csv", -} -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.134", - "source.reverse_dns": "host.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.171", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Block Listed IP Addresses', - "classification.identifier": "blacklisted-ip", - "classification.taxonomy": "other", - "classification.type": "blacklist", - "extra.naics": 517311, - "extra.reason": "Malicious Host AA", - "extra.source": "Alien Vault", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - "source.asn": 5678, - "source.geolocation.cc": "XX", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.network": "198.123.245.0/24", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T07:00:19+00:00" -},] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py b/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py deleted file mode 100644 index 53c5b247b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_compromised_website.py +++ /dev/null @@ -1,88 +0,0 @@ -# SPDX-FileCopyrightText: 2017 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/compromised_website.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Compromised Website", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-compromised_website-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'ShadowServer Compromised Website', - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'classification.identifier': 'compromised-website', - 'extra.server': 'Microsoft-IIS/7.5', - 'extra.system': 'WINNT', - 'extra.detected_since': '2015-05-09 05:51:12', - 'protocol.application': 'http', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 64496, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '203.0.113.1', - 'source.port': 80, - 'source.url': 'http://example.com/header.php', - 'source.fqdn': 'example.com', - 'source.reverse_dns': 'example.com', - 'malware.name': 'hacked-webserver-stealrat-t1', - 'event_description.text': 'spam', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-01-16T00:43:48+00:00'}, - {'__type': 'Event', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'classification.identifier': 'compromised-website', - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'event_description.text': 'phishing', - 'feed.name': 'ShadowServer Compromised Website', - 'malware.name': 'phishing', - 'protocol.application': 'http', - 'source.asn': 64496, - 'source.fqdn': 'example.com', - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'GRAZ', - 'source.geolocation.region': 'STEIERMARK', - 'source.ip': '203.0.113.1', - 'source.port': 80, - 'source.url': 'http://example.com/', - 'time.source': '2018-04-09T15:43:41+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_device_id.py b/intelmq/tests/bots/parsers/shadowserver/test_device_id.py deleted file mode 100644 index e8954e03c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_device_id.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/device_id.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Device ID', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-device_id-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'extra.naics' : 517311, - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 2116, - 'source.geolocation.cc' : 'NO', - 'source.geolocation.city' : 'TROMVIK', - 'source.geolocation.region' : 'TROMS OG FINNMARK', - 'source.ip' : '88.84.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 27843, - 'source.geolocation.cc' : 'PE', - 'source.geolocation.city' : 'LIMA', - 'source.geolocation.region' : 'METROPOLITANA DE LIMA', - 'source.ip' : '170.231.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'device-id', - 'classification.taxonomy' : 'other', - 'classification.type' : 'undetermined', - 'extra.device_model' : 'FortiGate', - 'extra.device_type' : 'firewall', - 'extra.device_vendor' : 'Fortinet', - 'extra.naics' : 517311, - 'feed.name' : 'Device ID', - 'extra.tag' : 'ssl,vpn', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 4181, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MILWAUKEE', - 'source.geolocation.region' : 'WISCONSIN', - 'source.ip' : '96.60.0.0', - 'source.port' : 10443, - 'source.reverse_dns' : '96-60-66-218.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py deleted file mode 100644 index badc53a73..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_ddos_participant.py +++ /dev/null @@ -1,131 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_ddos_participant.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DDoS Participant', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_ddos_participant-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.1', - 'destination.port': 443, - 'destination.reverse_dns': 'node01.example.net', - 'extra.application': 'https', - 'extra.domain': 'www.example.com', - 'extra.http_method': 'GET', - 'extra.http_path': '/??=GovpfOoaWYlk', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 38055, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.2', - 'destination.port': 53, - 'destination.reverse_dns': 'node02.example.net', - 'extra.application': 'dns', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 53, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ddos-participant', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'destination.asn': 65534, - 'destination.geolocation.cc': 'ZZ', - 'destination.geolocation.city': 'City', - 'destination.geolocation.region': 'Region', - 'destination.ip': '172.16.0.3', - 'destination.port': 53, - 'destination.reverse_dns': 'node03.example.net', - 'extra.application': 'dns', - 'extra.device_model': 'Exchange', - 'extra.device_type': 'email', - 'extra.device_vendor': 'Microsoft', - 'feed.name': 'DDoS Participant', - 'malware.name': 'ddos-participant', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 53, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py deleted file mode 100644 index 1d020f473..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_darknet.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_darknet.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Darknet", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_darknet.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.port': 23, - 'extra.source.naics': 518210, - 'extra.tag': 'mirai', - 'protocol.transport': 'tcp', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 9829, - 'source.geolocation.cc': 'IN', - 'source.geolocation.city': 'CHENGANNUR', - 'source.geolocation.region': 'KERALA', - 'source.ip': '61.3.1.2', - 'source.port': 4717, - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - {'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.port': 23, - 'protocol.transport': 'tcp', - 'extra.source.naics': 517311, - 'extra.tag': 'mirai', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 4766, - 'source.geolocation.cc': 'KR', - 'source.geolocation.city': 'PYEONGCHANG-EUP', - 'source.geolocation.region': 'GANGWON-DO', - 'source.ip': '211.218.3.4', - 'source.port': 4405, - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - }, - {'__type': 'Event', - 'classification.identifier': 'mirai', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.tag': 'mirai', - 'destination.port': 23, - 'protocol.transport': 'tcp', - 'feed.name': 'ShadowServer Darknet', - 'malware.name': 'mirai', - 'source.asn': 266915, - 'source.geolocation.cc': 'BR', - 'source.geolocation.city': 'VITORIA DA CONQUISTA', - 'source.geolocation.region': 'BAHIA', - 'source.ip': '45.225.5.6', - 'source.port': 59777, - 'source.reverse_dns': 'static-45-225-x-x.example.net', - 'time.source': '2021-03-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py deleted file mode 100644 index c62a610fa..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos.py +++ /dev/null @@ -1,148 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_ddos.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot DDoS Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_honeypot_ddos-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.1', - 'destination.port' : 88, - 'destination.reverse_dns' : 'node01.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '121.12.110.28/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 61234, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.2', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node02.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '180.97.183.94/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 61234, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.3', - 'destination.reverse_dns' : 'node03.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk7', - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '104.237.138.135/32', - 'extra.duration' : 10, - 'extra.family' : 'mirai', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 6379, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py deleted file mode 100644 index f379d1c88..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_ddos_target.py +++ /dev/null @@ -1,150 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_ddos_target.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot DDoS Target Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event4_honeypot_ddos_target-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.1', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node01.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk0', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '115.238.198.85/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 61234, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.2', - 'destination.port' : 43437, - 'destination.reverse_dns' : 'node02.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk0', - 'extra.destination.sector' : 'Information', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '52.184.50.250/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 61234, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'honeypot-ddos-target', - 'classification.taxonomy' : 'availability', - 'classification.type' : 'ddos', - 'destination.asn' : 65534, - 'destination.geolocation.cc' : 'ZZ', - 'destination.geolocation.city' : 'City', - 'destination.geolocation.region' : 'Region', - 'destination.ip' : '172.16.0.3', - 'destination.port' : 80, - 'destination.reverse_dns' : 'node03.example.net', - 'extra.application' : 'mirai', - 'extra.attack' : 'atk10', - 'extra.dst_netmask' : '32', - 'extra.dst_network' : '211.99.102.216/32', - 'extra.duration' : 30, - 'extra.family' : 'mirai', - 'extra.packet_length' : 1440, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'mirai', - 'feed.name' : 'Honeypot DDoS Target Events', - 'malware.name' : 'ddos', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 61234, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py deleted file mode 100644 index bcf268ba7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_honeypot_http_scan.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_honeypot_http_scan.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot-HTTP-Scan', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-08-01T12:00:00+00:00", - "extra.file_name": "2021-08-01-event4_honeypot_http_scan.csv", - } - -EVENTS = [{'__type': 'Event', - 'feed.name': 'Honeypot-HTTP-Scan', - 'classification.identifier': 'honeypot-http-scan', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'destination.asn': 5678, - 'destination.geolocation.cc': 'UK', - 'destination.geolocation.city': 'MAIDENHEAD', - 'destination.geolocation.region': 'WINDSOR AND MAIDENHEAD', - 'destination.ip': '109.87.65.43', - 'destination.port': 80, - 'extra.http_url': '/js/ueditor/wwwroot/way-board.cgi', - 'extra.destination.naics': 518210, - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'extra.public_source': 'CAPRICA-EU', - 'extra.request_raw': 'R0VUIC9qcy91ZWRpdG9yL3d3d3Jvb3Qvd2F5LWJvYXJkLmNnaSBIVFRQLzEuMHJuQWNjZXB0OiB0ZXh0L2h0bWwsYXBwbGljYXRpb24veGh0bWwreG1sLGFwcGxpY2F0aW9uL3htbDtxPTAuOSwqLyo7cT0wLjhybkFjY2VwdC1FbmNvZGluZzogZ3ppcCwgZGVmbGF0ZXJuQWNjZXB0LUxhbmd1YWdlOiBlbi1VUyxlbjtxPTAuNXJuQ29ubmVjdGlvbjogY2xvc2VybkRudDogMXJuSG9zdDogMTA5Ljg3LjY1LjQzcm5PcmlnaW46IGh0dHA6Ly8xMDkuODcuNjUuNDNyblJlZmVyZXI6IGh0dHA6Ly8xMDkuODcuNjUuNDMvcm5Vc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA2LjE7IFdPVzY0KSBBcHBsZVdlYktpdC81MzcuMzYgKEtIVE1MLCBsaWtlIEdlY2tvKSBDaHJvbWUvNTMuMC4yNzg1LjEwNCBTYWZhcmkvNTM3LjM2IENvcmUvMS41My4zMDg0LjQwMCBRUUJyb3dzZXIvOS42LjExMzQ2LjQwMA==', - 'extra.source.naics': 518210, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.version': '3.1.3-dev', - 'malware.name': 'http-scan', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 1234, - 'source.geolocation.cc': 'EE', - 'source.geolocation.city': 'TALLINN', - 'source.geolocation.region': 'HARJUMAA', - 'source.ip': '191.23.45.67', - 'source.port': 36455, - 'source.reverse_dns': '191-23-45-67-host.example.com', - 'time.observation': '2021-08-01T12:00:00+00:00', - 'time.source': '2021-08-01T00:24:08+00:00'}, - {'__type': 'Event', - 'feed.name': 'Honeypot-HTTP-Scan', - 'classification.identifier': 'honeypot-http-scan', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'destination.asn': 23456, - 'destination.geolocation.cc': 'UA', - 'destination.geolocation.city': 'KHARKIV', - 'destination.geolocation.region': "KHARKIVS'KA OBLAST'", - 'destination.ip': '82.41.20.10', - 'destination.port': 8080, - 'extra.http_url': '/', - 'extra.method': 'GET', - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'extra.public_source': 'CAPRICA-EU', - 'extra.request_raw': 'R0VUIC8gSFRUUC8xLjENCkhvc3Q6IDgyLjQxLjIwLjEwOjgwODANCkFjY2VwdDogdGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksKi8qO3E9MC44DQpBY2NlcHQtRW5jb2Rpbmc6IGRlZmxhdGUsIGd6aXAsIGlkZW50aXR5DQpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTO3E9MC42LGVuO3E9MC40DQpVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA1LjE7IHJ2OjkuMC4xKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzkuMC4xDQoNCg==', - 'extra.url_scheme': 'http', - 'extra.user_agent': 'Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1', - 'malware.name': 'http-scan', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 12345, - 'source.geolocation.cc': 'EE', - 'source.geolocation.city': 'TALLINN', - 'source.geolocation.region': 'HARJUMAA', - 'source.ip': '45.67.89.123', - 'source.port': 58610, - 'time.observation': '2021-08-01T12:00:00+00:00', - 'time.source': '2021-08-01T05:21:59+00:00'}, -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py deleted file mode 100644 index d21fb10c5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_ip_spoofer.py +++ /dev/null @@ -1,182 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/event4_ip_spoofer.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "CAIDA", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-event4_ip_spoofer.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T00:42:59+00:00", - "source.ip": "98.191.250.0", - - "source.asn": 22898, - - "source.geolocation.cc": "US", - "source.geolocation.region": "OKLAHOMA", - "source.geolocation.city": "OKLAHOMA CITY", - "source.network": "98.191.250.0/24", - "source.reverse_dns": 'ip-98.191.250.0.atlinkservices.com', - "extra.routedspoof": "received", - "extra.session": '1112907', - "extra.nat": True, - "extra.public_source": "caida", - "extra.source.naics": 517311, - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T01:36:22+00:00", - "source.ip": "191.7.16.0", - - "source.asn": 262485, - - "source.geolocation.cc": "BR", - "source.geolocation.region": "RIO DE JANEIRO", - "source.geolocation.city": "NOVA IGUACU", - "source.network": "191.7.16.0/24", - "extra.routedspoof": "received", - "extra.session": '1112914', - "extra.nat": False, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T02:10:58+00:00", - "source.ip": "202.53.160.0", - - "source.asn": 23923, - - "source.geolocation.cc": "BD", - "source.geolocation.region": "DHAKA", - "source.geolocation.city": "DHAKA", - "source.network": "202.53.160.0/24", - "extra.routedspoof": "received", - "extra.session": '1112931', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T03:41:51+00:00", - "source.ip": "87.121.75.0", - - "source.asn": 134697, - - "source.geolocation.cc": "AU", - "source.geolocation.region": "QUEENSLAND", - "source.geolocation.city": "BRISBANE", - "source.network": "87.121.75.0/24", - "extra.routedspoof": "received", - "extra.session": '1112953', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "CAIDA", - "time.source": "2021-03-28T06:07:17+00:00", - "source.ip": "189.201.194.0", - - "source.asn": 262944, - - "source.network": "189.201.194.0/24", - "source.geolocation.cc": 'MX', - "source.geolocation.city": 'SALTILLO', - "source.geolocation.region": 'COAHUILA', - "source.reverse_dns": 'ip-189-201-194-0.slw.spectro.mx', - "extra.routedspoof": "received", - "extra.session": '1113015', - "extra.nat": True, - "extra.public_source": "caida", - "extra.version": 'ipv4', - "protocol.transport": 'tcp', - "extra.infection": 'ip-spoofer', - - "classification.identifier": "ip-spoofer", - "classification.taxonomy": "fraud", - "classification.type": "masquerade", - - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py deleted file mode 100644 index f008fd18e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole.py +++ /dev/null @@ -1,135 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_microsoft_sinkhole.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Microsoft Sinkhole", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_microsoft_sinkhole.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'HK', - 'destination.geolocation.city': 'HONG KONG', - 'destination.geolocation.region': 'HONG KONG', - 'destination.ip': '168.63.134.179', - 'destination.port': 16464, - 'extra.destination.naics': 334111, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 7303, - 'source.geolocation.cc': 'AR', - 'source.geolocation.city': 'CASEROS', - 'source.geolocation.region': 'BUENOS AIRES', - 'source.ip': '190.229.1.2', - 'source.port': 52955, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - {'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'IE', - 'destination.geolocation.city': 'DUBLIN', - 'destination.geolocation.region': 'DUBLIN', - 'destination.ip': '52.169.3.4', - 'destination.port': 16464, - 'extra.destination.naics': 334111, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 5769, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'LAVAL', - 'source.geolocation.region': 'QUEBEC', - 'source.ip': '96.20.3.4', - 'source.port': 16464, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - }, - {'__type': 'Event', - 'classification.identifier': 'b68-zeroaccess-2-32bit', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'HK', - 'destination.geolocation.city': 'HONG KONG', - 'destination.geolocation.region': 'HONG KONG', - 'destination.ip': '168.63.134.179', - 'destination.port': 16464, - 'extra.tag': 'b68-zeroaccess-2-32bit', - 'extra.infection': 'b68-zeroaccess-2-32bit', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'feed.name': 'ShadowServer Microsoft Sinkhole', - 'malware.name': 'zeroaccess', - 'protocol.transport': 'tcp', - 'source.asn': 8151, - 'source.geolocation.cc': 'MX', - 'source.geolocation.city': 'MEXICO CITY', - 'source.geolocation.region': "CIUDAD DE MEXICO", - 'source.ip': '187.222.5.6', - 'source.port': 55049, - 'time.source': '2021-06-07T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py deleted file mode 100644 index 2f8c3d8e2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_microsoft_sinkhole_http.py +++ /dev/null @@ -1,202 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_microsoft_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'HTTP Microsoft Sinkhole IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_microsoft_sinkhole_http.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.infection': 'necurs', - 'extra.tag': 'necurs', - 'protocol.application': 'http', - 'malware.name': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 8386, - 'source.geolocation.cc': 'TR', - 'source.geolocation.city': 'KEPEZ', - 'source.geolocation.region': 'ANTALYA', - 'source.ip': '31.206.1.2', - 'source.port': 49245, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'caphaw', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.fqdn': '3fo8jrthz3y.rgk.cc', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'REDMOND', - 'destination.geolocation.region': 'WASHINGTON', - 'destination.ip': '204.95.99.204', - 'destination.port': 443, - 'destination.url': 'http://3fo8jrthz3y.rgk.cc/index.php', - 'protocol.application': 'http', - 'extra.infection': 'caphaw', - 'extra.tag': 'caphaw', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.http_agent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.6103)', - 'extra.http_referer': 'null', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517312, - 'malware.name': 'caphaw', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 28573, - 'source.geolocation.cc': 'BR', - 'source.geolocation.city': 'SAO PAULO', - 'source.geolocation.region': 'SAO PAULO', - 'source.ip': '177.140.3.4', - 'source.port': 35919, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'protocol.application': 'http', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'malware.name': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn': 132199, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'MANDAUE', - 'source.geolocation.region': 'CEBU', - 'source.ip': '180.190.5.6', - 'source.port': 49264, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.ip': '40.121.206.97', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/news/stream.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'malware.name': 'necurs', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'source.asn': 37129, - 'source.geolocation.cc': 'KE', - 'source.geolocation.city': 'NAIROBI', - 'source.geolocation.region': 'NAIROBI CITY', - 'source.ip': '197.157.7.8', - 'source.port': 55307, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Microsoft Sinkhole IPv4', - 'classification.identifier': 'necurs', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 8075, - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'ASHBURN', - 'destination.geolocation.region': 'VIRGINIA', - 'destination.ip': '40.121.206.97', - 'destination.port': 80, - 'destination.url': 'http://40.121.206.97/locator.php', - 'extra.destination.naics': 334111, - 'extra.destination.sector': 'Information', - 'extra.public_source': 'MSDCU', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'necurs', - 'protocol.application': 'http', - 'extra.tag': 'necurs', - 'extra.infection': 'necurs', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'source.asn': 812, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'OTTAWA', - 'source.geolocation.region': 'ONTARIO', - 'source.ip': '174.114.9.10', - 'source.port': 59000, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-06-07T00:00:01+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py deleted file mode 100644 index 2bb8aa698..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole.py +++ /dev/null @@ -1,73 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Sinkhole", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole.csv", - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'victorygate.b', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 28753, - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.ip': '178.162.1.2', - 'destination.port': 4455, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.public_source': 'eset', - 'feed.name': 'ShadowServer Sinkhole', - 'malware.name': 'victorygate.b', - 'extra.infection': 'victorygate.b', - 'protocol.transport': 'tcp', - 'source.asn': 12252, - 'source.geolocation.cc': 'PE', - 'source.geolocation.city': 'LIMA', - 'source.geolocation.region': 'METROPOLITANA DE LIMA', - 'source.ip': '190.113.1.2', - 'source.port': 17409, - 'time.source': '2021-03-04T00:00:00+00:00', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py deleted file mode 100644 index cf3bdb162..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_dns.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_sinkhole_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "Sinkhole DNS", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole_dns-test-geo.csv", - } - -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'msexchange', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'calypso', - 'extra.dns_query' : 'YolkIsh.COM', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'extra.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'calypso', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 8220, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'FRANKFURT AM MAIN', - 'source.geolocation.region' : 'HESSEN', - 'source.ip' : '217.110.0.0', - 'source.port' : 29614, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'rat', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'orcus', - 'extra.dns_query' : 'verble.rocks', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'orcus', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 40934, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '209.66.0.0', - 'source.port' : 46189, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sinkholedns', - 'extra.tag' : 'msexchange', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.count' : 1, - 'extra.infection' : 'calypso', - 'extra.dns_query' : 'RAwFuNS.COM', - 'extra.dns_query_type' : 'A', - 'extra.naics' : 518210, - 'extra.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Sinkhole DNS', - 'malware.name' : 'calypso', - 'protocol.application' : 'dns', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 8220, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'FRANKFURT AM MAIN', - 'source.geolocation.region' : 'HESSEN', - 'source.ip' : '217.110.0.0', - 'source.port' : 3590, - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2022-01-06T00:00:02+00:00' -} -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py deleted file mode 100644 index 60cd6b6ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http.py +++ /dev/null @@ -1,189 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'HTTP Sinkhole IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_sinkhole_http.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.1.2', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 134707, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'DEL PILAR', - 'source.geolocation.region': 'NUEVA ECIJA', - 'source.ip': '103.196.1.2', - 'source.port': 60902, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.3.4', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 8708, - 'source.geolocation.cc': 'RO', - 'source.geolocation.city': 'CONSTANTA', - 'source.geolocation.region': 'CONSTANTA', - 'source.ip': '5.14.3.4', - 'source.port': 55002, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'disorderstatus.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.5.6', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.source.naics': 517311, - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn': 9299, - 'source.geolocation.cc': 'PH', - 'source.geolocation.city': 'CEBU', - 'source.geolocation.region': 'CEBU', - 'source.ip': '49.145.5.6', - 'source.port': 31350, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.ip': '184.105.7.8', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.source.naics': 517311, - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'source.asn': 8048, - 'source.geolocation.cc': 'VE', - 'source.geolocation.city': 'VALENCIA', - 'source.geolocation.region': 'CARABOBO', - 'source.ip': '200.44.7.8', - 'source.port': 28063, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}, - {'__type': 'Event', - 'feed.name': 'HTTP Sinkhole IPv4', - 'classification.identifier': 'avalanche-andromeda', - 'extra.tag': 'avalanche-andromeda', - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'destination.asn': 6939, - 'destination.fqdn': 'differentia.ru', - 'destination.geolocation.cc': 'US', - 'destination.geolocation.city': 'FREMONT', - 'destination.geolocation.region': 'CALIFORNIA', - 'destination.ip': '184.105.9.10', - 'destination.port': 80, - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'andromeda', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'source.asn': 17072, - 'source.geolocation.cc': 'MX', - 'source.geolocation.city': 'JUAREZ', - 'source.geolocation.region': 'CHIHUAHUA', - 'source.ip': '187.189.9.10', - 'source.port': 45335, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:00+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py b/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py deleted file mode 100644 index b1ccacd31..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event4_sinkhole_http_referer.py +++ /dev/null @@ -1,213 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/event4_sinkhole_http_referer.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-03-05T00:00:00+00:00", - "extra.file_name": "2021-03-04-event4_sinkhole_http_referer.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'kovter', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': '12106.mobapptrack.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.ip': '85.17.31.82', - 'destination.port': 80, - 'destination.url': 'http://12106.mobapptrack.com/favicon.ico', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.event_id': '1614816002', - 'malware.name': 'kovter', - 'extra.http_referer': 'http://12106.mobapptrack.com/click/redirect?feed_id=12106&sub_id=7&q=8A5491983C8FBE7743E2D2C36E45EBC4-18307118D2626C9BD756B3F09D14BB910E381EE4', - 'extra.http_referer_asn': 28753, - 'extra.http_referer_city': 'FRANKFURT AM MAIN', - 'extra.http_referer_geo': 'DE', - 'extra.http_referer_hostname': '12106.mobapptrack.com', - 'extra.http_referer_ip': '178.162.203.211', - 'extra.http_referer_naics': 518210, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'HESSEN', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2021-03-04T00:00:02+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 28753, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/animalally.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '178.162.1.2', - 'extra.event_id': '1614816011', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://x.noizm.com/jump.php?u=http://freescanonline.com/animalally.com', - 'extra.http_referer_asn': 9370, - 'extra.http_referer_city': 'OSAKA', - 'extra.http_referer_geo': 'JP', - 'extra.http_referer_hostname': 'x.noizm.com', - 'extra.http_referer_naics': 518210, - 'extra.http_referer_port': 80, - 'extra.http_referer_ip': '59.106.1.2', - 'extra.http_referer_region': 'OSAKA', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.source': '2021-03-04T00:00:11+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'kovter', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 28753, - 'destination.fqdn': 'rxrtb.bid', - 'destination.geolocation.cc': 'DE', - 'destination.geolocation.city': 'FRANKFURT AM MAIN', - 'destination.geolocation.region': 'HESSEN', - 'destination.port': 80, - 'destination.url': 'http://rxrtb.bid/getjs?r=0.6393021999392658', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '178.162.1.2', - 'extra.event_id': '1614816012', - 'malware.name': 'kovter', - 'extra.http_referer': 'http://x.blogspot.com/', - 'extra.http_referer_ip': '142.250.3.4', - 'extra.http_referer_asn': 15169, - 'extra.http_referer_city': 'MOUNTAIN VIEW', - 'extra.http_referer_geo': 'US', - 'extra.http_referer_hostname': 'x.blogspot.com', - 'extra.http_referer_naics': 519130, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'CALIFORNIA', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'time.source': '2021-03-04T00:00:12+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.ip': '5.79.71.225', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/personalationmall.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'extra.event_id': '1614816013', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://www.example.com/teams/default.asp?u=EKL&t=c&s=lacrosse&p=remote&url=http://freescanonline.com/personalationmall.com', - 'extra.http_referer_asn': 14618, - 'extra.http_referer_city': 'ASHBURN', - 'extra.http_referer_geo': 'US', - 'extra.http_referer_hostname': 'www.example.com', - 'extra.http_referer_ip': '34.232.5.6', - 'extra.http_referer_naics': 454110, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'VIRGINIA', - 'extra.http_referer_sector': 'Retail Trade', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[4]])), - 'time.source': '2021-03-04T00:00:13+00:00'}, - {'__type': 'Event', - 'classification.identifier': 'sinkhole-http-referer', - 'extra.tag': 'sunburst', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'destination.asn': 60781, - 'destination.fqdn': 'freescanonline.com', - 'destination.geolocation.cc': 'NL', - 'destination.geolocation.city': 'AMSTERDAM', - 'destination.geolocation.region': 'NOORD-HOLLAND', - 'destination.port': 80, - 'destination.url': 'http://freescanonline.com/raftcomply.com', - 'extra.destination.naics': 518210, - 'extra.destination.sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'destination.ip': '5.79.1.2', - 'extra.event_id': '1614816086', - 'malware.name': 'sunburst', - 'extra.http_referer': 'http://x.communes.jp/?url=http://freescanonline.com/raftcomply.com', - 'extra.http_referer_asn': 2516, - 'extra.http_referer_city': 'SAPPORO', - 'extra.http_referer_geo': 'JP', - 'extra.http_referer_hostname': 'x.communes.jp', - 'extra.http_referer_ip': '210.172.7.8', - 'extra.http_referer_naics': 517312, - 'extra.http_referer_port': 80, - 'extra.http_referer_region': 'HOKKAIDO', - 'extra.http_referer_sector': 'Communications, Service Provider, and Hosting ' - 'Service', - 'protocol.transport': 'tcp', - 'feed.name': 'Sinkhole-Events-HTTP-Referer IPv4', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[5]])), - 'time.source': '2021-03-04T00:01:26+00:00'}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py b/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py deleted file mode 100644 index d6ff35dc1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_event6_sinkhole_http.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event6_sinkhole_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "Sinkhole-Events-HTTP IPv6", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-event6_sinkhole_http-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::fe', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)', - 'extra.infection' : 'boaxxe', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 7713, - 'source.geolocation.cc' : 'ID', - 'source.geolocation.city' : 'JAKARTA', - 'source.geolocation.region' : 'JAKARTA RAYA', - 'source.ip' : '2001:448a:1082:4d9b:7491:bf9e:3d5f:a634', - 'source.port' : 49431, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T09:14:19+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::ef', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)', - 'extra.infection' : 'boaxxe', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 7713, - 'source.geolocation.cc' : 'ID', - 'source.geolocation.city' : 'JAKARTA', - 'source.geolocation.region' : 'JAKARTA RAYA', - 'source.ip' : '2001:448a:1082:4d9b:7491:bf9e:3d5f:a634', - 'source.port' : 49460, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T09:15:10+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : '3ve', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'infected-system', - 'destination.asn' : 6939, - 'destination.fqdn' : 'devps.net', - 'destination.geolocation.cc' : 'US', - 'destination.geolocation.city' : 'FREMONT', - 'destination.geolocation.region' : 'CALIFORNIA', - 'destination.ip' : '2001:470:1:332::fe', - 'destination.port' : 80, - 'destination.url' : 'http://devps.net/WMoUNCvuKGzdqSCeQcadP1%2f0B%2f3bzpOmyKBU85Z25HVOhvDQUPFl%2fk8uOcLewS%2b1BsuHXalRAOIgGOYYs2igj6UX8FkdCAmDewWPvfDhPD45nwd2tx1lLf2IoIfuOtIpGR6bN5Q6hGpSBgfERqCa0ImHcwfcZ2EdO%2fWvg7R8H6SLcTiuUC0I4pzvlWt1CRLgLdIEU1hZ0nnFHIHhchb6D7ITEgBQ2chQDxy5TJMrGjm4Dac6dKl%2ft5uYhRhSjAHkLLtgrJjsqVtVbelTAkt5kdcqLlO09m1SH%2fvtAb%2fOvR2DbhBss7%2f64DG7g6cAnghNA6JrFn1uW7sw%2bnKH8koKQwzUjdSsbrQAvmg4r0KDDW8Diq64gfDzxFWkzCLOYifc%2fwlinXPCl7aJiNCoieDC1U98RNQg%2f5td4SZmJnDQ2%2f96CPbFeSpCez5WD1rCjrxLj1h2cqzIgkydEWACceWP9ztxc4QaObzEcgOGxbRckWC7H2aaLeT8jaYEYdKi1pwEKChSL3YdEt4ZIb2IFrWwzNaXEpQzFXf07f902OEdI9vVA1ZdEOBPG6rAIkzMdebfprfVyhKEWtrCd3Skg3COUFtRQks5jzG1nv4sVGijTfSgyn6xE9Taka668Nycik6nmHy8Huj3oC01j3tee%2f1Z3eI6tV7lgM5d3uFJ84slRGHUCwMfVozOGmZRwNo%2fz%2bA', - 'extra.destination.naics' : 518210, - 'extra.destination.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.http_agent' : 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko', - 'extra.infection' : 'boaxxe', - 'extra.source.naics' : 517311, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : '3ve', - 'feed.name' : 'Sinkhole-Events-HTTP IPv6', - 'malware.name' : 'boaxxe', - 'protocol.application' : 'http', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 11427, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'GARLAND', - 'source.geolocation.region' : 'TEXAS', - 'source.ip' : '2603:8080:b20a:dc00:f06e:8304:71f6:27e2', - 'source.port' : 62932, - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2022-03-02T14:15:10+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py b/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py deleted file mode 100644 index c376a73fb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_brute_force.py +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_honeypot_brute_force.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Honeypot-Brute-Force-Events', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_brute_force.csv" - } -EVENTS = [{'__type': 'Event', - 'classification.identifier': 'ssh', - 'classification.taxonomy': 'intrusion-attempts', - 'classification.type': 'brute-force', - 'extra.client_version': "b'SSH-2.0-Go'", - 'destination.asn': 26832, - 'destination.geolocation.cc': 'CA', - 'destination.geolocation.city': 'MONTREAL', - 'destination.geolocation.region': 'QUEBEC', - 'destination.ip': '162.250.1.2', - 'destination.port': 22, - 'extra.application': 'ssh', - 'extra.end_time': '2021-03-27T00:00:01.710968+00:00', - 'extra.public_source': 'CAPRICA-EU', - 'extra.start_time': '2021-03-27T00:00:00.521730+00:00', - 'malware.name': 'ssh-brute-force', - 'feed.name': 'Honeypot-Brute-Force-Events', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 209588, - 'source.geolocation.cc': 'NL', - 'source.geolocation.city': 'AMSTERDAM', - 'source.geolocation.region': 'NOORD-HOLLAND', - 'source.ip': '141.98.1.2', - 'source.port': 30123, - 'time.source': '2021-03-27T00:00:00+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py b/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py deleted file mode 100644 index e95e59dcb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_honeypot_ddos_amp.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/event4_honeypot_ddos_amp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Amplification DDoS Victim', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-event4_honeypot_ddos_amp.csv" - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Amplification DDoS Victim', - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation': '2019-01-01T00:00:00+00:00', - 'time.source': '2021-03-28T00:00:02+00:00', - 'source.ip': '107.141.1.2', - 'destination.port': 389, - 'source.reverse_dns': '192-0-2-10.example.net', - 'source.asn': 7018, - 'source.geolocation.cc': 'US', - 'source.geolocation.region': 'VISALIA', - 'source.geolocation.city': 'VISALIA', - 'source.geolocation.region': 'CALIFORNIA', - 'extra.end_time': '2021-03-28T00:20:22+00:00', - 'extra.public_source': 'CISPA', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'ddos-amplification', - 'source.reverse_dns': '107-141-x-x.lightspeed.frsnca.sbcglobal.net', - }, - {'__type': 'Event', - 'feed.name': 'Amplification DDoS Victim', - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.observation': '2019-01-01T00:00:00+00:00', - 'time.source': '2021-03-28T00:00:02+00:00', - 'source.ip': '74.59.3.4', - 'destination.port': 389, - 'source.reverse_dns': 'modemcablex-x-59-74.mc.videotron.ca', - 'source.asn': 5769, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'CHICOUTIMI', - 'source.geolocation.region': 'QUEBEC', - 'extra.end_time': '2021-03-28T00:13:50+00:00', - 'extra.public_source': 'CISPA', - 'extra.source.naics': 517311, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'malware.name': 'ddos-amplification', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py b/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py deleted file mode 100644 index b19b200b5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_malware_url.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/malware_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Malware URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-malware_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'source.url' : 'http://41.86.0.0:50008/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.hash.sha256' : '12013662c71da69de977c04cd7021f13a70cf7bed4ca6c82acbc100464d4b0ef', - 'malware.name' : 'cve-2016-10372', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37203, - 'source.geolocation.cc' : 'LR', - 'source.geolocation.city' : 'MONROVIA', - 'source.geolocation.region' : 'MONTSERRADO', - 'source.ip' : '41.86.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:02:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'extra.source.naics' : 517311, - 'source.url' : 'http://42.225.0.0:38173/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.name' : 'cve-2018-10562', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 4837, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'ZHUMADIAN', - 'source.geolocation.region' : 'HENAN SHENG', - 'source.ip' : '42.225.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:03:14+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'malware-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'extra.application' : 'http', - 'extra.source.naics' : 517311, - 'source.url' : 'http://211.52.0.0:53029/Mozi.m', - 'feed.name' : 'Malware URL', - 'malware.name' : 'cve-2018-10562', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 4766, - 'source.geolocation.cc' : 'KR', - 'source.geolocation.city' : 'SAGOK-MYEON', - 'source.geolocation.region' : 'CHUNGCHEONGNAM-DO', - 'source.ip' : '211.52.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-07T00:10:26+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py b/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py deleted file mode 100644 index 0783372f9..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_phish_url.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/phish_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Phish URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-phish_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'priceless-pare.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.naics' : 518210, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'source.url' : 'https://priceless-pare.example.net/Postal-/acec6/', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'BUFFALO', - 'source.geolocation.region' : 'NEW YORK', - 'source.ip' : '172.245.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'mailyahooattt.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.sector' : 'Professional, Scientific, and Technical Services', - 'source.url' : 'https://mailyahooattt.example.net/', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'SAN FRANCISCO', - 'source.geolocation.region' : 'CALIFORNIA', - 'source.ip' : '199.34.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'phish-url', - 'classification.taxonomy' : 'fraud', - 'classification.type' : 'phishing', - 'source.fqdn' : 'www.example.net', - 'extra.source' : 'openphish.com', - 'extra.source.naics' : 519130, - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'source.url' : 'https://www.example.net/viewer/vbid-730ec2b1-omsttuer', - 'feed.name' : 'Phish URL', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'DRAPER', - 'source.geolocation.region' : 'UTAH', - 'source.ip' : '216.58.0.0', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-02-01T08:00:07+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py b/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py deleted file mode 100644 index e9f11a47c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_population_http_proxy.py +++ /dev/null @@ -1,130 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/population_http_proxy.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible HTTP Proxy', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-population_http_proxy-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 3741, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"Squid proxy-caching web ' - 'server\\"\\""', - 'extra.server': 'squid/4.10', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3128, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 3833, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"00:23:24:43:1c:34\\"\\""', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3128, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.connection': 'keep-alive', - 'extra.content_length': 179, - 'extra.content_type': 'text/html;charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 407, - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.http_reason': 'Proxy Authentication Required', - 'extra.proxy_authenticate': 'Basic realm=\\\\"Proxy\\"\\""', - 'feed.name': 'Accessible HTTP Proxy', - 'malware.name': 'http-connect-proxy-closed', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3128, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py deleted file mode 100644 index c5da82346..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_conn.py +++ /dev/null @@ -1,99 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_conn.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox Connections', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_conn-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'time.windows.com', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'b575ce6dcce6502a8431db5610135c25', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 8075, - 'source.geolocation.cc' : 'US', - 'source.ip' : '40.119.6.228', - 'source.port' : 123, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -}, - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'c0d947f9a8685b0d9f3efdba966389c2', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 3356, - 'source.geolocation.cc' : 'US', - 'source.ip' : '8.252.70.126', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-conn', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'feed.name' : 'Sandbox Connections', - 'malware.hash.md5' : 'c0d947f9a8685b0d9f3efdba966389c2', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 8075, - 'source.geolocation.cc' : 'US', - 'source.ip' : '52.109.8.22', - 'source.port' : 443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:03+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py deleted file mode 100644 index 70cf1eee5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_dns.py +++ /dev/null @@ -1,95 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox DNS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_dns-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'time.windows.com', - 'extra.response' : '40.119.6.228', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : 'b575ce6dcce6502a8431db5610135c25', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:02+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'time.windows.com', - 'extra.response' : '40.119.6.228', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : '807679198a39c80d3ca07e60fd51b581', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:08+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-dns', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.request' : 'client-office365-tas.msedge.net', - 'extra.response' : '13.107.5.88', - 'extra.dns_query_type' : 'A', - 'feed.name' : 'Sandbox DNS', - 'malware.hash.md5' : 'd97e973b9bf073bd3a217425259cea26', - 'protocol.application' : 'dns', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:00:20+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py b/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py deleted file mode 100644 index 91b0154b8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_sandbox_url.py +++ /dev/null @@ -1,104 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/sandbox_url.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Sandbox URL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-sandbox_url-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'www.msftncsi.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://www.msftncsi.com/ncsi.txt', - 'extra.user_agent' : 'Microsoft NCSI', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : '37514b54e679a5313334e830ad780ec7', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'US', - 'source.ip' : '23.196.47.89', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'www.download.windowsupdate.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://www.download.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab', - 'extra.user_agent' : 'Microsoft-CryptoAPI/6.1', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : '37514b54e679a5313334e830ad780ec7', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 15133, - 'source.geolocation.cc' : 'US', - 'source.ip' : '72.21.81.240', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:28+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'sandbox-url', - 'classification.taxonomy' : 'malicious-code', - 'classification.type' : 'malware-distribution', - 'destination.fqdn' : 'crl.microsoft.com', - 'extra.http_request_method' : 'GET', - 'destination.url' : 'http://crl.microsoft.com/pki/crl/products/MicTimStaPCA_2010-07-01.crl', - 'extra.user_agent' : 'Microsoft-CryptoAPI/6.1', - 'feed.name' : 'Sandbox URL', - 'malware.hash.md5' : 'e97ea2820c0d79f3f3ca241d4dcd1060', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'US', - 'source.ip' : '23.56.4.57', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:08:24+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py deleted file mode 100644 index 6bc6e6146..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_adb.py +++ /dev/null @@ -1,98 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_adb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible ADB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_adb-test-test.csv", - - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible ADB', - 'time.observation': '2018-07-30T00:00:00+00:00', - 'time.source': '2018-07-26T02:07:16+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-adb', - 'protocol.application': 'adb', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 3462, - 'source.geolocation.cc': 'TW', - 'source.geolocation.city': 'TAOYUAN CITY', - 'source.geolocation.region': 'TAOYUAN COUNTY', - 'source.ip': '36.239.124.210', - 'source.port': 5555, - 'extra.name': 'hlteuc', - 'extra.model': 'SAMSUNG-SM-N900A', - 'extra.device': 'hlteatt', - 'extra.tag': 'adb', - 'extra.naics': 518210, - 'extra.sic': 737415, - 'source.reverse_dns': '36-239-124-210.dynamic-ip.hinet.net', - }, - {'__type': 'Event', - 'feed.name': 'Accessible ADB', - 'time.observation': '2018-07-30T00:00:00+00:00', - 'time.source': '2018-07-26T02:07:16+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-adb', - 'protocol.application': 'adb', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 3462, - 'source.geolocation.cc': 'TW', - 'source.geolocation.city': 'TAIPEI', - 'source.geolocation.region': 'TAIPEI CITY', - 'source.ip': '36.236.108.107', - 'source.port': 5555, - 'extra.name': 'marlin', - 'extra.model': 'Pixel XL', - 'extra.device': 'marlin', - 'extra.features': 'cmd,shell_v2', - 'extra.naics': 518210, - 'extra.sic': 737415, - 'extra.tag': 'adb', - 'source.reverse_dns': '36-236-108-107.dynamic-ip.hinet.net', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py deleted file mode 100644 index cc30b1e4c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_afp.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_afp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible AFP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_afp-test-test.csv", - - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible AFP', - "classification.identifier": "accessible-afp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.afp_versions": "AFP3.3,AFP3.2,AFP3.1", - "extra.flags": "SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient", - "extra.machine_type": "TimeCapsule8,119", - "extra.naics": 517311, - "extra.network_address": "198.33.24.165:548,10.0.1.1:548,fe80:0008:0000:0000:6e70:9fff:fed4::548,fe80:0009:0000:0000:6e70:9fff:fed4::548,179.24.24.165 (DNS address),", - "extra.server_name": "airport-time-capsule-de-jack", - "extra.signature": "4338364e37364442463948350069672d", - "extra.tag": "afp", - "extra.uams": "DHCAST128,DHX2,SRP,Recon1", - "extra.utf8_servername": "AirPort Time Capsule de jack", - "protocol.application": "afp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 6057, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.13.34.22", - "source.port": 548, - "source.reverse_dns": "host.local", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T05:05:53+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Accessible AFP', - "classification.identifier": "accessible-afp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.afp_versions": "AFP3.3,AFP3.2,AFP3.1", - "extra.flags": "SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient", - "extra.machine_type": "TimeCapsule8,119", - "extra.naics": 517311, - "extra.network_address": "0.0.0.1:548,10.0.1.1:548,198.33.42.1:548,fe80:000b:0000:0000:dea4:caff:feba::548,fe80:000c:0000:0000:dea4:caff:feba::548,fe80:000d:0000:0000:4c7d:ffff:fec7::548,0.0.0.1 (DNS address),", - "extra.server_name": "time-capsule-del-jack", - "extra.signature": "433836544b303147463948360069672d", - "extra.tag": "afp", - "extra.uams": "DHCAST128,DHX2,SRP,Recon1", - "extra.utf8_servername": "Time Capsule del Jack", - "protocol.application": "afp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 6057, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.40.27.212", - "source.port": 548, - "source.reverse_dns": "host.local", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T05:05:56+00:00" - },] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py deleted file mode 100644 index df707f30b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_amqp.py +++ /dev/null @@ -1,144 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_amqp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible AMQP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_amqp-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@iZuf63m0nnq9bwf7lhjxrkZ', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'PLAIN AMQPLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.3.5', - 'extra.naics' : 518210, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37963, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'SHANGHAI', - 'source.geolocation.region' : 'SHANGHAI SHI', - 'source.ip' : '47.103.0.0', - 'source.port' : 5672, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@mtk-breizh', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'AMQPLAIN PLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP 24.0.3', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.8.19', - 'extra.naics' : 518210, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 16276, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'SAARBRUCKEN', - 'source.geolocation.region' : 'SAARLAND', - 'source.ip' : '141.95.0.0', - 'source.port' : 5672, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'accessible-amqp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.capabilities' : 'publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to', - 'extra.class' : '10', - 'extra.cluster_name' : 'rabbit@1397a0e9629b', - 'extra.locales' : 'en_US', - 'extra.mechanisms' : 'PLAIN AMQPLAIN', - 'extra.message_length' : 509, - 'extra.method' : '10', - 'extra.platform' : 'Erlang/OTP 24.2', - 'extra.product' : 'RabbitMQ', - 'extra.product_version' : '3.9.11', - 'extra.naics' : 454110, - 'extra.tag' : 'amqp', - 'extra.version_minor' : '9', - 'feed.name' : 'Accessible AMQP', - 'protocol.application' : 'amqp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 14618, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '54.234.0.0', - 'source.port' : 5672, - 'source.reverse_dns' : 'ec2-54.234.0.0.compute-1.amazonaws.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T04:32:13+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py deleted file mode 100644 index 4d8420c3b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ard.py +++ /dev/null @@ -1,111 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Tomas Bellus -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ard.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible ARD', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-07-20T00:00:00+00:00", - "extra.file_name": "2020-01-01-scan_ard-test-test.csv", - - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': 'Macmini (radio)', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3283, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': 'biuro-rip-org-pl', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3283, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 201.2, - 'extra.machine_name': '127.0.0.1', - 'extra.response_size': 1006, - 'extra.tag': 'ard', - 'feed.name': 'Accessible ARD', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3283, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py deleted file mode 100644 index 3b72baa8d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_chargen.py +++ /dev/null @@ -1,110 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_chargen.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Chargen', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_chargen-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 19, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 19, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-chargen', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 74.0, - 'extra.response_size': 74, - 'extra.sector': 'Government', - 'extra.tag': 'chargen', - 'feed.name': 'Open Chargen', - 'protocol.application': 'chargen', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 19, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py deleted file mode 100644 index 46c963a79..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_cisco_smart_install.py +++ /dev/null @@ -1,82 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_cisco_smart_install.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Cisco Smart Install', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_cisco_smart_install-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible Cisco Smart Install', - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.type': 'vulnerable-system', - 'classification.taxonomy': 'vulnerable', - 'protocol.application': 'cisco-smart-install', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 8559, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '198.51.100.103', - 'source.port': 4786, - 'extra.tag': 'cisco-smart-install', - 'source.reverse_dns': '198-51-100-103.example.net', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-11-18T08:42:45+00:00'}, - {'__type': 'Event', - 'feed.name': 'Accessible Cisco Smart Install', - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.type': 'vulnerable-system', - 'classification.taxonomy': 'vulnerable', - 'protocol.application': 'cisco-smart-install', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 35609, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '198.51.100.218', - 'source.port': 4786, - 'extra.tag': 'cisco-smart-install', - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-11-18T08:47:54+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py deleted file mode 100644 index 773fc04d5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_coap.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_coap.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-CoAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-06-29T00:00:00+00:00", - "extra.file_name": "2020-06-28-scan_coap-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 2.05, - 'extra.response': ',,', - 'extra.response_size': 43, - 'extra.tag': 'coap', - 'extra.version': '2', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5683, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 5.38, - 'extra.response': ',,,,,,,,,', - 'extra.response_size': 113, - 'extra.tag': 'coap', - 'extra.version': '2', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5683, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 113.5, - 'extra.response': '`EsjAy************************************************************|CoAP ' - 'RFC 7252 ' - '|************************************************************|This ' - 'server is using the Eclipse Californium (Cf) CoAP ' - 'framework|published under EPL+EDL: ' - 'http://www.eclipse.org/californium/||(c) 2014, 2015, 2016 ' - 'Institute for Pervasive Computing, ETH Zurich and ' - 'others|************************************************************', - 'extra.response_size': 454, - 'extra.tag': 'coap', - 'extra.version': '1', - 'feed.name': 'Accessible-CoAP', - 'protocol.application': 'coap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5683, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py deleted file mode 100644 index 1bf6f321c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_couchdb.py +++ /dev/null @@ -1,128 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_couchdb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible CouchDB Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_couchdb-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '1.6.1', - 'extra.server_version' : 'CouchDB/1.6.1 (Erlang OTP/18)', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'Ubuntu 16.04', - 'extra.visible_databases' : '_replicator;_users;test;shops;god', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5984, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '3.2.1', - 'extra.features' : 'access-ready,partitioned,pluggable-storage-engines,reshard,scheduler', - 'extra.git_sha' : '244d428af', - 'extra.server_version' : 'CouchDB/3.2.1 (Erlang OTP/23)', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'The Apache Software Foundation', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5984, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-couchdb', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.couchdb_message' : 'Welcome', - 'extra.couchdb_version' : '3.2.1', - 'extra.features' : 'access-ready,partitioned,pluggable-storage-engines,reshard,scheduler', - 'extra.git_sha' : '244d428af', - 'extra.server_version' : 'CouchDB/3.2.1 (Erlang OTP/20)', - 'extra.source.sector' : 'Retail Trade', - 'extra.tag' : 'couchdb', - 'extra.vendor' : 'The Apache Software Foundation', - 'feed.name' : 'Accessible CouchDB Server', - 'protocol.application' : 'couchdb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5984, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py deleted file mode 100644 index b508b6450..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_cwmp.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_cwmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible CWMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_cwmp-test-test.csv", - - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible CWMP', - "classification.identifier": "open-cwmp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.connection": "keep-alive", - "extra.content_length": 5678, - "extra.content_type": "text/html", - "extra.date": "Wed, 04 Sep 2019 07:42:37 GMT", - "extra.http": "HTTP/1.1", - "extra.http_code": 200, - "extra.http_reason": "OK", - "extra.naics": 517311, - "extra.server": "DNVRS-Webs", - "extra.tag": "cwmp", - "protocol.application": "cwmp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.142", - "source.port": 30005, - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T10:44:55+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'Accessible CWMP', - "classification.identifier": "open-cwmp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.content_type": "text/html", - "extra.http": "HTTP/1.1", - "extra.http_code": 404, - "extra.http_reason": "Not Found", - "extra.naics": 517311, - "extra.server": "RomPager/4.07 UPnP/1.0", - "extra.tag": "cwmp", - "extra.transfer_encoding": "chunked", - "protocol.application": "cwmp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.162", - "source.port": 5678, - "source.reverse_dns": "localhost.localdomain", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2019-09-04T11:06:50+00:00" - },] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py deleted file mode 100644 index 423ebe8c5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_db2.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_db2.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Open-DB2-Discovery-Service", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_db2-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 14.9, - 'extra.db2_hostname': 'NOWAK_SERWER', - 'extra.servername': 'node01.example.com', - 'extra.size': 298, - 'extra.tag': 'db2', - 'feed.name': 'ShadowServer Open-DB2-Discovery-Service', - 'protocol.application': 'db2', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 523, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 14.9, - 'extra.db2_hostname': 'SPZOZ-DZIEWIN', - 'extra.servername': 'node02.example.com', - 'extra.size': 298, - 'extra.tag': 'db2', - 'feed.name': 'ShadowServer Open-DB2-Discovery-Service', - 'protocol.application': 'db2', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 523, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py deleted file mode 100644 index 9038a79ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ddos_middlebox.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ddos_middlebox.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DDoS Middlebox', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_ddos_middlebox-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source_port' : '49002', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 80, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.source_port' : '41200', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 80, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ddos-middlebox', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.amplification' : 2, - 'extra.bytes' : 99, - 'extra.method' : 'SYN+ACK:PSH', - 'extra.source_port' : '47492', - 'feed.name' : 'DDoS Middlebox', - 'protocol.application' : 'ddos-middlebox', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 80, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py deleted file mode 100644 index 3492f82ce..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_dns.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'DNS Open Resolvers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_dns-test-test.csv", - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'DNS Open Resolvers', - "classification.identifier": "dns-open-resolver", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.66", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.179", - "source.port": 53, - "source.reverse_dns": "198-51-100-189.example.net", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:34+00:00" -}, -{ - '__type': 'Event', - 'feed.name': 'DNS Open Resolvers', - "classification.identifier": "dns-open-resolver", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.51", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.8", - "source.port": 53, - "source.reverse_dns": "198-51-100-111.example.net", - "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:36+00:00" -},] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py deleted file mode 100644 index 31d0e4417..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_docker.py +++ /dev/null @@ -1,159 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_docker.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Docker Service', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_docker-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.37', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2018-05-09T22:18:36.000000000+00:00', - 'extra.content_type' : 'application/json; charset=UTF-8', - 'extra.date' : 'Fri, 06 May 2022 14:06:30 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : 'f150324', - 'extra.go_version' : 'go1.9.5', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-514.26.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.server' : 'Docker/18.05.0-ce (linux)', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.tag' : 'docker', - 'extra.version' : '18.05.0-ce', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 2375, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.26', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2022-03-02T15:25:43.414574467+00:00', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Fri, 06 May 2022 14:08:07 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : '7d71120/1.13.1', - 'extra.go_version' : 'go1.10.3', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-693.2.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.pkg_version' : 'docker-1.13.1-209.git7d71120.el7.centos.x86_64', - 'extra.server' : 'Docker/1.13.1 (linux)', - 'extra.tag' : 'docker', - 'extra.version' : '1.13.1', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 2375, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-docker', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.api_version' : '1.37', - 'extra.arch' : 'amd64', - 'extra.build_time' : '2018-05-09T22:18:36.000000000+00:00', - 'extra.content_type' : 'application/json; charset=UTF-8', - 'extra.date' : 'Fri, 06 May 2022 14:08:06 GMT', - 'extra.experimental' : 'false', - 'extra.git_commit' : 'f150324', - 'extra.go_version' : 'go1.9.5', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.kernel_version' : '3.10.0-514.26.2.el7.x86_64', - 'extra.min_api_version' : '1.12', - 'extra.os.name' : 'linux', - 'extra.server' : 'Docker/18.05.0-ce (linux)', - 'extra.tag' : 'docker', - 'extra.version' : '18.05.0-ce', - 'feed.name' : 'Accessible Docker Service', - 'protocol.application' : 'docker', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 2375, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py deleted file mode 100644 index 01e68db94..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_dvr_dhcpdiscover.py +++ /dev/null @@ -1,178 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dvr_dhcpdiscover.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible DVR DHCPDiscover', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_dvr_dhcpdiscover-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 794.0, - 'extra.device_model': 'BCS-TIP3401IR-E-V', - 'extra.device_serial': '6J0E022PAG35073', - 'extra.device_type': 'IPC', - 'extra.device_vendor': 'General', - 'extra.device_version': '2.800.106F004.0.R', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.1', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::1', - 'extra.ipv6_dhcp_enable': False, - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3ac4:e8ff:fe03:b3e2/64', - 'extra.mac_address': '38:c4:e8:03:b3:e2', - 'extra.machine_name': '6J0E022PAG35073', - 'extra.manufacturer': 'General', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 0, - 'extra.response_size': 794, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.video_input_channels': 1, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 37810, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 761.0, - 'extra.device_model': 'HCVR', - 'extra.device_serial': '2K0488CPAGS0ND6', - 'extra.device_type': 'HCVR', - 'extra.device_vendor': 'Private', - 'extra.device_version': '3.210.1.4', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.2', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::2', - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3eef:8cff:fe18:a507/64', - 'extra.mac_address': '3c:ef:8c:18:a5:07', - 'extra.machine_name': 'HCVR', - 'extra.manufacturer': 'Private', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 9, - 'extra.response_size': 761, - 'extra.video_input_channels': 3, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 37810, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-dvr-dhcpdiscover', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.alarm_input_channels': 0, - 'extra.alarm_output_channels': 0, - 'extra.amplification': 711.0, - 'extra.device_model': 'BCS-XVR0401-IV', - 'extra.device_serial': '5L034FAPAZA0E30', - 'extra.device_type': 'HCVR', - 'extra.device_vendor': 'General', - 'extra.device_version': '4.000.0000002.11', - 'extra.http_port': 80, - 'extra.internal_port': 37777, - 'extra.ipv4_address': '192.168.0.3', - 'extra.ipv4_dhcp_enable': False, - 'extra.ipv4_gateway': '192.168.0.240', - 'extra.ipv4_subnet_mask': '255.255.255.0', - 'extra.ipv6_address': 'fd09:4ab5:dae9:b078::3', - 'extra.ipv6_gateway': 'fd09:4ab5:dae9:b078::ff', - 'extra.ipv6_link_local': 'fe80::3ac4:e8ff:fe02:74da/64', - 'extra.mac_address': '38:c4:e8:02:74:da', - 'extra.machine_name': 'XVR', - 'extra.manufacturer': 'General', - 'extra.method': 'client.notifyDevInfo', - 'extra.remote_video_input_channels': 0, - 'extra.response_size': 711, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.video_input_channels': 4, - 'extra.video_output_channels': 0, - 'feed.name': 'Accessible DVR DHCPDiscover', - 'protocol.application': 'dvrdhcpdiscover', - 'protocol.transport': 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 37810, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py deleted file mode 100644 index 4e12a1b07..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_elasticsearch.py +++ /dev/null @@ -1,126 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_elasticsearch.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Elasticsearch', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_elasticsearch-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': '90f439ff60a3c0f497f91663701e64ccd01edbb4', - 'extra.build_snapshot': False, - 'extra.build_timestamp': '2016-07-27T10:36:52Z', - 'extra.cluster_name': 'elasticsearch', - 'extra.lucene_version': '5.5.0', - 'extra.name': 'Red Skull', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '2.3.5', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 9200, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': 'bee86328705acaa9a6daede7140defd4d9ec56bd', - 'extra.build_snapshot': False, - 'extra.cluster_name': 'docker-cluster', - 'extra.lucene_version': '8.11.1', - 'extra.name': 'allinonepod', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '7.17.0', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 9200, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-elasticsearch', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.build_hash': '79d65f6e357953a5b3cbcc5e2c7c21073d89aa29', - 'extra.build_snapshot': False, - 'extra.cluster_name': 'docker-cluster', - 'extra.lucene_version': '8.9.0', - 'extra.name': 'f547c2952610', - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'elasticsearch', - 'extra.tagline': 'You Know, for Search', - 'extra.version': '7.15.0', - 'feed.name': 'Open Elasticsearch', - 'protocol.application': 'elasticsearch', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 9200, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py deleted file mode 100644 index aeeffa3c2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_exchange.py +++ /dev/null @@ -1,149 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Birger Schacht -# -# SPDX-License-Identifier: AGPL-3.0-or-later -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/scan_exchange.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "Shadowserver CVE-2021-26855", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-scan_exchange.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:30+00:00", - "source.ip": "12.237.1.2", - "source.port": 443, - "source.asn": 7018, - "source.geolocation.cc": "US", - "source.geolocation.region": "CALIFORNIA", - "source.geolocation.city": "TURLOCK", - "source.reverse_dns": 'afs-exch-cas2.xxx.com', - "extra.version": '15.2.721', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 517311, - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "extra.servername": "AFS-EXCH2019", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:37+00:00", - "source.ip": "98.153.3.4", - "source.port": 443, - "source.asn": 20001, - "source.geolocation.cc": "US", - "source.geolocation.region": "CALIFORNIA", - "source.geolocation.city": "LOS ANGELES", - "source.reverse_dns": 'rrcs-98-153-x-x.west.biz.rr.com', - "extra.version": '15.0.847', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 517311, - "extra.tag": "exchange;webshell", - "classification.taxonomy": "intrusions", - "classification.type": "system-compromise", - "classification.identifier": "exchange-server-webshell", - "extra.servername": "SSAMAIL", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "206.210.5.6", - "source.port": 443, - "source.asn": 17054, - "source.geolocation.cc": "US", - "source.geolocation.region": "PENNSYLVANIA", - "source.geolocation.city": "PITTSBURGH", - "source.reverse_dns": 'webmail.xxx.com', - "extra.source.naics": 518210, - "extra.version": '15.0.1178', - "extra.servername": "OMNYXEXCH02", - "extra.tag": "exchange;webshell", - "classification.taxonomy": "intrusions", - "classification.type": "system-compromise", - "classification.identifier": "exchange-server-webshell", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "12.33.7.8", - "source.port": 443, - "source.asn": 7018, - "source.geolocation.cc": "US", - "source.geolocation.region": "ARKANSAS", - "source.geolocation.city": "LITTLE ROCK", - "source.reverse_dns": 'mail.xxx.org', - "extra.version": '15.1.2176', - "extra.source.sector": "Communications, Service Provider, and Hosting Service", - "extra.source.naics": 921120, - "extra.servername": "MHASVR02", - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Shadowserver CVE-2021-26855", - "time.source": "2021-05-14T00:11:38+00:00", - "source.ip": "41.204.9.10", - "source.port": 443, - "source.asn": 21042, - "source.geolocation.cc": 'MG', - "source.geolocation.city": 'ANTANANARIVO', - "source.geolocation.region": 'ANTANANARIVO', - "source.reverse_dns": 'mail.xxx.mg', - "extra.servername": "SABMHQE0232", - "classification.identifier": "vulnerable-exchange-server", - "extra.tag": "exchange;cve-2021-26855", - "classification.taxonomy": "vulnerable", - "classification.type": "infected-system", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py deleted file mode 100644 index 33daefd75..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ftp.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ftp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible FTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ftp-test-test.csv", - } -EVENTS = [{ - '__type': 'Event', - 'feed.name': 'Accessible FTP', - 'time.observation': '2019-03-25T00:00:00+00:00', - 'time.source': '2019-03-06T06:37:00+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-ftp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.ip': '61.126.3.70', - 'source.port': 21, - 'protocol.transport': 'tcp', - 'protocol.application': 'ftp', - 'source.reverse_dns': 'arcus-net.co.jp', - 'extra.tag': 'ftp', - 'source.asn': 4713, - 'source.geolocation.cc': 'JP', - 'source.geolocation.region': 'TOKYO', - 'source.geolocation.city': 'TOKYO', - 'extra.naics': 517311, - 'extra.sic': 737401, - 'extra.banner': '220 FTP Server ready.|', - 'extra.handshake': 'TLSv1.2', - 'extra.cipher_suite': 'TLS_RSA_WITH_AES_128_CBC_SHA', - 'extra.cert_length': 2048, - 'extra.subject_common_name': '*.bizmw.com', - 'extra.issuer_common_name': 'GlobalSign Organization Validation CA - SHA256 - G2', - 'extra.cert_issue_date': 'Jan 14 08:04:50 2015 GMT', - 'extra.cert_expiration_date': 'Jan 14 08:04:50 2020 GMT', - 'extra.sha1_fingerprint': 'D9:98:3F:2E:F9:D1:BE:9A:10:1E:DE:51:2C:C1:DF:01:18:0A:20:65', - 'extra.cert_serial_number': '1121DC7421AB7924C3B1D396AEA3707E9E29', - 'extra.ssl_version': 2, - 'extra.signature_algorithm': 'sha256WithRSAEncryption', - 'extra.key_algorithm': 'rsaEncryption', - 'extra.subject_organization_name': 'NTT Communications Corporation', - 'extra.subject_country': 'JP', - 'extra.subject_state_or_province_name': 'Tokyo', - 'extra.subject_locality_name': 'Minato-ku', - 'extra.issuer_organization_name': 'GlobalSign nv-sa', - 'extra.issuer_country': 'BE', - 'extra.sha256_fingerprint': '27:4A:8A:3A:A7:DF:82:D0:43:03:0E:6F:48:30:30:C9:24:77:11:1A:08:EF:F7:B9:74:0C:CE:40:87:03:D2:51', - 'extra.sha512_fingerprint': 'E5:93:8B:72:84:0F:35:52:8E:7A:6C:E3:EF:36:90:4C:F2:86:A7:4D:B2:DD:C0:C6:23:83:18:EF:DD:86:34:92:91:57:22:29:75:45:71:8B:3A:CD:F1:27:A9:CA:5F:70:5E:AC:15:A5:E6:63:FD:6F:BB:C5:E2:45:99:73:E9:E6', - 'extra.md5_fingerprint': 'D1:A7:BC:96:78:1D:16:D0:24:A8:62:7C:3A:95:5A:4A', - 'extra.cert_valid': False, - 'extra.self_signed': False, - 'extra.cert_expired': False, - 'extra.validation_level': 'OV', - 'extra.auth_tls_response': '234 AUTH TLS successful', - }, - { - '__type': 'Event', - 'feed.name': 'Accessible FTP', - 'time.observation': '2019-03-25T00:00:00+00:00', - 'time.source': '2019-03-06T06:37:00+00:00', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-ftp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.ip': '62.48.156.65', - 'source.port': 21, - 'protocol.transport': 'tcp', - 'protocol.application': 'ftp', - 'source.reverse_dns': 'dial-62-48-156-65.ptprime.net', - 'extra.tag': 'ftp', - 'source.asn': 15525, - 'source.geolocation.cc': 'PT', - 'source.geolocation.region': 'LISBOA', - 'source.geolocation.city': 'FRIELAS', - 'extra.banner': '220-================================================================| PT Empresas| Acesso Reservado| Acesso nao autorizado punido por lei: 109/91; 67/98| ----------------------------------------------------------------| HENNES & MAURITZ LDA - 149093| SITE: PT303 - Cascais Shopping| MORADA: | NIR: EWS1822940| ================================================================|220 FTP server ready, 1 active clients of 4 simultaneous clients allowed.|', - 'extra.auth_tls_response': '500 Syntax error, command unrecognized.', - 'extra.auth_ssl_response': '500 Syntax error, command unrecognized.' - } - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py deleted file mode 100644 index 0b5794cb7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_hadoop.py +++ /dev/null @@ -1,94 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_hadoop.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer Accessible-Hadoop", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_hadoop-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'ShadowServer Accessible-Hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-hadoop', - 'extra.version': '2.7.3, rbaa91f7c6bc9cb92be5982de4719c1c8af91ccff', - 'extra.server_type': 'namenode', - 'extra.clusterid': 'CID-64471a53-60cb-4302-9832-92f321f111fe', - 'extra.total_disk': 41567956992, - 'extra.used_disk': 53248, - 'extra.free_disk': 25160089600, - 'extra.livenodes': 'edmonton:50010', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn': 15296, - 'source.geolocation.cc': 'CA', - 'source.geolocation.city': 'CALGARY', - 'source.geolocation.region': 'ALBERTA', - 'source.ip': '199.116.235.200', - 'source.port': 50070, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-09-13T02:06:05+00:00'}, - {'__type': 'Event', - 'feed.name': 'ShadowServer Accessible-Hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'accessible-hadoop', - 'extra.version': '2.7.1.2.4.0.0-169', - 'extra.naics': 334111, - 'extra.sic': 357101, - 'extra.server_type': 'datanode', - 'extra.clusterid': 'CID-771bae52-9e4f-4ec4-bc1a-c867585751f0', - 'extra.namenodeaddress': 'sandbox.hortonworks.com', - 'extra.volumeinfo': '/hadoop/hdfs/data/current', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn': 8075, - 'source.geolocation.cc': 'US', - 'source.geolocation.city': 'DES MOINES', - 'source.geolocation.region': 'IOWA', - 'source.ip': '104.43.235.92', - 'source.port': 50075, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2017-09-13T02:07:48+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py deleted file mode 100644 index 793a95f22..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http.py +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible HTTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_http-test-test.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible HTTP', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'accessible-http', - 'extra.source.naics': 518111, - 'extra.source.sic': 737401, - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_reason': 'OK', - 'extra.content_type': 'text/html', - 'extra.server': 'lighttpd', - 'extra.transfer_encoding': 'chunked', - 'extra.http_date': '2018-04-19T00:02:28+00:00', - 'extra.tag': 'http', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.reverse_dns': 'c-75-74-78-113.hsd1.fl.comcast.net', - 'source.asn': 7922, - 'source.geolocation.cc': 'US', - 'source.geolocation.city': 'MIAMI', - 'source.geolocation.region': 'FLORIDA', - 'source.ip': '75.74.78.113', - 'source.port': 8080, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2018-04-19T00:02:26+00:00'}, - {'__type': 'Event', - 'feed.name': 'Accessible HTTP', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'accessible-http', - 'extra.source.naics': 518210, - 'extra.source.sic': 737415, - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_reason': 'OK', - 'extra.content_type': 'text/html', - 'extra.content_length': 17729, - 'extra.http_date': '2018-04-19T02:02:28+00:00', - 'extra.tag': 'http', - 'protocol.transport': 'tcp', - 'protocol.application': 'http', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.reverse_dns': 'sto95-3-88-162-174-130.fbx.proxad.net', - 'source.asn': 12322, - 'source.geolocation.cc': 'FR', - 'source.geolocation.city': 'SAINT-OUEN-LAUMONE', - 'source.ip': '88.162.174.130', - 'source.port': 8080, - 'time.observation': '2015-01-01T00:00:00+00:00', - 'time.source': '2018-04-19T00:02:26+00:00'}, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py deleted file mode 100644 index dc5e94e5e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_proxy.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http_proxy.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open HTTP Proxy', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_http_proxy-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3128, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'extra.via': 'HTTP/1.1 s_proxy_den1', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3128, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 200, - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.http_reason': 'Connection established', - 'extra.tag': 'http-connect-proxy', - 'extra.via': 'HTTP/1.1 s_proxy_yvr', - 'feed.name': 'Open HTTP Proxy', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3128, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py deleted file mode 100644 index d15232eaf..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_http_vulnerable.py +++ /dev/null @@ -1,125 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_http_vulnerable.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable HTTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-08-01T09:00:00+00:00", - "extra.file_name": "2021-08-01-scan_http_vulnerable-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.content_length': 149, - 'extra.content_type': 'text/html; charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 401, - 'extra.http_date': '2010-02-10T00:00:00+00:00', - 'extra.http_reason': 'Unauthorized', - 'extra.server': 'TwistedWeb/19.7.0', - 'extra.set_cookie': 'TWISTED_SESSION=5473ad3faa3de66685fb3a53bffb390b4fcec2039893009a06caf38e1bec8aa8', - 'extra.tag': 'basic-auth,http', - 'extra.www_authenticate': 'Basic realm=\\\\"OpenWebif\\"\\""', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 8080, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.content_length': 149, - 'extra.content_type': 'text/html; charset=utf-8', - 'extra.http': 'HTTP/1.1', - 'extra.http_code': 401, - 'extra.http_date': '2010-02-10T00:00:01+00:00', - 'extra.http_reason': 'Unauthorized', - 'extra.server': 'TwistedWeb/19.7.0', - 'extra.set_cookie': 'TWISTED_SESSION=d2460d37b7fdbdd6c27dd74423ead5704e553d4f2c230672313edc5602059e33', - 'extra.tag': 'basic-auth,http', - 'extra.www_authenticate': 'Basic realm=\\\\"OpenWebif\\"\\""', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 80, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.detail': 'repositoryformatversion = 0;filemode = false;bare = ' - 'false;logallrefupdates = true;symlinks = false;ignorecase = ' - 'true', - 'extra.http_date': '2010-02-10T00:00:02+00:00', - 'extra.tag': 'git-config-file', - 'feed.name': 'Vulnerable HTTP', - 'protocol.application': 'http', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 443, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py deleted file mode 100644 index f673f40c8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ics.py +++ /dev/null @@ -1,125 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ics.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Acessible ICS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ics-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 1', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDE=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.1', - 'source.port' : 502, - 'source.reverse_dns' : 'host1.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 2', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDI=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64513, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.2', - 'source.port' : 502, - 'source.reverse_dns' : 'host2.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ics', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.device_model' : 'device_model', - 'extra.device_type' : 'device_type', - 'extra.device_vendor' : 'Vendor 3', - 'extra.device_version' : 'device_version', - 'extra.raw_response' : 'dGVzdDM=', - 'extra.response_size' : 5, - 'extra.source.sector' : 'Sector', - 'feed.name' : 'Acessible ICS', - 'protocol.application' : 'modbus', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64514, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'CITY', - 'source.geolocation.region' : 'REGION', - 'source.ip' : '192.168.0.3', - 'source.port' : 502, - 'source.reverse_dns' : 'host3.example.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-03-02T00:34:22+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py deleted file mode 100644 index 08a9082af..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipmi.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ipmi.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open IPMI', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ipmi-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open IPMI', - "classification.identifier": "open-ipmi", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.anon_login": False, - "extra.defaultkg": "-", - "extra.ipmi_version": "1.5", - "extra.md2_auth": False, - "extra.md5_auth": True, - "extra.none_auth": True, - "extra.nulluser": True, - "extra.oem_auth": False, - "extra.passkey_auth": True, - "extra.permessage_auth": True, - "extra.tag": "ipmi", - "extra.userlevel_auth": True, - "extra.usernames": False, - "protocol.application": "ipmi", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 2914, - "source.geolocation.cc": "DE", - "source.geolocation.city": "BERLIN", - "source.geolocation.region": "BERLIN", - "source.ip": "198.51.100.4", - "source.port": 623, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:09:42+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open IPMI', - "classification.identifier": "open-ipmi", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.anon_login": False, - "extra.defaultkg": "default", - "extra.ipmi_version": "2.0", - "extra.md2_auth": False, - "extra.md5_auth": False, - "extra.none_auth": False, - "extra.nulluser": False, - "extra.oem_auth": False, - "extra.passkey_auth": False, - "extra.permessage_auth": False, - "extra.tag": "ipmi", - "extra.userlevel_auth": True, - "extra.usernames": True, - "protocol.application": "ipmi", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 28753, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.182", - "source.port": 623, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:09:43+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py deleted file mode 100644 index 9adc8485e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ipp.py +++ /dev/null @@ -1,79 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ipp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open-IPP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-06-09T00:00:00+00:00", - "extra.file_name": "2020-06-08-scan_ipp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open-IPP', - "classification.identifier": "open-ipp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "ipp", - "extra.ipp_version": "IPP/2.1", - "extra.cups_version": "CUPS/2.0", - "extra.printer_uris": "ipp://123.45.67.89:631/ipp/print", - "extra.printer_name": "NPI3F0D22", - "extra.printer_info": "HP Color LaserJet MFP M277dw", - "extra.printer_more_info": "http://123.45.67.89:631/hp/device/info_config_AirPrint.html?tab=Networking&menu=AirPrintStatus", - "extra.printer_make_and_model": "HP Color LaserJet MFP M277dw", - "extra.printer_firmware_name": "20191203", - "extra.printer_firmware_string_version": "20191203", - "extra.printer_firmware_version": "20191203", - "extra.printer_organization": "org", - "extra.printer_organization_unit": "unit", - "extra.printer_uuid": "urn:uuid:456e4238-4a44-4643-4c42-10e1813f0a18", - "extra.printer_wifi_ssid": "wifissid", - "protocol.application": "ipp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 12345, - "source.geolocation.cc": "AA", - "source.geolocation.city": "CITY", - "source.geolocation.region": "REGION", - "source.ip": "123.45.67.89", - "source.port": 631, - 'source.reverse_dns': 'some.host.com', - "time.observation": "2020-06-09T00:00:00+00:00", - "time.source": "2020-06-08T11:30:14+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py deleted file mode 100644 index 3192f508f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_isakmp.py +++ /dev/null @@ -1,105 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_isakmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable ISAKMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_isakmp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Vulnerable ISAKMP', - "classification.identifier": "open-ike", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.domain_of_interpretation": '00', - "extra.exchange_type": '05', - "extra.flags": '00', - "extra.initiator_spi": "3e35c70729dfedef", - "extra.message_id": "00000000", - "extra.naics": 517311, - "extra.next_payload": '11', - "extra.next_payload2": '00', - "extra.notify_message_type": '14', - "extra.responder_spi": "253acab7cbfda607", - "extra.spi_size": 0, - "extra.tag": "isakmp-vulnerable", - "protocol.application": "ipsec", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.42", - "source.port": 500, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T00:17:25+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Vulnerable ISAKMP', - "classification.identifier": "open-ike", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.domain_of_interpretation": '00', - "extra.exchange_type": '05', - "extra.flags": '00', - "extra.initiator_spi": "3e35c70729dfedef", - "extra.message_id": "00000000", - "extra.next_payload": '11', - "extra.next_payload2": '00', - "extra.notify_message_type": '14', - "extra.responder_spi": "b274460e7adc1bf0", - "extra.spi_size": 0, - "extra.tag": "isakmp-vulnerable", - "protocol.application": "ipsec", - "protocol.transport": "udp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.67", - "source.port": 500, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T00:17:28+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py deleted file mode 100644 index 2bac336a7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_kubernetes.py +++ /dev/null @@ -1,214 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_kubernetes.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Kubernetes API Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_kubernetes-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2021-11-17T13:00:29Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:13 GMT', - 'extra.git_commit' : '2444b3347a2c45eb965b182fb836e1f51dc61b70', - 'extra.git_tree_state' : 'clean', - 'extra.git_version' : 'v1.20.13', - 'extra.go_version' : 'go1.15.15', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '20', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 6443, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2022-02-25T06:26:46Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:12 GMT', - 'extra.git_commit' : '6f5a5295923a614a4202a7ad274b38b69f9ca8c0', - 'extra.git_tree_state' : 'clean', - 'extra.git_version' : 'v1.23.3+e419edf', - 'extra.go_version' : 'go1.17.5', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '23', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 6443, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-kubernetes', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.browser_trusted' : False, - 'extra.build_date' : '2020-05-08T07:29:59Z', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.compiler' : 'gc', - 'extra.content_type' : 'application/json', - 'extra.date' : 'Tue, 10 May 2022 14:24:12 GMT', - 'extra.git_commit' : '4f7ea78', - 'extra.git_version' : 'v1.16.9-aliyun.1', - 'extra.go_version' : 'go1.13.9', - 'extra.handshake' : 'TLSv1.2', - 'extra.http' : 'HTTP/1.1', - 'extra.http_code' : 200, - 'extra.http_reason' : 'OK', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.major' : '1', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.minor' : '16+', - 'extra.platform' : 'linux/amd64', - 'extra.self_signed' : False, - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'kubernetes', - 'feed.name' : 'Accessible Kubernetes API Server', - 'protocol.application' : 'kubernetes', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 6443, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py deleted file mode 100644 index b6abf6eba..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_tcp.py +++ /dev/null @@ -1,154 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ldap_tcp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open LDAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ldap_tcp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 2, - 'extra.ldap_service_name': 'node01.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'node01.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821124435.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 25029662, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node02.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'node02.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821124539.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.ldap_service_name': 'node03.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 0, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|', - 'extra.tag': 'ldap-tcp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'node03.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py deleted file mode 100644 index aa4deefb8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ldap_udp.py +++ /dev/null @@ -1,162 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ldap_udp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open LDAP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ldap_udp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 58.42, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821044533.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 222537, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node01.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 3038, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'node01.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 58.88, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.current_time': '20220821044948.0Z', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.domain_controller_functionality': 7, - 'extra.domain_functionality': 7, - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.forest_functionality': 7, - 'extra.highest_committed_usn': 1478714, - 'extra.is_global_catalog_ready': True, - 'extra.is_synchronized': True, - 'extra.ldap_service_name': 'node02.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 3062, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.supported_capabilities': '1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237', - 'extra.supported_control': '1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354', - 'extra.supported_ldap_policies': 'MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent', - 'extra.supported_ldap_version': '3|2', - 'extra.supported_sasl_mechanisms': 'GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'node02.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 0.69, - 'extra.configuration_naming_context': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.default_naming_context': 'DC=ad,DC=example,DC=com', - 'extra.ds_service_name': 'CN=Configuration,DC=ad,DC=example,DC=com', - 'extra.ldap_service_name': 'node03.example.com', - 'extra.naming_contexts': 'DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.root_domain_naming_context': 'DC=example,DC=com', - 'extra.schema_naming_context': 'CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.server_name': 'CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com', - 'extra.size': 36, - 'extra.subschema_subentry': 'CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com', - 'extra.tag': 'ldap-udp', - 'feed.name': 'Open LDAP', - 'protocol.application': 'ldap', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'node03.example.com', - 'source.port': 389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py deleted file mode 100644 index 9207aaf36..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mdns.py +++ /dev/null @@ -1,127 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mdns.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open mDNS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mdns-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_ipv4' : '192.168.0.1', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::1', - 'extra.services' : '_smb._tcp.local.; _device-info._tcp.local.; _http._tcp.local.; _dacp._tcp.local.;', - 'extra.tag' : 'mdns', - 'extra.workstation_ipv4' : '192.168.0.1', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::1', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5353, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_ipv4' : '192.168.0.2', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::2', - 'extra.services' : '_home-assistant._tcp.local.;', - 'extra.tag' : 'mdns', - 'extra.workstation_ipv4' : '192.168.0.2', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::2', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5353, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mdns', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.http_info' : '\\\\\"vendor=Synology\\\"\\\" \\\"\\\"model=DS218+\\\"\\\" \\\"\\\"serial=17A0PCN482002\\\"\\\" \\\"\\\"version_major=6\\\"\\\" \\\"\\\"version_minor=2\\\"\\\" \\\"\\\"version_build=25556\\\"\\\" \\\"\\\"admin_port=5000\\\"\\\" \\\"\\\"secure_admin_port=5001\\\"\\\" \\\"\\\"mac_address=00:11:32:80:fd:b5\\\"\\\"\"', - 'extra.http_ipv4' : '192.168.0.3', - 'extra.http_ipv6' : 'fd09:4ab5:dae9:b078::3', - 'extra.http_name' : 'snmeijer.local.', - 'extra.http_port' : 5000, - 'extra.http_ptr' : 'snmeijer._http._tcp.local.', - 'extra.http_target' : 'snmeijer.local.', - 'extra.services' : '_webdav._tcp.local.; _adisk._tcp.local.; _smb._tcp.local.; _http._tcp.local.; _dacp._tcp.local.; _afpovertcp._tcp.local.; _device-info._tcp.local.;', - 'extra.tag' : 'mdns,iot', - 'extra.workstation_ipv4' : '192.168.0.3', - 'extra.workstation_ipv6' : 'fd09:4ab5:dae9:b078::3', - 'feed.name' : 'Open mDNS', - 'protocol.application' : 'mdns', - 'protocol.transport' : 'udp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5353, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py deleted file mode 100644 index b54fc0ea5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_memcached.py +++ /dev/null @@ -1,130 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_memcached.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Memcached', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_memcached-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 81.71, - 'extra.curr_connections': 243, - 'extra.pid': 1010, - 'extra.pointer_size': 64, - 'extra.response_size': 1144, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:34:06', - 'extra.total_connections': 6106, - 'extra.uptime': 32908114, - 'extra.version': '1.4.15', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 50260, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 75.21, - 'extra.curr_connections': 9, - 'extra.pid': 5316, - 'extra.pointer_size': 64, - 'extra.response_size': 1053, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:39:21', - 'extra.total_connections': 2962, - 'extra.uptime': 9618498, - 'extra.version': '1.4.13', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 11211, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-memcached', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 31.57, - 'extra.curr_connections': 2, - 'extra.pid': 1460, - 'extra.pointer_size': 32, - 'extra.response_size': 442, - 'extra.tag': 'memcached', - 'extra.time': '2022-08-21 10:39:39', - 'extra.total_connections': 534, - 'extra.uptime': 1375159, - 'extra.version': '1.2.6', - 'feed.name': 'Open Memcached', - 'protocol.application': 'memcached', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 11211, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py deleted file mode 100644 index 3ecf7b21f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mongodb.py +++ /dev/null @@ -1,103 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mongodb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open MongoDB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mongodb-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open MongoDB', - "classification.identifier": "open-mongodb", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.allocator": "tcmalloc", - "extra.bits": "64", - "extra.gitversion": "a2ddc68ba7c9cee17bfe69ed840383ec3506602b", - "extra.javascriptengine": "V8", - "extra.maxbsonobjectsize": "16777216", - "extra.ok": True, - "extra.sysinfo": "Linux ip-198-51-100-100 198.51.100.103-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49", - "extra.tag": "mongodb", - "extra.version": "2.4.5", - "extra.visible_databases": "local | countly | admin", - "protocol.application": "mongodb", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 20773, - "source.geolocation.cc": "DE", - "source.geolocation.city": "WEEZE", - "source.geolocation.region": "NORDRHEIN-WESTFALEN", - "source.ip": "198.51.100.203", - "source.port": 27017, - "source.reverse_dns": "198-51-100-203.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:40:07+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open MongoDB', - "classification.identifier": "open-mongodb", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.allocator": "tcmalloc", - "extra.bits": "64", - "extra.gitversion": "d73c92b1c85703828b55c2916a5dd4ad46535f6a", - "extra.javascriptengine": "V8", - "extra.maxbsonobjectsize": "16777216", - "extra.ok": True, - "extra.sector": "Information Technology", - "extra.sysinfo": "Linux build5.ny.cbi.10gen.cc 2.6.32-431.3.1.el6.x86_64 #1 SMP Fri Jan 3 21:39:27 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49", - "extra.tag": "mongodb", - "extra.version": "2.6.12", - "extra.visible_databases": "none visible", - "protocol.application": "mongodb", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 24940, - "source.geolocation.cc": "DE", - "source.geolocation.city": "GUNZENHAUSEN", - "source.geolocation.region": "BAYERN", - "source.ip": "198.51.100.42", - "source.port": 27017, - "source.reverse_dns": "198-51-100-208.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:40:07+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py deleted file mode 100644 index 45d19f9ee..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt.py +++ /dev/null @@ -1,89 +0,0 @@ -# SPDX-FileCopyrightText: 2020 Thomas Hungenberg -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mqtt.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open-MQTT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-03-15T00:00:00+00:00", - "extra.file_name": "2020-03-14-scan_mqtt-test-geo.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.anonymous_access' : False, - 'extra.cert_expiration_date' : '2022-11-14 00:00:00', - 'extra.cert_issue_date' : '2020-08-12 00:00:00', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '085699743A23114C9B6B8DC975A8AF42', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Refused, not authorized', - 'extra.hex_code' : '05', - 'extra.issuer_common_name' : 'Sectigo RSA Domain Validation Secure Server CA', - 'extra.issuer_country' : 'GB', - 'extra.issuer_locality_name' : 'Salford', - 'extra.issuer_organization_name' : 'Sectigo Limited', - 'extra.issuer_state_or_province_name' : 'Greater Manchester', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'DE:2C:98:30:27:2E:7D:C9:ED:A3:9D:AF:9E:CE:14:CC', - 'extra.raw_response' : '20020005', - 'extra.sha1_fingerprint' : '70:84:F1:6D:28:DA:B6:E6:27:60:13:8B:2C:93:52:B6:7B:4B:13:7B', - 'extra.sha256_fingerprint' : 'D2:D7:54:52:EB:86:4E:2D:34:4D:FC:CE:CD:CF:39:41:E1:06:5C:8B:B8:54:E6:0C:DF:FD:6E:E3:F1:B5:41:00', - 'extra.sha512_fingerprint' : '17:57:FB:88:9D:BE:A7:F0:29:A5:31:FC:79:DF:F7:8A:1C:D6:4A:DF:1B:4A:DC:BF:05:E7:E8:2F:79:9A:FA:FE:F7:E8:66:22:CB:B9:4C:72:F7:FB:6C:1D:59:8C:54:63:70:05:DE:7F:3C:2F:BA:B8:37:18:CE:29:6F:11:E8:AB', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.naics' : 454110, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : '*.tracesafe.io', - 'extra.tag' : 'mqtt', - 'feed.name' : 'Open-MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 12345, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'COLUMBUS', - 'source.geolocation.region' : 'OHIO', - 'source.ip' : '18.220.0.0', - 'source.port' : 8883, - 'source.reverse_dns' : '18-220-0-0.example.com', - 'time.observation' : '2020-03-15T00:00:00+00:00', - 'time.source' : '2022-02-07T12:56:53+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py deleted file mode 100644 index 461895724..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mqtt_anon.py +++ /dev/null @@ -1,173 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_mqtt_anon.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Anonymous MQTT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_mqtt_anon-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2030-05-06 08:07:05', - 'extra.cert_issue_date' : '2020-05-08 08:07:05', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '02', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'RootCA', - 'extra.issuer_country' : 'CN', - 'extra.issuer_organization_name' : 'EMQ', - 'extra.issuer_state_or_province_name' : 'hangzhou', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'AB:A8:E0:2C:EF:AE:BF:9D:DD:FA:70:BA:2F:F2:CA:5C', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '70:1A:1E:1F:EC:5F:7E:A9:12:32:B2:C9:8A:C9:EE:91:8E:0B:82:45', - 'extra.sha256_fingerprint' : '85:26:A2:F2:A2:50:CD:96:33:19:A6:2D:12:2E:97:6B:D3:06:3C:11:EA:01:B4:B7:25:2A:B7:4F:0A:8F:45:40', - 'extra.sha512_fingerprint' : '72:50:07:30:9A:6F:CB:FD:E2:80:69:02:65:62:77:16:C3:B4:0C:98:44:4E:D4:2C:AC:6B:AF:F8:9E:AB:51:C2:FA:A8:72:A3:45:DF:81:09:50:08:18:EB:03:34:FC:92:33:A7:12:46:FE:90:20:91:86:C5:4D:89:48:86:4C:CD', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 518210, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'Server', - 'extra.subject_country' : 'CN', - 'extra.subject_organization_name' : 'EMQ', - 'extra.subject_state_or_province_name' : 'hangzhou', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 37963, - 'source.geolocation.cc' : 'CN', - 'source.geolocation.city' : 'SHENZHEN', - 'source.geolocation.region' : 'GUANGDONG SHENG', - 'source.ip' : '47.106.0.0', - 'source.port' : 8883, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2022-03-06 13:48:03', - 'extra.cert_issue_date' : '2021-12-06 13:48:04', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '06B25BEAD1F43266ABCFCDDE408D3544D04B', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'R3', - 'extra.issuer_country' : 'US', - 'extra.issuer_organization_name' : 'Lets Encrypt', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '23:99:39:C6:77:D8:9F:55:90:FC:A5:FB:BA:72:8B:42', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '20:0E:AC:E7:AF:07:8D:D3:16:7C:63:D1:B9:12:AD:1D:2C:F0:46:86', - 'extra.sha256_fingerprint' : 'DD:7A:4C:8A:1D:66:1D:7C:F5:17:04:5B:A0:B4:C4:E0:80:58:44:B4:DB:A7:5E:61:AE:43:9D:85:4C:9E:DC:83', - 'extra.sha512_fingerprint' : '55:B6:3D:56:A4:39:6E:99:B6:AF:72:AF:4D:3C:7C:C5:A8:C5:4F:A1:79:92:D0:46:8A:A2:9B:2A:48:0D:00:68:39:F0:B8:67:B4:E0:88:51:2A:D7:55:46:83:BD:ED:1E:09:6E:DB:3D:21:E2:AA:DB:42:6A:33:45:1A:2A:DB:4C', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 518210, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 24940, - 'source.geolocation.cc' : 'DE', - 'source.geolocation.city' : 'WERNIGERODE', - 'source.geolocation.region' : 'SACHSEN-ANHALT', - 'source.ip' : '144.76.0.0', - 'source.port' : 8883, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mqtt-anon', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.cert_expiration_date' : '2030-08-05 16:51:57', - 'extra.cert_issue_date' : '2020-08-07 16:51:57', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'A71541EFAE529B03', - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.code' : 'Connection Accepted', - 'extra.hex_code' : '00', - 'extra.issuer_common_name' : 'ClearView2Dev', - 'extra.issuer_organization_name' : 'Sohonet', - 'extra.issuer_organization_unit_name' : 'ClearView2Dev', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '43:0D:A7:89:9E:76:8D:6E:D5:AD:95:CC:F2:91:87:56', - 'extra.raw_response' : '20020000', - 'extra.sha1_fingerprint' : '32:4B:66:98:FA:5B:D2:D1:F2:53:83:21:19:11:5A:A9:BE:85:56:16', - 'extra.sha256_fingerprint' : 'AE:0D:65:34:2F:51:F7:32:1E:DF:B1:DA:12:C7:6A:DE:42:B5:4B:FF:80:2C:E5:EF:99:F6:CC:01:4B:C9:77:68', - 'extra.sha512_fingerprint' : '44:C4:B8:19:FA:39:55:51:EC:E4:6D:C4:6D:0F:A5:46:BB:D5:F9:FD:A6:8D:DF:F3:2D:D2:92:6C:0B:D5:D3:25:CB:19:50:9D:A6:A4:D4:D3:2E:53:10:F5:8D:77:F7:90:F8:65:A7:79:AB:14:62:72:01:F3:EA:38:E2:68:C7:25', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 0, - 'extra.subject_common_name' : 'foo.example.com', - 'extra.subject_locality_name' : '<', - 'extra.subject_organization_name' : 'Sohonet', - 'extra.tag' : 'mqtt,mqtt-anon', - 'feed.name' : 'Open Anonymous MQTT', - 'protocol.application' : 'mqtt', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 5555, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'BURBANK', - 'source.geolocation.region' : 'CALIFORNIA', - 'source.ip' : '173.0.0.0', - 'source.port' : 8883, - 'source.reverse_dns' : 'example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:59:34+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py deleted file mode 100644 index 0f12014e6..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mssql.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_mssql.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open MSSQL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_mssql-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 310.0, - 'extra.instance_name': 'OPTIMA', - 'extra.named_pipe': '\\\\\\\\ERPOPTIMA\\\\pipe\\\\MSSQL$OPTIMA\\\\sql\\\\query', - 'extra.response_size': 310, - 'extra.tag': 'mssql', - 'extra.tcp_port': 49729, - 'extra.version': '13.2.5026.0', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.local_hostname': 'ERPOPTIMA', - 'source.port': 1434, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 226.0, - 'extra.instance_name': 'MSSQLSERVER', - 'extra.response_size': 226, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'mssql', - 'extra.tcp_port': 1433, - 'extra.version': '13.0.1601.5', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.local_hostname': 'SERWER', - 'source.port': 1434, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-mssql', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 304.0, - 'extra.instance_name': 'INSERTGT', - 'extra.named_pipe': '\\\\\\\\ILONY\\\\pipe\\\\MSSQL$INSERTGT\\\\sql\\\\query', - 'extra.response_size': 304, - 'extra.tag': 'mssql', - 'extra.tcp_port': 49358, - 'extra.version': '10.50.2500.0', - 'feed.name': 'Open MSSQL', - 'protocol.application': 'mssql', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.local_hostname': 'ILONY', - 'source.port': 1434, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py deleted file mode 100644 index 3e008f950..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_mysql.py +++ /dev/null @@ -1,258 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_mysql.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible MySQL Server', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_mysql-test.csv", - } -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '5.7.37-0ubuntu0.18.04.1', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 3306, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '5.7.30-0ubuntu0.18.04.1-log', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 3306, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-mysql', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: failed to load system roots and no roots provided', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_can_handle_expired_passwords' : True, - 'extra.client_compress' : True, - 'extra.client_connect_attrs' : True, - 'extra.client_connect_with_db' : True, - 'extra.client_deprecated_eof' : True, - 'extra.client_found_rows' : True, - 'extra.client_ignore_sigpipe' : True, - 'extra.client_ignore_space' : True, - 'extra.client_interactive' : True, - 'extra.client_local_files' : True, - 'extra.client_long_flag' : True, - 'extra.client_long_password' : True, - 'extra.client_multi_results' : True, - 'extra.client_multi_statements' : True, - 'extra.client_no_schema' : True, - 'extra.client_odbc' : True, - 'extra.client_plugin_auth' : True, - 'extra.client_plugin_auth_len_enc_client_data' : True, - 'extra.client_protocol_41' : True, - 'extra.client_ps_multi_results' : True, - 'extra.client_reserved' : True, - 'extra.client_secure_connection' : True, - 'extra.client_session_track' : False, - 'extra.client_ssl' : False, - 'extra.client_transactions' : False, - 'extra.error_code' : '1', - 'extra.error_id' : '1', - 'extra.error_message' : '1', - 'extra.handshake' : 'TLSv1.2', - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.mysql_protocol_version' : '10', - 'extra.server_version' : '8.0.23', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.tag' : 'mysql', - 'feed.name' : 'Accessible MySQL Server', - 'protocol.application' : 'mysql', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 3306, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py deleted file mode 100644 index beeac2717..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_nat_pmp.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_nat_pmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open NATPMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_nat_pmp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.1', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 291278940, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5351, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.2', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 768416, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5351, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, - {'__type': 'Event', - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.0, - 'extra.external_ip': '192.168.0.3', - 'extra.opcode': '128', - 'extra.response_size': 12, - 'extra.tag': 'nat-pmp', - 'extra.uptime': 19629454, - 'feed.name': 'Open NATPMP', - 'protocol.application': 'natpmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5351, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py deleted file mode 100644 index febe8305c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_netbios.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_netbios.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Netbios', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_netbios-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 4.58, - 'extra.mac_address': '00-00-00-00-00-00', - 'extra.machine_name': 'NBG6503', - 'extra.response_size': 229, - 'extra.tag': 'netbios', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.account': 'NBG6503', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 137, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.86, - 'extra.mac_address': '00-00-00-00-00-00', - 'extra.machine_name': 'NAS-OLD', - 'extra.response_size': 193, - 'extra.tag': 'netbios', - 'extra.workgroup': 'PRACOWNIAELN.', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.account': 'NAS-OLD', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 137, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.14, - 'extra.mac_address': '00-25-90-F0-64-64', - 'extra.machine_name': 'HR-SRV01', - 'extra.response_size': 157, - 'extra.sector': 'Government', - 'extra.tag': 'netbios', - 'extra.workgroup': 'HRSIGMA', - 'feed.name': 'Netbios', - 'protocol.application': 'netbios-nameservice', - 'protocol.transport': 'udp', - 'raw': 'InRpbWVzdGFtcCIsImlwIiwicHJvdG9jb2wiLCJwb3J0IiwiaG9zdG5hbWUiLCJ0YWciLCJtYWNfYWRkcmVzcyIsImFzbiIsImdlbyIsInJlZ2lvbiIsImNpdHkiLCJ3b3JrZ3JvdXAiLCJtYWNoaW5lX25hbWUiLCJ1c2VybmFtZSIsIm5haWNzIiwic2ljIiwic2VjdG9yIiwicmVzcG9uc2Vfc2l6ZSIsImFtcGxpZmljYXRpb24iCiIyMDEwLTAyLTEwIDAwOjAwOjAyIiwxOTIuMTY4LjAuMyx1ZHAsMTM3LG5vZGUwMy5leGFtcGxlLmNvbSxuZXRiaW9zLDAwLTI1LTkwLUYwLTY0LTY0LDY0NTEyLFpaLFJlZ2lvbixDaXR5LEhSU0lHTUEsSFItU1JWMDEsLDAsMCxHb3Zlcm5tZW50LDE1NywzLjE0', - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 137, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py deleted file mode 100644 index 043cdf1aa..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_netis_router.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_netis_router.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_netis_router-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 53413, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 53413, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 18.0, - 'extra.response': 'Login:', - 'extra.response_size': 18, - 'extra.tag': 'netis_vulnerability', - 'feed.name': 'Open-Netis', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 53413, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py deleted file mode 100644 index 85ef710d4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntp.py +++ /dev/null @@ -1,161 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ntp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'NTP Version', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ntp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.0, - 'extra.clock': '0xe6ac3809.363028e7', - 'extra.frequency': 2.018, - 'extra.jitter': 0.977, - 'extra.leap': 0.0, - 'extra.noise': '0.984', - 'extra.offset': 0.557, - 'extra.peer': 18986, - 'extra.poll': 10, - 'extra.precision': -10, - 'extra.refid': '81.15.252.130', - 'extra.reftime': '0xe6ac35ba.2d2e8f2b', - 'extra.response_size': 324, - 'extra.rootdelay': 17.685, - 'extra.rootdispersion': 61.254, - 'extra.stability': '0.027', - 'extra.state': '4', - 'extra.stratum': 4, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 123, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.33, - 'extra.clk_wander': 0.007, - 'extra.clock': '0xE6AC3806.7DF3B7A0', - 'extra.frequency': -20.407, - 'extra.jitter': 8.776, - 'extra.leap': 0.0, - 'extra.mintc': '3', - 'extra.offset': -14.502, - 'extra.peer': 19244, - 'extra.precision': -10, - 'extra.refid': '10.48.21.21', - 'extra.reftime': '0xE6AC3431.B3B64790', - 'extra.response_size': 328, - 'extra.rootdelay': 32.25, - 'extra.rootdispersion': 105.778, - 'extra.sector': 'Transportation and Warehousing', - 'extra.stratum': 8, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.tc': 10, - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 123, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 27.0, - 'extra.clk_wander': 0.001, - 'extra.clock': '0xE6AC380A.5A1CAD00', - 'extra.frequency': -24.01, - 'extra.jitter': 2.343, - 'extra.leap': 0.0, - 'extra.mintc': '3', - 'extra.offset': 0.49, - 'extra.peer': 51892, - 'extra.precision': -10, - 'extra.refid': '172.28.0.1', - 'extra.reftime': '0xE6AC3020.0C49BA80', - 'extra.response_size': 324, - 'extra.rootdelay': 7.749, - 'extra.rootdispersion': 81.612, - 'extra.stratum': 4, - 'extra.system': 'UNIX', - 'extra.tag': 'ntpversion', - 'extra.tc': 10, - 'extra.version': '4', - 'feed.name': 'NTP Version', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 123, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py deleted file mode 100644 index ff0e95f3e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ntpmonitor.py +++ /dev/null @@ -1,108 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ntpmonitor.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'NTP Monitor', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ntpmonitor-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 55.33, - 'extra.packets': 2, - 'extra.size': 664, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 123, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3666.67, - 'extra.packets': 100, - 'extra.size': 44000, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 123, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3666.67, - 'extra.packets': 100, - 'extra.size': 44000, - 'feed.name': 'NTP Monitor', - 'protocol.application': 'ntp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 123, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py deleted file mode 100644 index 11caec78a..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_portmapper.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_portmapper.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Portmapper', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_portmapper-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 111, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 111, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-portmapper', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.7, - 'extra.exports': '/mnt/export 192.168.0.0', - 'extra.programs': '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; ' - '100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;', - 'extra.response_size': 148, - 'extra.sector': 'Government', - 'extra.tag': 'portmapper', - 'feed.name': 'Open Portmapper', - 'protocol.application': 'portmapper', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 111, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py deleted file mode 100644 index 43a297f78..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_postgres.py +++ /dev/null @@ -1,199 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_postgres.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-PostgreSQL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_postgres-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 5432, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 5432, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-postgres', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.cert_expiration_date' : '2021-11-12 11:18:27', - 'extra.cert_expired' : True, - 'extra.cert_issue_date' : '2012-11-14 11:18:27', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : 'B3F13DFBDBA2D8B2', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_AES_256_GCM_SHA384', - 'extra.client_ssl' : False, - 'extra.issuer_common_name' : 'example.com', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00', - 'extra.protocol_error_code' : '0A000', - 'extra.protocol_error_file' : 'postmaster.c', - 'extra.protocol_error_line' : '1798', - 'extra.protocol_error_message' : 'unsupported frontend protocol 255.255: server supports 1.0 to 3.0', - 'extra.protocol_error_routine' : 'ProcessStartupPacket', - 'extra.protocol_error_severity' : 'FATAL', - 'extra.sha1_fingerprint' : '03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55', - 'extra.sha256_fingerprint' : 'E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0', - 'extra.sha512_fingerprint' : '1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.startup_error_code' : '28000', - 'extra.startup_error_file' : 'postmaster.c', - 'extra.startup_error_line' : 1893, - 'extra.startup_error_message' : 'no PostgreSQL user name specified in startup packet', - 'extra.startup_error_routine' : 'ProcessStartupPacket', - 'extra.startup_error_severity' : 'FATAL', - 'extra.subject_common_name' : 'example.com', - 'extra.subject_country' : 'US', - 'extra.supported_protocols' : '1.0-3.0', - 'extra.tag' : 'postgres', - 'feed.name' : 'Accessible-PostgreSQL', - 'protocol.application' : 'postgres', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 5432, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py deleted file mode 100644 index de52af625..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_qotd.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_qotd.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open QOTD', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_qotd-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 166.0, - 'extra.quote': '_The secret of being miserable is to have leisure to bother ' - 'about whether?? you are happy or not. The cure for it is ' - 'occupation._?? George Bernard Shaw (1856-1950)?', - 'extra.response_size': 166, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 17, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 162.0, - 'extra.quote': '_Oh the nerves, the nerves; the mysteries of this machine ' - 'called man!?? Oh the little that unhinges it, poor creatures ' - 'that we are!_?? Charles Dickens (1812-70)?', - 'extra.response_size': 162, - 'extra.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 17, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-qotd', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 162.0, - 'extra.quote': '_Oh the nerves, the nerves; the mysteries of this machine ' - 'called man!?? Oh the little that unhinges it, poor creatures ' - 'that we are!_?? Charles Dickens (1812-70)?', - 'extra.response_size': 162, - 'extra.tag': 'qotd', - 'feed.name': 'Open QOTD', - 'protocol.application': 'qotd', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 17, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py deleted file mode 100644 index 23d11ce99..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_quic.py +++ /dev/null @@ -1,118 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_quic.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible QUIC Report', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_quic-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517311, - 'extra.tag' : 'quic', - 'extra.version_field_1' : 'Q050', - 'extra.version_field_3' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 5607, - 'source.geolocation.cc' : 'UK', - 'source.geolocation.city' : 'LONDON', - 'source.geolocation.region' : 'LONDON', - 'source.ip' : '176.255.0.0', - 'source.port' : 443, - 'source.reverse_dns' : 'test1.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517311, - 'extra.tag' : 'quic', - 'extra.version_field_1' : 'Q050', - 'extra.version_field_2' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 6327, - 'source.geolocation.cc' : 'CA', - 'source.geolocation.city' : 'MEACHAM', - 'source.geolocation.region' : 'SASKATCHEWAN', - 'source.ip' : '24.244.0.0', - 'source.port' : 443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-quic', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.source.naics' : 517919, - 'extra.tag' : 'quic', - 'extra.version_field_2' : 'Q050', - 'extra.version_field_3' : 'Q046', - 'extra.version_field_4' : 'Q043', - 'feed.name' : 'Accessible QUIC Report', - 'protocol.transport' : 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 20940, - 'source.geolocation.cc' : 'JP', - 'source.geolocation.city' : 'OSAKA', - 'source.geolocation.region' : 'OSAKA', - 'source.ip' : '23.60.0.0', - 'source.port' : 443, - 'source.reverse_dns' : 'test3.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T14:31:17+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py deleted file mode 100644 index 7c052c451..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_radmin.py +++ /dev/null @@ -1,236 +0,0 @@ -# SPDX-FileCopyrightText: 2020 sinus-x -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), "testdata/scan_radmin.csv")) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = { - "feed.name": "Accessible Radmin", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2020-08-19T00:00:00+00:00", - "extra.file_name": "2020-08-19-scan_radmin-test-test.csv", -} - -EVENTS = [ - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin (Details Unknown)", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.asn": 701, - "source.geolocation.cc": "US", - "source.geolocation.city": "BROOKLYN", - "source.geolocation.region": "NEW YORK", - "source.ip": "74.101.218.75", - "source.port": 4899, - "source.reverse_dns": "static-74-101-218-75.nycmny.fios.verizon.net", - "time.source": "2020-07-06T13:55:26+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[1]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.asn": 56618, - "source.geolocation.cc": "RU", - "source.geolocation.city": "MURMANSK", - "source.geolocation.region": "MURMANSKAYA OBLAST", - "source.ip": "192.162.189.171", - "source.port": 4899, - "source.reverse_dns": "rubin.an.ru", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[2]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin (Details Unknown)", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "CN", - "source.geolocation.city": "BEIJING", - "source.geolocation.region": "BEIJING SHI", - "source.asn": 4808, - "source.ip": "111.197.143.69", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[3]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "KR", - "source.geolocation.city": "DAEIN-DONG", - "source.geolocation.region": "GWANGJU-GWANGYEOKSI", - "source.asn": 4766, - "source.ip": "121.147.215.220", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[4]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "KR", - "source.geolocation.city": "DAEIN-DONG", - "source.geolocation.region": "GWANGJU-GWANGYEOKSI", - "source.asn": 4766, - "source.ip": "121.147.215.178", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[5]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "CN", - "source.geolocation.city": "CHONGQING", - "source.geolocation.region": "CHONGQING SHI", - "source.asn": 9808, - "source.ip": "183.230.5.219", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[6]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "RU", - "source.geolocation.city": "MOSCOW", - "source.geolocation.region": "MOSKVA", - "source.asn": 34300, - "source.ip": "85.93.154.74", - "source.port": 4899, - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[7]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517311, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "BE", - "source.geolocation.city": "BRASSCHAAT", - "source.geolocation.region": "ANTWERPEN", - "source.asn": 5432, - "source.ip": "81.246.135.247", - "source.port": 4899, - "source.reverse_dns": "247.135-246-81.adsl-dyn.isp.belgacom.be", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[8]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, - { - "__type": "Event", - "feed.name": "Accessible Radmin", - "classification.identifier": "accessible-radmin", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.naics": 517312, - "extra.tag": "radmin", - "extra.version": "Radmin v3.X Radmin Authentication", - "feed.name": "Accessible Radmin", - "protocol.transport": "tcp", - "source.geolocation.cc": "ES", - "source.geolocation.city": "LAS PALMAS DE GRAN CANARIA", - "source.geolocation.region": "LAS PALMAS", - "source.asn": 12430, - "source.ip": "46.27.146.22", - "source.port": 4899, - "source.reverse_dns": "static-22-146-27-46.ipcom.comunitel.net", - "time.source": "2020-07-06T13:55:27+00:00", - "raw": utils.base64_encode("\n".join((EXAMPLE_LINES[0], EXAMPLE_LINES[9]))), - "time.observation": "2020-08-19T00:00:00+00:00", - }, -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == "__main__": - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py deleted file mode 100644 index 28a4a02c2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdp.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rdp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible RDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rdp-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible RDP', - "classification.identifier": "open-rdp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.bluekeep_vulnerable": False, - "extra.cert_expiration_date": "2019-10-29 02:22:06", - "extra.cert_issue_date": "2019-04-29 02:22:06", - "extra.cert_length": 5678, - "extra.cert_serial_number": "1EF2B37AF850C9BF4E88F18177001D6B", - "extra.cve20190708_vulnerable": False, - "extra.issuer_common_name": "KABESRV.KABE.local", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "BC:6E:C3:E2:98:22:EC:BA:5B:30:E2:53:FD:4A:9D:FF", - "extra.naics": 517311, - "extra.rdp_protocol": "RDP", - "extra.sha1_fingerprint": "EC:BB:4D:DB:9F:0C:D3:FF:5B:49:EA:B1:56:62:B6:A7:5D:60:54:42", - "extra.sha256_fingerprint": "B7:C9:F4:07:D5:C0:75:1D:EA:0C:40:E7:26:39:C2:30:C6:13:83:7E:18:46:D8:E9:4C:45:3F:88:1B:0B:70:76", - "extra.sha512_fingerprint": "08:AC:75:FA:EB:A3:2B:44:15:DE:6D:A7:0B:C0:AE:17:94:F3:55:D9:EC:70:AC:5B:B7:94:79:F0:D7:84:83:89:CB:A9:11:E0:08:D7:54:4D:33:85:89:D2:A8:DD:9D:15:F4:CC:95:DE:6A:E3:DF:6B:FA:8B:27:E3:DA:16:AF:0A", - "extra.signature_algorithm": "sha256WithRSAEncryption", - "extra.ssl_version": 2, - "extra.subject_common_name": "KABESRV.KABE.local", - "extra.tag": "rdp", - "protocol.application": "rdp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.178", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T15:45:51+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible RDP', - "classification.identifier": "open-rdp", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.bluekeep_vulnerable": False, - "extra.cert_expiration_date": "2019-10-16 06:15:20", - "extra.cert_issue_date": "2019-04-16 06:15:20", - "extra.cert_length": 5678, - "extra.cert_serial_number": "3FF3EBC5CF154BA54D128A8548C8AAF5", - "extra.cve20190708_vulnerable": False, - "extra.issuer_common_name": "RAMBLA01.rambla.local", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "38:73:6A:B3:AA:41:69:C9:BA:E7:3D:D7:40:16:F8:AA", - "extra.naics": 517311, - "extra.rdp_protocol": "RDP", - "extra.sector": "Information Technology", - "extra.sha1_fingerprint": "7A:67:1F:F8:87:C6:B0:AC:A9:84:15:B7:40:EC:CB:19:AA:E3:19:52", - "extra.sha256_fingerprint": "8F:CD:7D:C4:80:2D:8D:9B:06:A0:40:18:9F:ED:73:7A:BA:83:55:BE:1B:56:83:A2:97:DF:BB:B4:06:57:CB:F1", - "extra.sha512_fingerprint": "E8:9B:9A:93:69:B4:58:01:D8:46:C2:DC:01:20:1E:DD:93:E1:EB:E3:9D:6B:65:A0:C5:00:6C:A4:44:08:FE:A4:A6:19:FF:55:79:F2:AA:61:68:C8:1C:B0:CE:78:EB:84:DD:29:9D:64:2F:4E:25:31:3A:6C:B8:02:C9:AF:F5:1F", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.ssl_version": 2, - "extra.subject_common_name": "RAMBLA01.rambla.local", - "extra.tag": "rdp", - "protocol.application": "rdp", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.233", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T15:45:51+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py deleted file mode 100644 index 54be35a26..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rdpeudp.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rdpeudp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible MS RDPEUDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rdpeudp-test-geo.csv", - } - -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '05b28c0c', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3389, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '053d355f', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3389, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 77.0, - 'extra.response_size': 1232, - 'extra.sessionid': '0567a8cb', - 'extra.tag': 'rdpeudp', - 'feed.name': 'Accessible MS RDPEUDP', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3389, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py deleted file mode 100644 index 04552e2ec..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_redis.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_redis.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Redis', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_redis-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Open Redis', - "classification.identifier": "open-redis", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.build_id": "26069fb482f6334b", - "extra.connected_clients": "50", - "extra.gcc_version": "4.7.2", - "extra.git_sha1": "00000000", - "extra.mode": "standalone", - "extra.multiplexing_api": "epoll", - "extra.naics": 541512, - "extra.os.name": "Linux 3.2.0-4-amd64 x86_64", - "extra.process_id": "2127", - "extra.run_id": "d440b0b2fb3d1db655ad607e11e6f38011a0f599", - "extra.sic": 737999, - "extra.tag": "redis", - "extra.uptime": 27946314, - "extra.version": "2.8.19", - "protocol.application": "redis", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 201229, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.152", - "source.port": 6379, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:42:33+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Open Redis', - "classification.identifier": "open-redis", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.build_id": "e41bf84a0cecf09d", - "extra.connected_clients": "25376", - "extra.gcc_version": "4.8.4", - "extra.git_sha1": "00000000", - "extra.mode": "standalone", - "extra.multiplexing_api": "epoll", - "extra.os.name": "Linux 3.18.24-sirzion x86_64", - "extra.process_id": "343519", - "extra.run_id": "53d63f23511dc0080b49aaa8e8203d65619f1c8c", - "extra.tag": "redis", - "extra.uptime": 310556, - "extra.version": "3.0.6", - "protocol.application": "redis", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 12586, - "source.geolocation.cc": "DE", - "source.geolocation.city": "FRANKFURT AM MAIN", - "source.geolocation.region": "HESSEN", - "source.ip": "198.51.100.67", - "source.port": 6379, - "source.reverse_dns": "198-51-100-67.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2016-07-24T00:42:43+00:00" - }, - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py deleted file mode 100644 index e2a961f71..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_rsync.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_rsync.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Rsync', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_rsync-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 873, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 873, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.has_password': 'N', - 'extra.module': 'system|Backup system;system_full|Backup full ' - 'system;mysql|Backup virtual mysql;netadmin|Backup virtual ' - 'netadmin;', - 'extra.tag': 'rsync', - 'feed.name': 'Accessible Rsync', - 'protocol.application': 'rsync', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 873, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py deleted file mode 100644 index 6b972ec5d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_sip.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_sip.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-SIP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_sip-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.sip_allow': 'INVITE,ACK,BYE,CANCEL,REGISTER', - 'extra.amplification': 15.57, - 'extra.content_length': 0, - 'extra.response_size': 109, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '489', - 'extra.sip_reason': 'Event Package Not Supported', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 5060, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 62.57, - 'extra.content_length': 364, - 'extra.content_type': 'text/plain', - 'extra.response_size': 438, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '400', - 'extra.sip_reason': 'Bad Request', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 5060, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-sip', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.57, - 'extra.content_length': 0, - 'extra.response_size': 46, - 'extra.sip': 'SIP/2.0', - 'extra.sip_code': '400', - 'extra.sip_reason': 'Bad Request', - 'extra.tag': 'sip', - 'feed.name': 'Accessible-SIP', - 'protocol.application': 'sip', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 5060, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py deleted file mode 100644 index f05973cf5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_slp.py +++ /dev/null @@ -1,137 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_slp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SLP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_slp-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 427, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 427, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-slp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.error_code': '5', - 'extra.error_code_text': 'Unsupported SLP SPI', - 'extra.flags': '0x0000', - 'extra.function': '2', - 'extra.function_text': 'Service reply', - 'extra.language_tag': 'en', - 'extra.language_tag_length': '2', - 'extra.raw_response': 'MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA==', - 'extra.response_size': 40, - 'extra.tag': 'slp', - 'extra.version': '2', - 'extra.xid': '5', - 'feed.name': 'Accessible SLP', - 'protocol.application': 'slp', - 'protocol.transport': 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 427, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py deleted file mode 100644 index 921525122..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_smb.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SMB', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_smb-test-geo.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py deleted file mode 100644 index cae83d273..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smb_json.py +++ /dev/null @@ -1,123 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Sebastian Waldbauer -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest -import json - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser_json import ShadowserverJSONParserBot -from intelmq.tests.bots.parsers.shadowserver.test_testdata import csvtojson - -EXAMPLE_FILE = csvtojson(os.path.join(os.path.dirname(__file__), 'testdata/scan_smb.csv')) - -EXAMPLE_REPORT = { - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_smb-test-geo.json", - } - -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[0]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'open-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode(json.dumps([json.loads(EXAMPLE_FILE)[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverJSONParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverJSONParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverJSONParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py deleted file mode 100644 index 4428420cf..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_smtp_vulnerable.py +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-FileCopyrightText: 2021 Mikk Margus Möll -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_smtp_vulnerable.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Vulnerable SMTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2021-07-08T00:00:00+00:00", - "extra.file_name": "2021-07-08-scan_smtp_vulnerable-test-test.csv", - } - -EVENTS = [ - { - '__type' : 'Event', - 'classification.identifier' : 'vulnerable-smtp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.banner' : '220 smtp-server.invalid ESMTP Exim 4.80 Wed, 11 Jun 2021 10:00:00 +0300|', - 'extra.tag' : 'smtp;21nails', - 'feed.name' : 'Vulnerable SMTP', - 'protocol.application' : 'smtp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 12345, - 'source.geolocation.cc' : 'EE', - 'source.geolocation.city' : 'TALLINN', - 'source.geolocation.region' : 'HARJUMAA', - 'source.ip' : '1.2.3.4', - 'source.port' : 25, - 'source.reverse_dns' : 'smtp-server.invalid', - 'time.observation' : '2021-07-08T00:00:00+00:00', - 'time.source' : '2021-07-08T11:58:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'vulnerable-smtp', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.banner' : '220 smtp-out.invalid, ESMTP EXIM 4.86_2|', - 'extra.tag' : 'smtp;21nails', - 'feed.name' : 'Vulnerable SMTP', - 'protocol.application' : 'smtp', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 23456, - 'source.geolocation.cc' : 'EE', - 'source.geolocation.city' : 'TALLINN', - 'source.geolocation.region' : 'HARJUMAA', - 'source.ip' : '5.6.7.8', - 'source.port' : 25, - 'source.reverse_dns' : 'smtp-out.invalid', - 'time.observation' : '2021-07-08T00:00:00+00:00', - 'time.source' : '2021-07-08T11:58:44+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py deleted file mode 100644 index e6da5b34f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_snmp.py +++ /dev/null @@ -1,120 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_snmp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SNMP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_snmp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.94, - 'extra.community': 'public', - 'extra.response_size': 165, - 'extra.sysdesc': 'Linux localhost 3.18.20 #1 SMP Mon Jul 9 14:11:21 CST 2018 ' - 'armv7l', - 'extra.tag': 'snmp', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 161, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.35, - 'extra.community': 'public', - 'extra.device_sector': 'consumer', - 'extra.device_type': 'router', - 'extra.device_vendor': 'MikroTik', - 'extra.response_size': 115, - 'extra.sysdesc': 'RouterOS CCR1009-8G-1S-1S+', - 'extra.tag': 'snmp,iot', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 161, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-snmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.0, - 'extra.community': 'public', - 'extra.response_size': 85, - 'extra.tag': 'snmp', - 'extra.version': '2', - 'feed.name': 'Open SNMP', - 'protocol.application': 'snmp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 161, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py deleted file mode 100644 index 067602aa1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_socks.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_socks.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SOCKS', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_socks-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks4', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 1080, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks5', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 1080, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-socks', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Retail Trade', - 'feed.name' : 'Open SOCKS', - 'protocol.application' : 'socks4', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 1080, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2010-02-10T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py deleted file mode 100644 index 0811f15ed..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssdp.py +++ /dev/null @@ -1,136 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssdp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open SSDP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssdp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 3.35, - 'extra.cache_control': 'max-age=100', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node01.example.com', - 'extra.location': 'http://192.168.200.254:49152/description.xml', - 'extra.response_size': 325, - 'extra.search_target': 'upnp:rootdevice', - 'extra.sector': 'Government', - 'extra.server': 'Linux/2.6.26, UPnP/1.0, Portable SDK for UPnP devices/1.3.1', - 'extra.systime': 'Sun, 21 Aug 2022 09:51:13 GMT', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:28802880-2880-1880-a880-001bc502f600::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 60194, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 2.71, - 'extra.cache_control': 'max-age = 1800', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node02.example.com', - 'extra.location': 'http://95.160.216.14:52235/dmr/SamsungMRDesc.xml', - 'extra.response_size': 263, - 'extra.search_target': 'upnp:rootdevice', - 'extra.server': 'Linux/9.0 UPnP/1.0 PROTOTYPE/1.0', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:f144ca92-6816-94b5-b95f-b58180834044::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 38732, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ssdp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 4.79, - 'extra.cache_control': 'max-age=1800', - 'extra.header': 'HTTP/1.1 200 OK', - 'extra.host': 'node03.example.com', - 'extra.location': 'http://192.168.1.3:8008/ssdp/device-desc.xml', - 'extra.response_size': 465, - 'extra.search_target': 'upnp:rootdevice', - 'extra.sector': 'Government', - 'extra.server': 'Linux/3.10.79, UPnP/1.0, Portable SDK for UPnP ' - 'devices/1.6.18', - 'extra.systime': 'Sun, 03 Jan 2016 21:37:50 GMT', - 'extra.tag': 'ssdp', - 'extra.unique_service_name': 'uuid:62fa0fc8-079d-d00f-2e22-59b49fb488f9::upnp:rootdevice', - 'feed.name': 'Open SSDP', - 'protocol.application': 'ssdp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 57626, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py deleted file mode 100644 index a01383713..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssh.py +++ /dev/null @@ -1,182 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssh.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SSH', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ssh-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ecdsa-sha2-nistp256', - 'extra.available_ciphers' : 'chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com, aes128-cbc, aes192-cbc, aes256-cbc, blowfish-cbc, cast128-cbc, 3des-cbc', - 'extra.available_compression' : 'none, zlib@openssh.com', - 'extra.available_kex' : 'curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha256, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1', - 'extra.available_mac' : 'umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1', - 'extra.ecdsa_curve' : 'P-256', - 'extra.ecdsa_curve25519' : '1xx7ASut7BF4ED8b592bebZBMBKTCzOsmbH4cjwx/0U=', - 'extra.ecdsa_public_key_b' : 'WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=', - 'extra.ecdsa_public_key_gx' : 'axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=', - 'extra.ecdsa_public_key_gy' : 'T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=', - 'extra.ecdsa_public_key_length' : '256', - 'extra.ecdsa_public_key_n' : '/////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=', - 'extra.ecdsa_public_key_p' : '/////wAAAAEAAAAAAAAAAAAAAAD///////////////8=', - 'extra.ecdsa_public_key_x' : 'NIQdotpy2HAHfSu0DjEmA3cb3NeQKabZaFX9OU0GsPY=', - 'extra.ecdsa_public_key_y' : '0fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=', - 'extra.selected_cipher' : 'aes128-ctr', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'curve25519-sha256@libssh.org', - 'extra.selected_mac' : 'hmac-sha2-256', - 'extra.server_cookie' : 'bGjsifbPIDWT7tAu8BMjyg==', - 'extra.server_host_key' : 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDSEHaLacthwB30rtA4xJgN3G9zXkCmm2WhV/TlNBrD20fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=', - 'extra.server_host_key_sha256' : 'a6e4e1c16ba25d51bcddc58a6e16797144575dd18d02d9dedf75093d2b15c557', - 'extra.server_signature_raw' : 'AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAABKAAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=', - 'extra.server_signature_value' : 'AAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=', - 'extra.serverid_raw' : 'SSH-2.0-OpenSSH_7.4', - 'extra.serverid_software' : 'OpenSSH_7.4', - 'extra.serverid_version' : '2.0', - 'extra.source.naics' : 454110, - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 16509, - 'source.geolocation.cc' : 'JP', - 'source.geolocation.city' : 'TOKYO', - 'source.geolocation.region' : 'TOKYO', - 'source.ip' : '18.179.0.0', - 'source.port' : 22, - 'source.reverse_dns' : 'ec2-18-179-0-0.ap-northeast-1.compute.amazonaws.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ssh-rsa', - 'extra.available_ciphers' : 'aes128-cbc, 3des-cbc, aes256-cbc, twofish256-cbc, twofish-cbc, twofish128-cbc, blowfish-cbc', - 'extra.available_compression' : 'none', - 'extra.available_kex' : 'diffie-hellman-group1-sha1', - 'extra.available_mac' : 'hmac-sha1-96, hmac-sha1, hmac-md5', - 'extra.device_vendor' : 'Arris', - 'extra.rsa_exponent' : '65537', - 'extra.rsa_length' : '1040', - 'extra.rsa_modulus' : 'g6tZBqqsHUFa8gjXKMg2waZy/6JLzNnfTZkdDo6a3E6Amdk2eVlGbdgRsDxNUCyWLJI4b+xsJX0OKoRKhw4N+cDK5Du4/S00Auylt8YrDi7lIpeBZjMT5KJerN/feOcwRqY8NJfwZiZ1A5brcWr5HXfOH6aRTMFzadr248VdUmFXPQ==', - 'extra.selected_cipher' : 'aes128-cbc', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'diffie-hellman-group1-sha1', - 'extra.selected_mac' : 'hmac-sha1', - 'extra.server_cookie' : 'Y4RQS9sdRgEFwNJKVP6bZg==', - 'extra.server_host_key' : 'AAAAB3NzaC1yc2EAAAADAQABAAAAgwCDq1kGqqwdQVryCNcoyDbBpnL/okvM2d9NmR0OjprcToCZ2TZ5WUZt2BGwPE1QLJYskjhv7GwlfQ4qhEqHDg35wMrkO7j9LTQC7KW3xisOLuUil4FmMxPkol6s39945zBGpjw0l/BmJnUDlutxavkdd84fppFMwXNp2vbjxV1SYVc9', - 'extra.server_host_key_sha256' : 'd53fedbfe92e631264629882b2e85bfd213ca4b07b824cd31f8de1fcb8d0ddcb', - 'extra.server_signature_raw' : 'AAAAB3NzaC1yc2EAAACCLQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==', - 'extra.server_signature_value' : 'LQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==', - 'extra.serverid_raw' : 'SSH-2.0-ARRIS_0.50', - 'extra.serverid_software' : 'ARRIS_0.50', - 'extra.serverid_version' : '2.0', - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey, password', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 11976, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MARSHALL', - 'source.geolocation.region' : 'TEXAS', - 'source.ip' : '170.10.0.0', - 'source.port' : 22, - 'source.reverse_dns' : '170-10-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssh', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.algorithm' : 'ssh-rsa', - 'extra.available_ciphers' : 'aes128-cbc, 3des-cbc, aes192-cbc, aes256-cbc', - 'extra.available_compression' : 'none', - 'extra.available_kex' : 'diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1', - 'extra.available_mac' : 'hmac-sha1, hmac-sha1-96, hmac-md5, hmac-md5-96', - 'extra.device_sector' : 'enterprise', - 'extra.device_vendor' : 'Cisco', - 'extra.rsa_exponent' : '65537', - 'extra.rsa_length' : '4096', - 'extra.rsa_modulus' : 'yFVwcChoYt+YGm8BzWYugcZbNQRrQ1VWRYcL4U6SSkyoVeE9h5wxRu/hQaWHo3PdsB9Nuln/riRyKZypFUEZ5zlffMyl1uvE8/jp8E/GgUHSyPkGAwu8C8BkX/nDolxAJKTK6djiZnvhsEPe6AXHBMHbto/b3GABUNPngjzX8D63GYcFW9NJLf5qC1UsVkXbAzM0IjQ2X9s3pfhUCAJeXAn2i0gEGtUyF8vEjNdwdG655aXciKrpEEtM1L/zy/+gLH4YC13kAYI7NVyH+qi/mXbULLOQClA7iYK1g3Et58jWUIPwgLfF3SLC57bt2wp/lRgNTv4FBi0tWvRqBnf5UQK5ZjgzbW3bO+Ju4cWgH/4M4NCxSceh4cLm5lQs01xB5feSh2ByqA7wrVDoFJu81LoMVo4bCz30+lH2QsLwmNtUhlWLKBD4k09g4bgBa4jPj0/Nya3rBR4GQ6LG6ltFQotm8wCkgbv76YWqk20nQ6NMYZFvSQm981JFtoHv3vxq48VeHDV0QvV0P12BCFprRf4B0otIvSsHl+LDeUxJAf+Nbw78gzncjyfCbWtCPbwaJQ8CeqnTBzj5TluaFvN8goG5lCTWJGfjIrwAZXOokv9NOqmIiMJJx3s22OX6GHfJAzje2ALLDsAiXBub4iCOdGdTfVbBpFL+bGTK9qfa8vE=', - 'extra.selected_cipher' : 'aes128-cbc', - 'extra.selected_compression' : 'none', - 'extra.selected_kex' : 'diffie-hellman-group14-sha1', - 'extra.selected_mac' : 'hmac-sha1', - 'extra.server_cookie' : 'Z2fOfWsrLlh76Y0bOqa1cw==', - 'extra.server_host_key' : 'AAAAB3NzaC1yc2EAAAADAQABAAACAQDIVXBwKGhi35gabwHNZi6Bxls1BGtDVVZFhwvhTpJKTKhV4T2HnDFG7+FBpYejc92wH026Wf+uJHIpnKkVQRnnOV98zKXW68Tz+OnwT8aBQdLI+QYDC7wLwGRf+cOiXEAkpMrp2OJme+GwQ97oBccEwdu2j9vcYAFQ0+eCPNfwPrcZhwVb00kt/moLVSxWRdsDMzQiNDZf2zel+FQIAl5cCfaLSAQa1TIXy8SM13B0brnlpdyIqukQS0zUv/PL/6AsfhgLXeQBgjs1XIf6qL+ZdtQss5AKUDuJgrWDcS3nyNZQg/CAt8XdIsLntu3bCn+VGA1O/gUGLS1a9GoGd/lRArlmODNtbds74m7hxaAf/gzg0LFJx6HhwubmVCzTXEHl95KHYHKoDvCtUOgUm7zUugxWjhsLPfT6UfZCwvCY21SGVYsoEPiTT2DhuAFriM+PT83JresFHgZDosbqW0VCi2bzAKSBu/vphaqTbSdDo0xhkW9JCb3zUkW2ge/e/GrjxV4cNXRC9XQ/XYEIWmtF/gHSi0i9KweX4sN5TEkB/41vDvyDOdyPJ8Jta0I9vBolDwJ6qdMHOPlOW5oW83yCgbmUJNYkZ+MivABlc6iS/006qYiIwknHezbY5foYd8kDON7YAssOwCJcG5viII50Z1N9VsGkUv5sZMr2p9ry8Q==', - 'extra.server_host_key_sha256' : '06ff3cce443ed832927576d982b69d5a526d0e63334c72e87201deda61679406', - 'extra.server_signature_raw' : 'AAAAB3NzaC1yc2EAAAIAlrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=', - 'extra.server_signature_value' : 'lrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=', - 'extra.serverid_raw' : 'SSH-1.99-Cisco-1.25', - 'extra.serverid_software' : 'Cisco-1.25', - 'extra.serverid_version' : '1.99', - 'extra.source.naics' : 517311, - 'extra.tag' : 'ssh', - 'extra.userauth_methods' : 'publickey, keyboard-interactive, password', - 'feed.name' : 'Accessible SSH', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 33363, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ORLANDO', - 'source.geolocation.region' : 'FLORIDA', - 'source.ip' : '72.17.0.0', - 'source.port' : 22, - 'source.reverse_dns' : '072-017-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T02:20:37+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py deleted file mode 100644 index f96c03e56..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl.py +++ /dev/null @@ -1,218 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssl.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible SSL', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_ssl-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: unknown error', - 'extra.browser_trusted' : False, - 'extra.cert_expiration_date' : '2038-01-19 03:14:07', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2014-06-23 09:56:32', - 'extra.cert_length' : 1024, - 'extra.cert_serial_number' : '168CAE', - 'extra.cert_valid' : True, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.content_length' : 131, - 'extra.content_type' : 'text/html', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : 'support', - 'extra.issuer_country' : 'US', - 'extra.issuer_email_address' : 'support@fortinet.com', - 'extra.issuer_locality_name' : 'Sunnyvale', - 'extra.issuer_organization_name' : 'Fortinet', - 'extra.issuer_organization_unit_name' : 'Certificate Authority', - 'extra.issuer_state_or_province_name' : 'California', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '99:45:1F:2E:AE:EB:88:91:27:43:33:79:FA:93:7D:CA', - 'extra.self_signed' : False, - 'extra.server_type' : 'xxxxxxxx-xxxxx', - 'extra.sha1_fingerprint' : '5A:3D:FF:06:F9:E9:25:37:57:F9:09:52:33:A4:85:15:24:2D:88:7F', - 'extra.sha256_fingerprint' : '35:AB:B6:76:2A:3D:17:B2:FB:40:45:1B:FC:0A:99:0A:6E:48:57:F7:30:0A:3B:B1:1A:E6:99:70:5B:7C:32:41', - 'extra.sha512_fingerprint' : '88:7B:16:DB:39:44:0C:47:0E:4A:8F:0B:C5:FB:4D:45:BC:93:5A:00:43:A1:D9:7F:05:1D:86:33:02:F8:FC:57:67:A6:1D:C0:FF:F7:D2:40:D8:9A:21:AE:4E:6D:DC:E7:FF:72:BF:13:CB:EE:A7:5F:CD:83:EA:8A:5E:FB:87:DD', - 'extra.signature_algorithm' : 'sha1WithRSAEncryption', - 'extra.source.naics' : 517311, - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : 'FGT60D4614030700', - 'extra.subject_country' : 'US', - 'extra.subject_email_address' : 'support@fortinet.com', - 'extra.subject_locality_name' : 'Sunnyvale', - 'extra.subject_organization_name' : 'Fortinet', - 'extra.subject_organization_unit_name' : 'FortiGate', - 'extra.subject_state_or_province_name' : 'California', - 'extra.tag' : 'ssl,vpn', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 4181, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'MILWAUKEE', - 'source.geolocation.region' : 'WISCONSIN', - 'source.ip' : '96.60.0.0', - 'source.port' : 10443, - 'source.reverse_dns' : '96-60-0-0.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_error' : 'x509: unknown error', - 'extra.browser_trusted' : False, - 'extra.cert_expiration_date' : '2023-02-06 01:01:34', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2022-01-04 01:01:34', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '36974C4C6B1B3785', - 'extra.cert_valid' : False, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'extra.content_type' : 'text/html; charset=UTF-8', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_connection' : 'keep-alive', - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : '1078-btb-tbi-HungHa-61d39c6d5a7e2', - 'extra.issuer_organization_name' : 'pfSense webConfigurator Self-Signed Certificate', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : '16:93:9A:F4:35:7F:9A:85:45:71:91:C7:7C:80:88:00', - 'extra.self_signed' : True, - 'extra.server_type' : 'nginx', - 'extra.set_cookie' : 'PHPSESSID=e15bdfa5739c36877608eb4cf46cc388; path=/; secure; HttpO', - 'extra.sha1_fingerprint' : 'A9:00:BB:E1:54:4D:56:54:59:F1:B7:EA:F1:1A:D5:36:5C:63:90:8E', - 'extra.sha256_fingerprint' : '38:85:F0:44:1E:AD:84:B8:2F:43:68:BA:AC:EE:17:13:A4:BF:86:1D:48:75:7E:22:FA:08:4C:28:5F:AC:3E:5F', - 'extra.sha512_fingerprint' : 'AE:1B:4F:D1:E4:C0:35:9D:2A:4F:7A:37:B8:7B:11:9D:84:25:23:21:AB:EF:B2:0F:DC:C9:F2:A3:72:28:92:E1:74:72:FA:E1:09:6C:E1:F6:B6:E3:A7:61:1C:58:89:34:D7:06:5C:3D:0A:A7:F6:CC:8A:D6:24:D0:04:4C:03:02', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 517311, - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_common_name' : '1078-btb-tbi-HungHa-61d39c6d5a7e2', - 'extra.subject_organization_name' : 'pfSense webConfigurator Self-Signed Certificate', - 'extra.tag' : 'ssl', - 'extra.transfer_encoding' : 'chunked', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 45899, - 'source.geolocation.cc' : 'VN', - 'source.geolocation.city' : 'THAI BINH', - 'source.geolocation.region' : 'THAI BINH', - 'source.ip' : '113.160.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-ssl', - 'classification.taxonomy' : 'other', - 'classification.type' : 'other', - 'extra.browser_trusted' : True, - 'extra.cert_expiration_date' : '2022-11-06 15:30:28', - 'extra.cert_expired' : False, - 'extra.cert_issue_date' : '2021-10-07 15:30:28', - 'extra.cert_length' : 2048, - 'extra.cert_serial_number' : '7B388364A24B88E77E5553B5C6748100', - 'extra.cert_valid' : True, - 'extra.cipher_suite' : 'TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA', - 'extra.content_length' : 131, - 'extra.content_type' : 'text/html', - 'extra.freak_vulnerable' : False, - 'extra.handshake' : 'TLSv1.2', - 'extra.http_code' : 200, - 'extra.http_date' : '2022-01-10T00:01:44+00:00', - 'extra.http_reason' : 'OK', - 'extra.http_response_type' : 'HTTP/1.1', - 'extra.issuer_common_name' : 'Entrust Certification Authority - L1K', - 'extra.issuer_country' : 'US', - 'extra.issuer_organization_name' : 'Entrust, Inc.', - 'extra.issuer_organization_unit_name' : '(c) 2012 Entrust, Inc. - for authorized use only', - 'extra.key_algorithm' : 'rsaEncryption', - 'extra.md5_fingerprint' : 'E7:34:BC:92:84:FA:39:DE:E1:46:6C:27:DA:5A:01:F4', - 'extra.self_signed' : False, - 'extra.server_type' : 'xxxxxxxx-xxxxx', - 'extra.sha1_fingerprint' : 'AD:19:B2:1C:CB:88:70:9B:DB:8E:7E:F5:65:50:13:D6:43:6C:BE:6E', - 'extra.sha256_fingerprint' : '9A:64:73:0B:8A:FA:DE:22:D4:6D:5A:C6:C4:6F:D4:A4:2A:28:FA:41:1E:FF:81:DC:D4:D9:00:FD:78:DF:C4:DD', - 'extra.sha512_fingerprint' : '9A:B7:BD:68:7D:F3:E7:C1:B7:D3:F4:2F:01:B6:C4:77:90:A3:2B:1E:C0:89:F5:08:EC:43:87:35:60:36:D4:87:61:AA:B8:A8:B3:8A:E9:F1:04:AA:5B:67:12:FF:63:D5:14:80:77:6E:8F:7D:C3:E2:3A:F3:13:DF:08:43:6C:B0', - 'extra.signature_algorithm' : 'sha256WithRSAEncryption', - 'extra.source.naics' : 454110, - 'extra.source.sector' : 'Retail Trade', - 'extra.ssl_poodle' : False, - 'extra.ssl_version' : 2, - 'extra.subject_country' : 'US', - 'extra.subject_locality_name' : 'Hanover', - 'extra.subject_organization_name' : 'Ciena Corporation', - 'extra.subject_state_or_province_name' : 'Maryland', - 'extra.tag' : 'ssl,vpn', - 'extra.validation_level' : 'OV', - 'feed.name' : 'Accessible SSL', - 'protocol.application': 'https', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 14618, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'ASHBURN', - 'source.geolocation.region' : 'VIRGINIA', - 'source.ip' : '34.224.0.0', - 'source.port' : 10443, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T00:01:42+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py deleted file mode 100644 index 42221bda2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_freak.py +++ /dev/null @@ -1,136 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ssl_freak.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SSL FREAK Vulnerable Servers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssl_freak-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'SSL FREAK Vulnerable Servers', - 'protocol.application': 'https', - "classification.identifier": "ssl-freak", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.browser_error": "x509: unknown error", - "extra.browser_trusted": False, - "extra.cert_expiration_date": "2032-05-05 00:01:19", - "extra.cert_expired": False, - "extra.cert_issue_date": "2012-05-10 00:01:19", - "extra.cert_length": 1024, - "extra.cert_serial_number": "4FAB054F", - "extra.cert_valid": True, - "extra.cipher_suite": "TLS_RSA_WITH_RC4_128_SHA", - "extra.content_type": "text/html", - "extra.freak_cipher_suite": "TLS_RSA_EXPORT_WITH_RC4_40_MD5", - "extra.freak_vulnerable": True, - "extra.handshake": "TLSv1.0", - "extra.http_code": 200, - "extra.http_date": "2018-04-23T13:25:26+00:00", - "extra.http_reason": "OK", - "extra.http_response_type": "HTTP/1.1", - "extra.issuer_common_name": "usg50_B0B2DC2FA69D", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE", - "extra.self_signed": True, - "extra.sha1_fingerprint": "14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2", - "extra.sha256_fingerprint": "57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1", - "extra.sha512_fingerprint": "E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.subject_common_name": "usg50_B0B2DC2FA69D", - "extra.tag": "ssl-freak", - "extra.transfer_encoding": "chunked", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 8447, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.232", - "source.port": 443, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2018-04-23T13:25:21+00:00" - }, - {'__type': 'Event', - 'feed.name': 'SSL FREAK Vulnerable Servers', - 'protocol.application': 'https', - "classification.identifier": "ssl-freak", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.browser_error": "x509: unknown error", - "extra.browser_trusted": False, - "extra.cert_expiration_date": "2029-12-27 00:00:53", - "extra.cert_expired": False, - "extra.cert_issue_date": "2010-01-01 00:00:53", - "extra.cert_length": 1024, - "extra.cert_serial_number": "4B3D3B35", - "extra.cert_valid": True, - "extra.cipher_suite": "TLS_RSA_WITH_RC4_128_SHA", - "extra.content_type": "text/html", - "extra.freak_cipher_suite": "TLS_RSA_EXPORT_WITH_RC4_40_MD5", - "extra.freak_vulnerable": True, - "extra.handshake": "TLSv1.0", - "extra.http_code": 200, - "extra.http_date": "2018-04-23T13:25:29+00:00", - "extra.http_reason": "OK", - "extra.http_response_type": "HTTP/1.1", - "extra.issuer_common_name": "usg20w_C86C870287EC", - "extra.key_algorithm": "rsaEncryption", - "extra.md5_fingerprint": "1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE", - "extra.self_signed": True, - "extra.sha1_fingerprint": "14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2", - "extra.sha256_fingerprint": "57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1", - "extra.sha512_fingerprint": "E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87", - "extra.signature_algorithm": "sha1WithRSAEncryption", - "extra.subject_common_name": "usg20w_C86C870287EC", - "extra.tag": "ssl-freak", - "extra.transfer_encoding": "chunked", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 12577, - "source.geolocation.cc": "AT", - "source.geolocation.city": "BADEN", - "source.geolocation.region": "NIEDEROSTERREICH", - "source.ip": "198.51.100.224", - "source.port": 443, - "source.reverse_dns": "198-51-100-224.example.net", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2018-04-23T13:25:26+00:00" - }] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py deleted file mode 100644 index 41535e67a..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ssl_poodle.py +++ /dev/null @@ -1,91 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_ssl_poodle.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SSL POODLE Vulnerable Servers', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ssl_poodle-test-geo.csv", - } -EVENTS = [{'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'ssl-poodle', - 'extra.browser_error': 'x509: unknown error', - 'extra.browser_trusted': False, - 'extra.cert_expiration_date': '2034-06-20 00:00:42', - 'extra.cert_expired': False, - 'extra.cert_issue_date': '2014-06-25 00:00:42', - 'extra.cert_length': 1024, - 'extra.cert_serial_number': '53AA112A', - 'extra.cert_valid': True, - 'extra.cipher_suite': 'TLS_RSA_WITH_RC4_128_SHA', - 'extra.content_type': 'text/html', - 'extra.handshake': 'TLSv1.0', - 'extra.http_code': 200, - 'extra.http_date': '2018-08-08T00:51:44+00:00', - 'extra.http_reason': 'OK', - 'extra.http_response_type': 'HTTP/1.1', - 'extra.issuer_common_name': 'usg20_107BEF394BA5', - 'extra.key_algorithm': 'rsaEncryption', - 'extra.md5_fingerprint': '33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC', - 'extra.self_signed': True, - 'extra.sha1_fingerprint': '04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3', - 'extra.sha256_fingerprint': '16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E', - 'extra.sha512_fingerprint': '0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE', - 'extra.signature_algorithm': 'sha1WithRSAEncryption', - 'extra.ssl_poodle': True, - 'extra.ssl_version': 2, - 'extra.subject_common_name': 'usg20_107BEF394BA5', - 'extra.tag': 'ssl-poodle', - 'extra.transfer_encoding': 'chunked', - 'feed.name': 'SSL POODLE Vulnerable Servers', - 'protocol.application': 'https', - 'source.asn': 65540, - 'source.geolocation.cc': 'AT', - 'source.geolocation.city': 'VIENNA', - 'source.geolocation.region': 'WIEN', - 'source.ip': '203.0.113.85', - 'source.port': 8443, - 'source.reverse_dns': 'example.com', - 'time.source': '2018-08-08T00:51:42+00:00', - "time.observation": "2015-01-01T00:00:00+00:00", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - '__type': 'Event', - }, - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py deleted file mode 100644 index 7fd5f6ec2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_stun.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_stun.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_stun-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 5.4, - 'extra.fingerprint': '0xfaedd06e', - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '192.168.0.1', - 'extra.mapped_family': '01', - 'extra.mapped_port': 3243, - 'extra.message_length': 88, - 'extra.message_type': '0101', - 'extra.response_size': 108, - 'extra.software': "Coturn-4.5.1.1 'dan Eider'", - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '192.168.0.1', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 3243, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3478, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 5.4, - 'extra.fingerprint': '0x21128641', - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '51.77.39.195', - 'extra.mapped_family': '01', - 'extra.mapped_port': 45877, - 'extra.message_length': 88, - 'extra.message_type': '0101', - 'extra.response_size': 108, - 'extra.software': "Coturn-4.5.1.1 'dan Eider'", - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '192.168.0.2', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 45877, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3478, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-stun', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'extra.amplification': 4.8, - 'extra.magic_cookie': '2112a442', - 'extra.mapped_address': '192.168.0.3', - 'extra.mapped_family': '01', - 'extra.mapped_port': 16321, - 'extra.message_length': 76, - 'extra.message_type': '0101', - 'extra.response_size': 96, - 'extra.software': "ApolloProxy-1.20.1.28 'sunflower'", - 'extra.tag': 'stun', - 'extra.transaction_id': '000000000000000000000000', - 'extra.xor_mapped_address': '188.68.240.32', - 'extra.xor_mapped_family': '01', - 'extra.xor_mapped_port': 16321, - 'feed.name': 'Accessible-Session-Traversal-Utilities-for-NAT', - 'protocol.application': 'session traversal utilities for nat', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3478, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py deleted file mode 100644 index 9b7e1fd3d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_synfulknock.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_synfulknock.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'SYNful Knock', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-scan_synfulknock-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '3cfdfec601e4700f6a9a2000080045000034c3780000f706789442099555b869f7ee0050b20800000000000c123e8012200002aa0000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - 'source.asn' : 18885, - 'source.geolocation.cc' : 'US', - 'source.geolocation.city' : 'JERSEY CITY', - 'source.geolocation.region' : 'NEW JERSEY', - 'source.ip' : '66.9.0.0', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:18:23+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '90e2baaf0b84700f6a9a200008004500003434100000f2064382d58337d2b8698b720050916200000000000c123e8012200059d50000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - 'source.asn' : 35805, - 'source.geolocation.cc' : 'GE', - 'source.geolocation.city' : 'TBILISI', - 'source.geolocation.region' : 'TBILISI', - 'source.ip' : '213.131.0.0', - 'source.port' : 80, - 'source.reverse_dns' : 'host-213-131-55-210-customer.wanex.net', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:19:17+00:00' -}, -{ - '__type' : 'Event', - 'classification.identifier' : 'open-synfulknock', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.ack_number' : 791102, - 'extra.raw_packet' : '90e2bab9cfd4700f6a9a20000800450000340f1d0000ea068bdad5b2e6914a522f360050eb5200000000000c123e801220001b4a0000020405b40101040201030305', - 'extra.tag' : 'synfulknock', - 'extra.tcp_flags' : '4608', - 'extra.window_size' : 8192, - 'feed.name' : 'SYNful Knock', - 'protocol.transport' : 'tcp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[3]])), - 'source.asn' : 29256, - 'source.geolocation.cc' : 'SY', - 'source.geolocation.city' : 'DAMASCUS', - 'source.geolocation.region' : 'DIMASHQ', - 'source.ip' : '213.178.0.0', - 'source.port' : 80, - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2022-01-10T09:27:39+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py deleted file mode 100644 index 66408db4c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_telnet.py +++ /dev/null @@ -1,87 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_telnet.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible Telnet', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_telnet-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible Telnet', - "classification.identifier": "open-telnet", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "|MikroTik v6.5|Login:", - "extra.tag": "telnet-alt", - "protocol.application": "telnet", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.145", - "source.port": 5678, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T12:27:34+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible Telnet', - "classification.identifier": "open-telnet", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "|MikroTik v6.45.3 (stable)|Login:", - "extra.tag": "telnet-alt", - "protocol.application": "telnet", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 20255, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.145", - "source.port": 5678, - "source.reverse_dns": "example.local", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T12:27:40+00:00" - }] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py deleted file mode 100644 index 3cf3688f9..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_tftp.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_tftp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open TFTP', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_tftp-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.57, - 'extra.error': 'Not defined', - 'extra.errormessage': 'Get not supported', - 'extra.opcode': '5', - 'extra.size': 22, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 35067, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.36, - 'extra.error': 'File not found', - 'extra.errorcode': '1', - 'extra.errormessage': 'File not found', - 'extra.opcode': '5', - 'extra.size': 19, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 56709, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-tftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 1.5, - 'extra.error': 'Access violation', - 'extra.errorcode': '2', - 'extra.errormessage': 'Access violation', - 'extra.opcode': '5', - 'extra.size': 21, - 'extra.tag': 'tftp', - 'feed.name': 'Open TFTP', - 'protocol.application': 'tftp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 32785, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py deleted file mode 100644 index 396bff1e3..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ubiquiti.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ubiquiti.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Open Ubiquiti', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2018-03-04T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ubiquiti-test-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 37.0, - 'extra.essid': 'Kachine-Meta-Lidia-Tereixa', - 'extra.firmwarerev': 'XS5.ar2313.v3.5.4494.091109.1459', - 'extra.mac_address': '00156db98c3a', - 'extra.model': 'NS5', - 'extra.radio_name': 'kachine.meta.lidia.tereixa', - 'extra.response_size': 148, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 10001, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 39.0, - 'extra.essid': 'Adana-Mason-Lanikai-Ozaner', - 'extra.firmwarerev': 'XM.ar7240.v5.6.3.28591.151130.1749', - 'extra.mac_address': '00156d7c9188', - 'extra.model': 'LM5', - 'extra.model_full': 'NanoStation Loco M5', - 'extra.radio_name': 'adana.mason.lanikai.ozaner', - 'extra.response_size': 156, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 10001, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 36.25, - 'extra.essid': 'Tailynn-Kadija-Noreen-Dinkar', - 'extra.firmwarerev': 'XW.ar934x.v5.6.5.29033.160515.2108', - 'extra.mac_address': '0418d6000fd5', - 'extra.model': 'P2B-400', - 'extra.model_full': 'PowerBeam M2 400', - 'extra.radio_name': 'tailynn.kadija.noreen.dinkar', - 'extra.response_size': 145, - 'extra.tag': 'ubiquiti,iot', - 'feed.name': 'Open Ubiquiti', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 10001, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py deleted file mode 100644 index 457ec4425..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_vnc.py +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_vnc.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible VNC', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_vnc-test-geo.csv", - } -EVENTS = [{'__type': 'Event', - 'feed.name': 'Accessible VNC', - "classification.identifier": "open-vnc", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "RFB 003.889", - "extra.product": "Apple remote desktop vnc", - "protocol.application": "vnc", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[1]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.53", - "source.port": 5678, - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T14:51:44+00:00" - }, - {'__type': 'Event', - 'feed.name': 'Accessible VNC', - "classification.identifier": "open-vnc", - "classification.taxonomy": "vulnerable", - "classification.type": "vulnerable-system", - "extra.banner": "RFB 005.000", - "extra.naics": 517311, - "extra.product": "RealVNC Enterprise v5.3 or later", - "protocol.application": "vnc", - "protocol.transport": "tcp", - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], - EXAMPLE_LINES[2]])), - "source.asn": 5678, - "source.geolocation.cc": "AA", - "source.geolocation.city": "LOCATION", - "source.geolocation.region": "LOCATION", - "source.ip": "198.123.245.112", - "source.port": 5678, - "source.reverse_dns": "localhost.localdomain", - "time.observation": "2015-01-01T00:00:00+00:00", - "time.source": "2019-09-04T14:51:44+00:00"}] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py deleted file mode 100644 index 41ab55e58..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_ws_discovery.py +++ /dev/null @@ -1,119 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ws_discovery.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Accessible-WS-Discovery-Service', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2010-02-10T00:00:00+00:00", - "extra.file_name": "2010-02-10-scan_ws_discovery-test.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 164.83, - 'extra.error': 'Validation constraint violation: SOAP message expected', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 989, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 3702, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 183.6, - 'extra.error': 'Validation constraint violation: missing root element', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 918, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 3702, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-ws-discovery', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 197.8, - 'extra.error': 'Validation constraint violation: SOAP message expected', - 'extra.raw_response': 'c2FtcGxlIHJlc3BvbnNlIGRhdGEK', - 'extra.response_size': 989, - 'extra.source.sector': 'Communications, Service Provider, and Hosting Service', - 'extra.tag': 'ws-discovery', - 'feed.name': 'Accessible-WS-Discovery-Service', - 'protocol.application': 'ws-discovery', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 3702, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py b/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py deleted file mode 100644 index d17482e71..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_scan_xdmcp.py +++ /dev/null @@ -1,117 +0,0 @@ -# SPDX-FileCopyrightText: 2019 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), 'testdata/scan_xdmcp.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {"feed.name": "ShadowServer XDMCP", - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_xdmcp-test-geo.csv", - } -EVENTS = [ -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.29, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node01.example.com', - 'extra.size': 44, - 'extra.status': 'Linux 3.0.101-100-default', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.1', - 'source.port': 177, - 'source.reverse_dns': 'node01.example.com', - 'time.source': '2010-02-10T00:00:00+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.86, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node02.example.com', - 'extra.size': 48, - 'extra.status': 'Linux 2.6.9-103.ELsmp', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.2', - 'source.port': 47074, - 'source.reverse_dns': 'node02.example.com', - 'time.source': '2010-02-10T00:00:01+00:00' -}, -{ - '__type': 'Event', - 'classification.identifier': 'open-xdmcp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'extra.amplification': 6.57, - 'extra.opcode': 'Willing', - 'extra.reported_hostname': 'node03.example.com', - 'extra.size': 46, - 'extra.status': '1 user, load: 6,5, 6,6, 6,6', - 'extra.tag': 'xdmcp', - 'feed.name': 'ShadowServer XDMCP', - 'protocol.application': 'xdmcp', - 'protocol.transport': 'udp', - 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn': 64512, - 'source.geolocation.cc': 'ZZ', - 'source.geolocation.city': 'City', - 'source.geolocation.region': 'Region', - 'source.ip': '192.168.0.3', - 'source.port': 177, - 'source.reverse_dns': 'node03.example.com', - 'time.source': '2010-02-10T00:00:02+00:00' -} - ] - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_special.py b/intelmq/tests/bots/parsers/shadowserver/test_special.py deleted file mode 100644 index abad86cac..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_special.py +++ /dev/null @@ -1,106 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import unittest - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot - -with open(os.path.join(os.path.dirname(__file__), - 'testdata/special.csv')) as handle: - EXAMPLE_FILE = handle.read() -EXAMPLE_LINES = EXAMPLE_FILE.splitlines() - -EXAMPLE_REPORT = {'feed.name': 'Special', - "raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2022-01-07T00:00:00+00:00", - "extra.file_name": "2022-01-07-special-test.csv", - } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Communications, Service Provider, and Hosting Service', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'special', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.source.sector' : 'Professional, Scientific, and Technical Services', - 'extra.status' : 'likely compromised', - 'feed.name' : 'Special', - 'malware.name' : 'cyclops-blink', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2022-01-07T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} -] - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - cls.default_input_message = EXAMPLE_REPORT - - def test_event(self): - """ Test if correct Event has been produced. """ - self.run_bot() - for i, EVENT in enumerate(EVENTS): - self.assertMessageEqual(i, EVENT) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_testdata.py b/intelmq/tests/bots/parsers/shadowserver/test_testdata.py deleted file mode 100644 index 19cbdd7d7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/test_testdata.py +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-FileCopyrightText: 2017 Sebastian Wagner -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import csv -import json -import os -import os.path -import unittest -import pathlib - -import intelmq.lib.test as test -import intelmq.lib.utils as utils -from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot -from intelmq.bots.parsers.shadowserver.parser_json import ShadowserverJSONParserBot - -def csvtojson(csvfile): - datalist = [] - - with open(csvfile) as fop: - reader = csv.DictReader(fop, restval="") - - for row in reader: - datalist.append(row) - - return json.dumps(datalist, indent=4) - -CSVREPORTS = {} -JSONREPORTS = {} -testdata = pathlib.Path(__file__).parent / 'testdata' -for filename in testdata.glob('*.csv'): - EXAMPLE_FILE = filename.read_text() - shortname = filename.stem - CSVREPORTS[shortname] = {"raw": utils.base64_encode(EXAMPLE_FILE), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": f"2019-01-01-{shortname}-test-test.csv", - } - JSONREPORTS[shortname] = {"raw": utils.base64_encode(csvtojson(filename)), - "__type": "Report", - "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": f"2019-01-01-{shortname}-test-test.json", - } - - -def generate_feed_function(feedname, reports): - def test_feed(self): - """ Test if no errors happen for feed %s. """ % feedname - self.input_message = reports[feedname] - self.run_bot() - return test_feed - - -class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverParserBot - -class TestShadowserverJSONParserBot(test.BotTestCase, unittest.TestCase): - """ - A TestCase for a ShadowserverParserBot. - """ - - @classmethod - def set_bot(cls): - cls.bot_reference = ShadowserverJSONParserBot - -for key in CSVREPORTS: - setattr(TestShadowserverParserBot, 'test_feed_%s' % key, generate_feed_function(key, CSVREPORTS)) -for key in JSONREPORTS: - setattr(TestShadowserverJSONParserBot, 'test_feed_%s' % key, generate_feed_function(key, JSONREPORTS)) - - -if __name__ == '__main__': # pragma: no cover - unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv deleted file mode 100644 index cfadcbb2d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","hostname","source","reason","asn","geo","region","city","naics","sic","sector","tag" -"2019-09-04 07:00:19","198.123.245.134",host.local,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,0, -"2019-09-04 07:00:19","198.123.245.171",,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,, -"2019-09-04 07:00:19","198.123.245.0/24",,"Alien Vault","Malicious Host AA",5678,"XX","LOCATION","LOCATION",517311,0,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license deleted file mode 100644 index 476908eeb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/blocklist.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/botnet_drone.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license deleted file mode 100644 index 456b03316..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/caida_ip_spoofer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv deleted file mode 100644 index 117dd6560..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","application","asn","geo","region","city","url","http_host","category","system","detected_since","server","redirect_target","naics","sic","sector","cc_url","family" -"2017-01-16 00:43:48","203.0.113.1",80,"example.com","hacked-webserver-stealrat-t1","http",64496,"AT","WIEN","VIENNA","/header.php","example.com","spam","WINNT","2015-05-09 05:51:12","Microsoft-IIS/7.5",,0,0,,, -"2018-04-09 15:43:41","203.0.113.1","80","","phishing","http","64496","AT","STEIERMARK","GRAZ","/","example.com","phishing","","","","","0","0","",, -"2022-02-07 21:52:29","66.249.0.0",,"66-249-0-0.example.com","magecart",,1234,"US","CALIFORNIA","MOUNTAIN VIEW",,,"stealer",,,,,519130,,"Communications, Service Provider, and Hosting Service","https://lolfree.pw/ads.txt", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/compromised_website.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/darknet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/ddos_amplification.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv deleted file mode 100644 index 22cfdd69e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model" -"2022-01-10 00:01:42","88.84.0.0","tcp",10443,,"ssl,vpn",2116,"NO","TROMS OG FINNMARK","TROMVIK",517311,,,"Fortinet","firewall","FortiGate" -"2022-01-10 00:01:42","170.231.0.0","tcp",10443,,"ssl,vpn",27843,"PE","METROPOLITANA DE LIMA","LIMA",,,,"Fortinet","firewall","FortiGate" -"2022-01-10 00:01:42","96.60.0.0","tcp",10443,"96-60-66-218.example.com","ssl,vpn",4181,"US","WISCONSIN","MILWAUKEE",517311,,,"Fortinet","firewall","FortiGate" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/device_id.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/drone_brute_force.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv deleted file mode 100644 index 3114c26b1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized","http_agent" -"2010-02-10 00:00:00",tcp,192.168.0.1,38055,64512,ZZ,Region,City,node01.example.com,0,,,,,172.16.0.1,443,65534,ZZ,Region,City,node01.example.net,0,"",,,ddos-participant,,,https,,,,,,,,,www.example.com,,,GET,/??=GovpfOoaWYlk,,,,,,,,,,,,,,,,,, -"2010-02-10 00:00:01",udp,192.168.0.2,53,64512,ZZ,Region,City,node02.example.com,0,,,,,172.16.0.2,53,65534,ZZ,Region,City,node02.example.net,0,"",,,ddos-participant,,,dns,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, -"2010-02-10 00:00:02",udp,192.168.0.3,53,64512,ZZ,Region,City,node03.example.com,0,,Microsoft,email,Exchange,172.16.0.3,53,65534,ZZ,Region,City,node03.example.net,0,"",,,ddos-participant,,,dns,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license deleted file mode 100644 index 9f58c89ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ddos_participant.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv deleted file mode 100644 index 17ff15ee6..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","service","start_time","end_time","client_version","username","password","payload_url","payload_md5" -"2021-03-27 00:00:00","tcp","141.98.1.2",30123,209588,"NL","NOORD-HOLLAND","AMSTERDAM",,,,,,,"162.250.1.2",22,26832,"CA","QUEBEC","MONTREAL",,,,"CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.521730Z","2021-03-27T00:00:01.710968Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","5.188.3.4",55690,57172,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"162.250.3.4",22,26832,"CA","QUEBEC","MONTREAL",,,,"CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.520927Z","2021-03-27T00:00:01.670993Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","45.14.5.6",38636,44220,"RO","BIHOR","ORADEA",,,,,,,"82.118.5.6",23,204957,"PL","MAZOWIECKIE","WARSAW",,,,"CAPRICA-EU","telnet-brute-force",,,"telnet",,,,"2021-03-27T00:00:00.781774Z","2021-03-27T00:00:00.857244Z",,,,, -"2021-03-27 00:00:00","tcp","5.188.6.7",56385,49453,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"102.16.6.7",22,37054,"MG","ANTANANARIVO","ANTANANARIVO",,,"Communications, Service Provider, and Hosting Service","CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.163870Z","2021-03-27T00:00:02.896640Z","b'SSH-2.0-Go'",,,, -"2021-03-27 00:00:00","tcp","45.14.7.8",35802,44220,"RO","BIHOR","ORADEA",,,,,,,"82.118.7.8",23,204957,"PL","MAZOWIECKIE","WARSAW",,,,"CAPRICA-EU","telnet-brute-force",,,"telnet",,,,"2021-03-27T00:00:00.781272Z","2021-03-27T00:00:00.856606Z",,,,, -"2021-03-27 00:00:00","tcp","5.188.9.10",33289,49453,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,,,,,"60.234.9.10",22,9790,"NZ","WELLINGTON","LOWER HUTT",,,"Communications, Service Provider, and Hosting Service","CAPRICA-EU","ssh-brute-force",,,"ssh",,,,"2021-03-27T00:00:00.044871Z","2021-03-27T00:00:00.077322Z","b'SSH-2.0-Go'",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license deleted file mode 100644 index 8b9580cf1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_brute_force.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv deleted file mode 100644 index dc78c1c1a..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv +++ /dev/null @@ -1,9 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","count" -"2021-03-07 00:00:00","tcp","61.3.1.2",4717,9829,"IN","KERALA","CHENGANNUR",,518210,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","211.218.3.4",4405,4766,"KR","GANGWON-DO","PYEONGCHANG-EUP",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","45.225.5.6",59777,266915,"BR","BAHIA","VITORIA DA CONQUISTA","static-45-225-x-x.example.net",,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","125.122.7.8",8460,4134,"CN","ZHEJIANG SHENG","HANGZHOU",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","219.77.9.10",21867,4760,"HK","HONG KONG","HONG KONG","n219077092196.example.com",517311,,,,,,5555,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","24.137.11.12",4680,14638,"PR","PUERTO RICO","SAN JUAN","dynamic.libertypr.net",,,,,,,5555,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","119.182.13.14",13175,4837,"CN","SHANDONG SHENG","JINING",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, -"2021-03-07 00:00:00","tcp","27.198.15.16",56133,4837,"CN","SHANDONG SHENG","JINAN",,517311,,,,,,23,,,,,,,,,"mirai",,"mirai",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_darknet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv deleted file mode 100644 index f41cb508f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized","http_agent" -"2010-02-10 00:00:00",,192.168.0.1,61234,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.1,88,65534,ZZ,Region,City,node01.example.net,0,,,,ddos,mirai,mirai,mirai,,,121.12.110.28/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:01",,192.168.0.2,61234,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.2,80,65534,ZZ,Region,City,node02.example.net,0,,,,ddos,mirai,mirai,mirai,,,180.97.183.94/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:02",,192.168.0.3,6379,64512,ZZ,Region,City,node03.example.com,0,"Communications, Service Provider, and Hosting Service",,,,172.16.0.3,,65534,ZZ,Region,City,node03.example.net,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,104.237.138.135/32,32,atk7,10,,,,,,,,,,,,,,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv deleted file mode 100644 index a7d0bc4f1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","request","count","bytes","end_time","duration","avg_pps","max_pps" -"2021-03-28 00:00:02",,"107.141.1.2",,7018,"US","CALIFORNIA","VISALIA","107-141-x-x.lightspeed.frsnca.sbcglobal.net",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,"2021-03-28 00:20:22",,, -"2021-03-28 00:00:02",,"74.59.3.4",,5769,"CA","QUEBEC","CHICOUTIMI","modemcablex-x-59-74.mc.videotron.ca",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,"2021-03-28 00:13:50",,, -"2021-03-28 00:00:02",,"65.131.5.6",,209,"US","WYOMING","CASPER","65-131-x-x.chyn.qwest.net",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, -"2021-03-28 00:00:02",,"104.162.7.8",,12271,"US","NEW YORK","KINGSTON","cpe-104-162-x-x.hvc.res.rr.com",517311,"Communications, Service Provider, and Hosting Service",,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, -"2021-03-28 00:00:02",,"37.120.178.9.10",,197540,"DE","NIEDERSACHSEN","GIFHORN","v22020111328131649.ultrasrv.de",,,,,,,389,,,,,,,,"CISPA","ddos-amplification",,,,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license deleted file mode 100644 index 8b9580cf1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_amp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv deleted file mode 100644 index 0e5b1e5e9..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","device_vendor","device_type","device_model","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","domain_source","public_source","infection","family","tag","application","version","event_id","dst_network","dst_netmask","attack","duration","attack_src_ip","attack_src_port","domain","domain_transaction_id","gcip","http_method","http_path","http_postdata","http_usessl","ip_header_ack","ip_header_acknum","ip_header_dont_fragment","ip_header_fin","ip_header_identity","ip_header_psh","ip_header_rst","ip_header_seqnum","ip_header_syn","ip_header_tos","ip_header_ttl","ip_header_urg","number_of_connections","packet_length","packet_randomized" -"2010-02-10 00:00:00",,172.16.0.1,80,65534,ZZ,Region,City,node01.example.net,0,,,,,192.168.0.1,61234,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,115.238.198.85/32,32,atk0,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:01",,172.16.0.2,43437,65534,ZZ,Region,City,node02.example.net,0,Information,,,,192.168.0.2,61234,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,52.184.50.250/32,32,atk0,30,,,,,,,,,,,,,,,,,,,,,,,1440, -"2010-02-10 00:00:02",,172.16.0.3,80,65534,ZZ,Region,City,node03.example.net,0,,,,,192.168.0.3,61234,64512,ZZ,Region,City,node03.example.com,0,"Communications, Service Provider, and Hosting Service",,,ddos,mirai,mirai,mirai,,,211.99.102.216/32,32,atk10,30,,,,,,,,,,,,,,,,,,,,,,,1440, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_ddos_target.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv deleted file mode 100644 index d9448bd83..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","pattern","http_url","http_agent","http_request_method","url_scheme","session_tags","vulnerability_enum","vulnerability_id","vulnerability_class","vulnerability_score","vulnerability_severity","vulnerability_version","threat_framework","threat_tactic_id","threat_technique_id","target_vendor","target_product","target_class","file_md5","file_sha256","request_raw","body_raw" -"2021-08-01 00:24:08","tcp","191.23.45.67",36455,1234,"EE","HARJUMAA","TALLINN","191-23-45-67-host.example.com",518210,"Communications, Service Provider, and Hosting Service",,,,"109.87.65.43",80,5678,"UK","WINDSOR AND MAIDENHEAD","MAIDENHEAD",,518210,,"CAPRICA-EU","http-scan",,,,"3.1.3-dev",,"unknown","/js/ueditor/wwwroot/way-board.cgi",,,,,,,,,,,,,,,,,,,"GET /js/ueditor/wwwroot/way-board.cgi HTTP/1.0rnAccept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8rnAccept-Encoding: gzip, deflaternAccept-Language: en-US,en;q=0.5rnConnection: closernDnt: 1rnHost: 109.87.65.43rnOrigin: http://109.87.65.43rnReferer: http://109.87.65.43/rnUser-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3084.400 QQBrowser/9.6.11346.400", -"2021-08-01 05:21:59","tcp","45.67.89.123",58610,12345,"EE","HARJUMAA","TALLINN",,,,,,,"82.41.20.10",8080,23456,"UA","KHARKIVS'KA OBLAST'","KHARKIV",,,,"CAPRICA-EU","http-scan",,,,,,,"/","Mozilla/5.0 (Windows NT 5.1; rv:9.0.1) Gecko/20100101 Firefox/9.0.1","GET","http",,,,,,,,,,,,,,,,"R0VUIC8gSFRUUC8xLjENCkhvc3Q6IDgyLjQxLjIwLjEwOjgwODANCkFjY2VwdDogdGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksKi8qO3E9MC44DQpBY2NlcHQtRW5jb2Rpbmc6IGRlZmxhdGUsIGd6aXAsIGlkZW50aXR5DQpBY2NlcHQtTGFuZ3VhZ2U6IGVuLVVTO3E9MC42LGVuO3E9MC40DQpVc2VyLUFnZW50OiBNb3ppbGxhLzUuMCAoV2luZG93cyBOVCA1LjE7IHJ2OjkuMC4xKSBHZWNrby8yMDEwMDEwMSBGaXJlZm94LzkuMC4xDQoNCg==", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license deleted file mode 100644 index c1900637f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_honeypot_http_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv deleted file mode 100644 index 174360bbd..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","network","routedspoof","session","nat" -"2021-03-28 00:42:59","tcp","98.191.250.0",,22898,"US","OKLAHOMA","OKLAHOMA CITY","ip-98.191.250.0.atlinkservices.com",517311,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"98.191.250.0/24","received",1112907,"True" -"2021-03-28 01:36:22","tcp","191.7.16.0",,262485,"BR","RIO DE JANEIRO","NOVA IGUACU",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"191.7.16.0/24","received",1112914,"False" -"2021-03-28 02:10:58","tcp","202.53.160.0",,23923,"BD","DHAKA","DHAKA",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"202.53.160.0/24","received",1112931,"True" -"2021-03-28 03:41:51","tcp","87.121.75.0",,134697,"AU","QUEENSLAND","BRISBANE",,,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"87.121.75.0/24","received",1112953,"True" -"2021-03-28 06:07:17","tcp","189.201.194.0",,262944,"MX","COAHUILA","SALTILLO","ip-189-201-194-0.slw.spectro.mx",,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"189.201.194.0/24","received",1113015,"True" -"2021-03-28 06:59:53","tcp","197.15.48.0",,37671,"TN","TUNIS","TUNIS",,517311,,,,,,,,,,,,,,"caida","ip-spoofer",,,,"ipv4",,"197.15.48.0/24","received",1113035,"True" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_ip_spoofer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv deleted file mode 100644 index eb0cbbab9..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv +++ /dev/null @@ -1,7 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id" -"2021-06-07 00:00:00","tcp","190.229.1.2",52955,7303,"AR","BUENOS AIRES","CASEROS",,517311,,,,,"168.63.134.179",16464,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","96.20.3.4",16464,5769,"CA","QUEBEC","LAVAL",,517311,"Communications, Service Provider, and Hosting Service",,,,"52.169.3.4",16464,8075,"IE","DUBLIN","DUBLIN",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","187.222.5.6",55049,8151,"MX","CIUDAD DE MEXICO","MEXICO CITY",,517311,,,,,"168.63.134.179",16464,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","75.84.7.8",64190,20001,"US","CALIFORNIA","NORTH HOLLYWOOD",,517311,"Communications, Service Provider, and Hosting Service",,,,"52.169.7.8",16464,8075,"IE","DUBLIN","DUBLIN",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","24.15.9.10",60373,7922,"US","ILLINOIS","HOMER GLEN",,517311,"Communications, Service Provider, and Hosting Service",,,,"104.40.6.5",16464,8075,"US","CALIFORNIA","SAN FRANCISCO",,334111,"Information","MSDCU","b68-zeroaccess-2-32bit","zeroaccess","b68-zeroaccess-2-32bit",,, -"2021-06-07 00:00:00","tcp","124.101.11.12",50386,4713,"JP","FUKUOKA","FUKUOKA",,517311,"Communications, Service Provider, and Hosting Service",,,,"23.99.101.165",16465,8075,"HK","HONG KONG","HONG KONG",,334111,"Information","MSDCU","b68-zeroaccess-2-64bit","zeroaccess","b68-zeroaccess-2-64bit",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv deleted file mode 100644 index c56d1f218..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2021-06-07 00:00:00","tcp","31.206.1.2",49245,8386,"TR","ANTALYA","KEPEZ",,,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, -"2021-06-07 00:00:00","tcp","177.140.3.4",35919,28573,"BR","SAO PAULO","SAO PAULO",,517312,,,,,"204.95.99.204",443,8075,"US","WASHINGTON","REDMOND",,334111,"Information","MSDCU","caphaw","caphaw","caphaw",,,,"/index.php","3fo8jrthz3y.rgk.cc","Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.0.6103)",,,"null" -"2021-06-07 00:00:01","tcp","180.190.5.6",49264,132199,"PH","CEBU","MANDAUE",,517311,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, -"2021-06-07 00:00:01","tcp","197.157.7.8",55307,37129,"KE","NAIROBI CITY","NAIROBI",,,,,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/news/stream.php","40.121.206.97",,,, -"2021-06-07 00:00:01","tcp","174.114.9.10",59000,812,"CA","ONTARIO","OTTAWA",,517311,"Communications, Service Provider, and Hosting Service",,,,"40.121.206.97",80,8075,"US","VIRGINIA","ASHBURN",,334111,"Information","MSDCU","necurs","necurs","necurs",,,,"/locator.php","40.121.206.97",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_microsoft_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv deleted file mode 100644 index c5126c843..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id" -"2021-03-04 00:00:00","tcp","190.113.1.2",17409,12252,"PE","METROPOLITANA DE LIMA","LIMA",,,,,,,"178.162.1.2",4455,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service","eset","victorygate.b","victorygate.b",,,, -"2021-03-04 00:00:00","tcp","35.205.9.10",44696,15169,"BE","BRUXELLES-CAPITALE","BRUSSELS","x.x.205.35.bc.googleusercontent.com",519130,"Communications, Service Provider, and Hosting Service",,,,"148.81.9.10",80,1887,"PL","MAZOWIECKIE","WARSAW",,,,,"virut","virut",,,, -"2021-03-04 00:00:00","tcp","35.197.11.12",36968,15169,"US","OREGON","THE DALLES","x.x.197.35.bc.googleusercontent.com",519130,"Communications, Service Provider, and Hosting Service",,,,"148.81.111.11.12",80,1887,"PL","MAZOWIECKIE","WARSAW",,,,,"virut","virut",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv deleted file mode 100644 index 3e85690d8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","infection","family","tag","query_type","query","count" -"2022-01-06 00:00:02","udp","217.110.0.0",29614,8220,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,,"calypso","calypso","msexchange","A","YolkIsh.COM",1 -"2022-01-06 00:00:02","udp","209.66.0.0",46189,40934,"US","VIRGINIA","ASHBURN",,518210,,,,,"orcus","orcus","rat","A","verble.rocks",1 -"2022-01-06 00:00:02","udp","217.110.0.0",3590,8220,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,,"calypso","calypso","msexchange","A","RAwFuNS.COM",1 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license deleted file mode 100644 index 662bb20b7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv deleted file mode 100644 index 4514f248e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2021-03-04 00:00:00","tcp","103.196.1.2",60902,134707,"PH","NUEVA ECIJA","DEL PILAR",,,,,,,"184.105.1.2",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","5.14.3.4",55002,8708,"RO","CONSTANTA","CONSTANTA",,517311,"Communications, Service Provider, and Hosting Service",,,,"184.105.3.4",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","49.145.5.6",31350,9299,"PH","CEBU","CEBU",,517311,,,,,"184.105.5.6",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"disorderstatus.ru",,,, -"2021-03-04 00:00:00","tcp","200.44.7.8",28063,8048,"VE","CARABOBO","VALENCIA",,517311,,,,,"184.105.7.8",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, -"2021-03-04 00:00:00","tcp","187.189.9.10",45335,17072,"MX","CHIHUAHUA","JUAREZ",,,,,,,"184.105.9.10",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,,"andromeda","avalanche-andromeda",,,,,"differentia.ru",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv deleted file mode 100644 index 23a3cb2b6..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv +++ /dev/null @@ -1,6 +0,0 @@ -"timestamp","protocol","http_referer_ip","http_referer_port","http_referer_asn","http_referer_geo","http_referer_region","http_referer_city","http_referer_hostname","http_referer_naics","http_referer_sector","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_referer" -"2021-03-04 00:00:02","tcp","178.162.203.211",80,28753,"DE","HESSEN","FRANKFURT AM MAIN","12106.mobapptrack.com",518210,"Communications, Service Provider, and Hosting Service","85.17.31.82",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"kovter","kovter",,,1614816002,"GET /favicon.ico HTTP/1.1","12106.mobapptrack.com","http://12106.mobapptrack.com/click/redirect?feed_id=12106&sub_id=7&q=8A5491983C8FBE7743E2D2C36E45EBC4-18307118D2626C9BD756B3F09D14BB910E381EE4" -"2021-03-04 00:00:11","tcp","59.106.1.2",80,9370,"JP","OSAKA","OSAKA","x.noizm.com",518210,"Communications, Service Provider, and Hosting Service","178.162.1.2",80,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816011,"GET /animalally.com HTTP/1.1","freescanonline.com","http://x.noizm.com/jump.php?u=http://freescanonline.com/animalally.com" -"2021-03-04 00:00:12","tcp","142.250.3.4",80,15169,"US","CALIFORNIA","MOUNTAIN VIEW","x.blogspot.com",519130,"Communications, Service Provider, and Hosting Service","178.162.1.2",80,28753,"DE","HESSEN","FRANKFURT AM MAIN",,518210,"Communications, Service Provider, and Hosting Service",,,"kovter","kovter",,,1614816012,"GET /getjs?r=0.6393021999392658 HTTP/1.1","rxrtb.bid","http://x.blogspot.com/" -"2021-03-04 00:00:13","tcp","34.232.5.6",80,14618,"US","VIRGINIA","ASHBURN","www.example.com",454110,"Retail Trade","5.79.71.225",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816013,"GET /personalationmall.com HTTP/1.1","freescanonline.com","http://www.example.com/teams/default.asp?u=EKL&t=c&s=lacrosse&p=remote&url=http://freescanonline.com/personalationmall.com" -"2021-03-04 00:01:26","tcp","210.172.7.8",80,2516,"JP","HOKKAIDO","SAPPORO","x.communes.jp",517312,"Communications, Service Provider, and Hosting Service","5.79.1.2",80,60781,"NL","NOORD-HOLLAND","AMSTERDAM",,518210,"Communications, Service Provider, and Hosting Service",,,"sunburst","sunburst",,,1614816086,"GET /raftcomply.com HTTP/1.1","freescanonline.com","http://x.communes.jp/?url=http://freescanonline.com/raftcomply.com" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license deleted file mode 100644 index c1900637f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event4_sinkhole_http_referer.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv deleted file mode 100644 index 016d2f912..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","protocol","src_ip","src_port","src_asn","src_geo","src_region","src_city","src_hostname","src_naics","src_sector","device_vendor","device_type","device_model","dst_ip","dst_port","dst_asn","dst_geo","dst_region","dst_city","dst_hostname","dst_naics","dst_sector","public_source","infection","family","tag","application","version","event_id","http_url","http_host","http_agent","forwarded_by","ssl_cipher","http_referer" -"2022-03-02 09:14:19","tcp","2001:448a:1082:4d9b:7491:bf9e:3d5f:a634",49431,7713,"ID","JAKARTA RAYA","JAKARTA",,,,,,,"2001:470:1:332::fe",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM HTTP/1.1","devps.net","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)",,, -"2022-03-02 09:15:10","tcp","2001:448a:1082:4d9b:7491:bf9e:3d5f:a634",49460,7713,"ID","JAKARTA RAYA","JAKARTA",,,,,,,"2001:470:1:332::ef",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /QKMSvF2hl11j%2fbMkyPbF5EpHYhd6VWTG4u19K3Rt7JGU3lMYRqpq8wPYEuOGKKeidKW3pefVfKSjBnL0cXizZbmuWWu8AQNRqw5g9Ny5vZtiv638XKoWwCLuUOTISTV%2fLcpcS1%2f22NjWqgXkHGISAuyVtafqyCC%2f5cA0eYg9Me8VzAIFDdTArogQOdYhElf2xluhEFPsstGQ%2bwrM4VmKHJpzyjD7Y%2fN%2bQV3wnZNdVkEVk1k2iKBJkotYv3ajgYWr56xxCbY5vE1IpZBRNhhaUDNZo0kJgi%2b6knXZ4m7JHt%2fGtJeP%2bNTxHSUL2ELlTIiT3ENlPYD6FdH6ZBxT1OneW%2f0ih%2fcN7vctb5B5Qwa1ez7ZjN2QxgBYkFDDHHTs42ej5eF2BysWAQDSUr%2fcySyGxcfPveIpfQEdrynGKR6z3OYqkFnP%2bYRDQp2rt1qt0FwCB4L9cg05TQlSSTJVGfPDrtcqjvKY4c9hWwSHtE8jMRpeCYO4Es%2bWgwr5DjzMicmuZo%2f4Ycr16jpN7xlDJdJ8iCFZxbSGgVC7ksVlGE8wlfWPI4KTuX5U5s61eNWPTlAC%2fOGb8grtw%2ffzizoIX9D6ZUMvslGLQIp%2fvNmNQkZy8HhNoV6Lns%2figITP%2fpN0H8h9HjUTl9qn65xFOEVpc0motSy8alcTPtTRKq5Jvc4Ao0x3N%2fvCB1v4Epx7XC0UpFbw8TrYEvAczEfGsGM HTTP/1.1","devps.net","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET4.0C; .NET4.0E)",,, -"2022-03-02 14:15:10","tcp","2603:8080:b20a:dc00:f06e:8304:71f6:27e2",62932,11427,"US","TEXAS","GARLAND",,517311,"Communications, Service Provider, and Hosting Service",,,,"2001:470:1:332::fe",80,6939,"US","CALIFORNIA","FREMONT",,518210,"Communications, Service Provider, and Hosting Service",,"boaxxe","boaxxe","3ve",,,,"GET /WMoUNCvuKGzdqSCeQcadP1%2f0B%2f3bzpOmyKBU85Z25HVOhvDQUPFl%2fk8uOcLewS%2b1BsuHXalRAOIgGOYYs2igj6UX8FkdCAmDewWPvfDhPD45nwd2tx1lLf2IoIfuOtIpGR6bN5Q6hGpSBgfERqCa0ImHcwfcZ2EdO%2fWvg7R8H6SLcTiuUC0I4pzvlWt1CRLgLdIEU1hZ0nnFHIHhchb6D7ITEgBQ2chQDxy5TJMrGjm4Dac6dKl%2ft5uYhRhSjAHkLLtgrJjsqVtVbelTAkt5kdcqLlO09m1SH%2fvtAb%2fOvR2DbhBss7%2f64DG7g6cAnghNA6JrFn1uW7sw%2bnKH8koKQwzUjdSsbrQAvmg4r0KDDW8Diq64gfDzxFWkzCLOYifc%2fwlinXPCl7aJiNCoieDC1U98RNQg%2f5td4SZmJnDQ2%2f96CPbFeSpCez5WD1rCjrxLj1h2cqzIgkydEWACceWP9ztxc4QaObzEcgOGxbRckWC7H2aaLeT8jaYEYdKi1pwEKChSL3YdEt4ZIb2IFrWwzNaXEpQzFXf07f902OEdI9vVA1ZdEOBPG6rAIkzMdebfprfVyhKEWtrCd3Skg3COUFtRQks5jzG1nv4sVGijTfSgyn6xE9Taka668Nycik6nmHy8Huj3oC01j3tee%2f1Z3eI6tV7lgM5d3uFJ84slRGHUCwMfVozOGmZRwNo%2fz%2bA HTTP/1.1","devps.net","Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license deleted file mode 100644 index 662bb20b7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/event6_sinkhole_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_http_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/hp_ics_scan.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv deleted file mode 100644 index ccafbab3f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","url","host","ip","asn","geo","region","city","naics","sector","tag","source","sha256","application" -"2022-01-07 00:02:07","http://41.86.0.0:50008/Mozi.m","41.86.0.0","41.86.0.0",37203,"LR","MONTSERRADO","MONROVIA",,,"CVE-2016-10372",,"12013662c71da69de977c04cd7021f13a70cf7bed4ca6c82acbc100464d4b0ef","http" -"2022-01-07 00:03:14","http://42.225.0.0:38173/Mozi.m","42.225.0.0","42.225.0.0",4837,"CN","HENAN SHENG","ZHUMADIAN",517311,,"CVE-2018-10562",,,"http" -"2022-01-07 00:10:26","http://211.52.0.0:53029/Mozi.m","211.52.0.0","211.52.0.0",4766,"KR","CHUNGCHEONGNAM-DO","SAGOK-MYEON",517311,,"CVE-2018-10562",,,"http" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/malware_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/outdated_dnssec_key.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv deleted file mode 100644 index 965d763a3..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","url","host","ip","asn","geo","region","city","naics","sector","source" -"2022-02-01 08:00:07","https://priceless-pare.example.net/Postal-/acec6/","priceless-pare.example.net","172.245.0.0",64512,"US","NEW YORK","BUFFALO",518210,"Communications, Service Provider, and Hosting Service","openphish.com" -"2022-02-01 08:00:07","https://mailyahooattt.example.net/","mailyahooattt.example.net","199.34.0.0",64512,"US","CALIFORNIA","SAN FRANCISCO",,"Professional, Scientific, and Technical Services","openphish.com" -"2022-02-01 08:00:07","https://www.example.net/viewer/vbid-730ec2b1-omsttuer","www.example.net","216.58.0.0",64512,"US","UTAH","DRAPER",519130,"Communications, Service Provider, and Hosting Service","openphish.com" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/phish_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv deleted file mode 100644 index d5baa730f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","proxy_authenticate","via","server","content_length","transfer_encoding","http_date" -"2010-02-10 00:00:00",192.168.0.1,tcp,3128,node01.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"Squid proxy-caching web server\"\"",,squid/4.10,3741,,"Wed, 10 Feb 2010 00:00:00 GMT" -"2010-02-10 00:00:01",192.168.0.2,tcp,3128,node02.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"00:23:24:43:1c:34\"\"",,,3833,,"Wed, 10 Feb 2010 00:00:01 GMT" -"2010-02-10 00:00:02",192.168.0.3,tcp,3128,node03.example.com,http-connect-proxy-closed,64512,ZZ,Region,City,0,0,HTTP/1.1,407,"Proxy Authentication Required",text/html;charset=utf-8,keep-alive,"Basic realm=\"\"Proxy\"\"",,,179,,"Wed, 10 Feb 2010 00:00:02 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/population_http_proxy.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv deleted file mode 100644 index 4710af974..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","asn","geo","md5","protocol","port","host","bytes_in","bytes_out" -"2022-01-10 00:00:03","40.119.6.228",8075,"US","b575ce6dcce6502a8431db5610135c25","udp",123,"time.windows.com",0,0 -"2022-01-10 00:00:03","8.252.70.126",3356,"US","c0d947f9a8685b0d9f3efdba966389c2","tcp",80,,0,0 -"2022-01-10 00:00:03","52.109.8.22",8075,"US","c0d947f9a8685b0d9f3efdba966389c2","tcp",443,,0,0 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_conn.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv deleted file mode 100644 index 697cb6209..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","md5hash","request","type","response","family","tag","source" -"2022-01-10 00:00:02","b575ce6dcce6502a8431db5610135c25","time.windows.com","A","40.119.6.228",,, -"2022-01-10 00:00:08","807679198a39c80d3ca07e60fd51b581","time.windows.com","A","40.119.6.228",,, -"2022-01-10 00:00:20","d97e973b9bf073bd3a217425259cea26","client-office365-tas.msedge.net","A","13.107.5.88",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv deleted file mode 100644 index bbfe596a2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","asn","geo","md5","url","user_agent","host","method" -"2022-01-10 00:01:13","23.196.47.89",20940,"US","37514b54e679a5313334e830ad780ec7","http://www.msftncsi.com/ncsi.txt","Microsoft NCSI","www.msftncsi.com","GET" -"2022-01-10 00:01:28","72.21.81.240",15133,"US","37514b54e679a5313334e830ad780ec7","http://www.download.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab","Microsoft-CryptoAPI/6.1","www.download.windowsupdate.com","GET" -"2022-01-10 00:08:24","23.56.4.57",20940,"US","e97ea2820c0d79f3f3ca241d4dcd1060","http://crl.microsoft.com/pki/crl/products/MicTimStaPCA_2010-07-01.crl","Microsoft-CryptoAPI/6.1","crl.microsoft.com","GET" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sandbox_url.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv deleted file mode 100644 index c0ff0bdf1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","name","model","device","features","device_vendor","device_type","device_model","device_version","device_sector" -"2018-07-26 02:07:16","36.239.124.210","tcp",5555,"36-239-124-210.dynamic-ip.hinet.net","adb",3462,"TW","TAOYUAN COUNTY","TAOYUAN CITY",518210,737415,"hlteuc","SAMSUNG-SM-N900A","hlteatt",,,,,, -"2018-07-26 02:07:16","36.236.108.107","tcp",5555,"36-236-108-107.dynamic-ip.hinet.net","adb",3462,"TW","TAIPEI CITY","TAIPEI",518210,737415,"marlin","Pixel XL","marlin","cmd,shell_v2",,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_adb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv deleted file mode 100644 index c5494d458..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","machine_type","afp_versions","uams","flags","server_name","signature","directory_service","utf8_servername","network_address" -"2019-09-04 05:05:53","198.13.34.22","tcp",548,"host.local","afp",6057,"AA","LOCATION","LOCATION",517311,0,"TimeCapsule8,119","AFP3.3,AFP3.2,AFP3.1","DHCAST128,DHX2,SRP,Recon1","SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient","airport-time-capsule-de-jack","4338364e37364442463948350069672d",,"AirPort Time Capsule de jack","198.33.24.165:548,10.0.1.1:548,fe80:0008:0000:0000:6e70:9fff:fed4::548,fe80:0009:0000:0000:6e70:9fff:fed4::548,179.24.24.165 (DNS address)," -"2019-09-04 05:05:56","198.40.27.212","tcp",548,"host.local","afp",6057,"AA","LOCATION","LOCATION",517311,0,"TimeCapsule8,119","AFP3.3,AFP3.2,AFP3.1","DHCAST128,DHX2,SRP,Recon1","SupportsCopyFile,SupportsChgPwd,SupportsServerMessages,SupportsServerSignature,SupportsTCP/IP,SupportsSrvrNotifications,SupportsReconnect,SupportsOpenDirectory,SupportsUTF8Servername,SupportsUUIDs,SupportsSuperClient","time-capsule-del-jack","433836544b303147463948360069672d",,"Time Capsule del Jack","0.0.0.1:548,10.0.1.1:548,198.33.42.1:548,fe80:000b:0000:0000:dea4:caff:feba::548,fe80:000c:0000:0000:dea4:caff:feba::548,fe80:000d:0000:0000:4c7d:ffff:fec7::548,0.0.0.1 (DNS address)," diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_afp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv deleted file mode 100644 index 92f078af7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","channel","message_length","class","method","version_major","version_minor","capabilities","cluster_name","platform","product","product_version","mechanisms","locales","sector" -"2022-01-10 04:32:13","47.103.0.0","tcp",5672,,"amqp",37963,"CN","SHANGHAI SHI","SHANGHAI",518210,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos","rabbit@iZuf63m0nnq9bwf7lhjxrkZ","Erlang/OTP","RabbitMQ","3.3.5","PLAIN AMQPLAIN","en_US", -"2022-01-10 04:32:13","141.95.0.0","tcp",5672,,"amqp",16276,"DE","SAARLAND","SAARBRUCKEN",518210,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to","rabbit@mtk-breizh","Erlang/OTP 24.0.3","RabbitMQ","3.8.19","AMQPLAIN PLAIN","en_US", -"2022-01-10 04:32:13","54.234.0.0","tcp",5672,"ec2-54.234.0.0.compute-1.amazonaws.com","amqp",14618,"US","VIRGINIA","ASHBURN",454110,,0,509,10,10,0,9,"publisher_confirms,exchange_exchange_bindings,basic.nack,consumer_cancel_notify,connection.blocked,consumer_priorities,authentication_failure_close,per_consumer_qos,direct_reply_to","rabbit@1397a0e9629b","Erlang/OTP 24.2","RabbitMQ","3.9.11","PLAIN AMQPLAIN","en_US", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_amqp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv deleted file mode 100644 index 9c43f8598..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","machine_name","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,3283,node01.example.com,ard,64512,ZZ,Region,City,0,0,"Macmini (radio)",1006,201.20 -"2010-02-10 00:00:01",192.168.0.2,udp,3283,node02.example.com,ard,64512,ZZ,Region,City,0,0,biuro-rip-org-pl,1006,201.20 -"2010-02-10 00:00:02",192.168.0.3,udp,3283,node03.example.com,ard,64512,ZZ,Region,City,0,0,127.0.0.1,1006,201.20 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ard.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv deleted file mode 100644 index 7bd2b20e0..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","size","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,19,node01.example.com,chargen,,64512,ZZ,Region,City,0,0,,74,74.00 -"2010-02-10 00:00:01",192.168.0.2,udp,19,node02.example.com,chargen,,64512,ZZ,Region,City,0,0,,74,74.00 -"2010-02-10 00:00:02",192.168.0.3,udp,19,node03.example.com,chargen,,64512,ZZ,Region,City,0,0,Government,74,74.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_chargen.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv deleted file mode 100644 index 5182817c1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic" -"2017-11-18 08:42:45","198.51.100.103","tcp",4786,"198-51-100-103.example.net","cisco-smart-install",8559,"AT","WIEN","VIENNA",0,0 -"2017-11-18 08:47:54","198.51.100.218","tcp",4786,,"cisco-smart-install",35609,"AT","WIEN","VIENNA",0,0 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cisco_smart_install.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv deleted file mode 100644 index 6d72dac53..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","response","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,5683,node01.example.com,coap,2,64512,ZZ,Region,City,0,0,",,",43,2.05 -"2010-02-10 00:00:01",192.168.0.2,udp,5683,node02.example.com,coap,2,64512,ZZ,Region,City,0,0,",,,,,,,,,",113,5.38 -"2010-02-10 00:00:02",192.168.0.3,udp,5683,node03.example.com,coap,1,64512,ZZ,Region,City,0,0,"`EsjAy************************************************************|CoAP RFC 7252 |************************************************************|This server is using the Eclipse Californium (Cf) CoAP framework|published under EPL+EDL: http://www.eclipse.org/californium/||(c) 2014, 2015, 2016 Institute for Pervasive Computing, ETH Zurich and others|************************************************************",454,113.50 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_coap.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv deleted file mode 100644 index f4074f3ed..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","server_version","couchdb_message","couchdb_version","git_sha","features","vendor","visible_databases","error","error_reason" -"2010-02-10 00:00:00",192.168.0.1,tcp,5984,node01.example.com,couchdb,64512,ZZ,Region,City,0,0,,"CouchDB/1.6.1 (Erlang OTP/18)",Welcome,1.6.1,,,"Ubuntu 16.04",_replicator;_users;test;shops;god,, -"2010-02-10 00:00:01",192.168.0.2,tcp,5984,node02.example.com,couchdb,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service","CouchDB/3.2.1 (Erlang OTP/23)",Welcome,3.2.1,244d428af,"access-ready,partitioned,pluggable-storage-engines,reshard,scheduler","The Apache Software Foundation",,, -"2010-02-10 00:00:02",192.168.0.3,tcp,5984,node03.example.com,couchdb,64512,ZZ,Region,City,0,0,"Retail Trade","CouchDB/3.2.1 (Erlang OTP/20)",Welcome,3.2.1,244d428af,"access-ready,partitioned,pluggable-storage-engines,reshard,scheduler","The Apache Software Foundation",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_couchdb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv deleted file mode 100644 index 5aebed050..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","date","sector" -"2019-09-04 10:44:55","198.123.245.142","tcp",30005,,"cwmp",5678,"AA","LOCATION","LOCATION",517311,0,"HTTP/1.1",200,"OK","text/html","keep-alive",,,"DNVRS-Webs",5678,,"Wed, 04 Sep 2019 07:42:37 GMT", -"2019-09-04 11:06:50","198.123.245.162","tcp",5678,"localhost.localdomain","cwmp",5678,"AA","LOCATION","LOCATION",517311,0,"HTTP/1.1",404,"Not Found","text/html",,,,"RomPager/4.07 UPnP/1.0",,"chunked",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_cwmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv deleted file mode 100644 index c4bb32e57..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","db2_hostname","servername","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,523,node01.example.com,db2,64512,ZZ,Region,City,0,0,NOWAK_SERWER,node01.example.com,298,14.90 -"2010-02-10 00:00:01",192.168.0.2,udp,523,node02.example.com,db2,64512,ZZ,Region,City,0,0,SPZOZ-DZIEWIN,node02.example.com,298,14.90 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_db2.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv deleted file mode 100644 index 25e6f11d0..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","source_port","bytes","amplification","method" -"2010-02-10 00:00:00",192.168.0.1,tcp,80,node01.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,,49002,99,2,SYN+ACK:PSH -"2010-02-10 00:00:01",192.168.0.2,tcp,80,node02.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",41200,99,2,SYN+ACK:PSH -"2010-02-10 00:00:02",192.168.0.3,tcp,80,node03.example.com,ddos-middlebox,64512,ZZ,Region,City,0,0,,47492,99,2,SYN+ACK:PSH diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ddos_middlebox.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv deleted file mode 100644 index 05b807883..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv +++ /dev/null @@ -1,101 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","dns_version","asn","geo","region","city","min_amplification","p0f_genre","p0f_detail","naics","sic","sector" -"2018-04-14 00:14:34","198.51.100.179","udp",53,"198-51-100-189.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.8","udp",53,"198-51-100-111.example.net","openresolver","dnsmasq-2.51",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:34","198.51.100.179","udp",53,"198-51-100-189.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.8","udp",53,"198-51-100-111.example.net","openresolver","dnsmasq-2.51",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:14:36","198.51.100.158","udp",53,,"openresolver",,8447,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:37","198.51.100.7","udp",53,"198-51-100-7.example.net","openresolver","9.9.4-rpz2.13269.14-P2",13292,"AT","STEIERMARK","EISENERZ","4.6190",,,0,0, -"2018-04-14 00:14:38","198.51.100.167","udp",53,"198-51-100-167.example.net","openresolver","ZyWALL DNS",6830,"AT","KARNTEN","VILLACH","4.6667",,,0,0, -"2018-04-14 00:14:40","198.51.100.10","udp",53,"198-51-100-10.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:14:41","198.51.100.191","udp",53,"198-51-100-63.example.net","openresolver",,25255,"AT","TIROL","LIENZ","4.6190",,,0,0, -"2018-04-14 00:14:43","198.51.100.25","udp",53,"198-51-100-187.example.net","openresolver","p.4.0",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:54","198.51.100.174","udp",53,"198-51-100-174.example.net","openresolver",,8445,"AT","SALZBURG","SALZBURG","6.4048",,,0,0, -"2018-04-14 00:14:54","198.51.100.181","udp",53,"198-51-100-181.example.net","openresolver","PowerDNS Recursor tele2/sil",8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:14:54","198.51.100.80","udp",53,"198-51-100-80.example.net","openresolver",,1901,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:14:57","198.51.100.43","udp",53,"198-51-100-43.example.net","openresolver","vi2zcnsat10, Customer DNS",6830,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:14:58","198.51.100.124","udp",53,"198-51-100-124.example.net","openresolver","dnsmasq-2.47",28919,"AT","TIROL","EIBERG","3.8095",,,0,0, -"2018-04-14 00:15:00","198.51.100.60","udp",53,"198-51-100-60.example.net","openresolver",,24992,"AT","VORARLBERG","DORNBIRN","3.4762",,,0,0, -"2018-04-14 00:15:00","198.51.100.201","udp",53,"198-51-100-201.example.net","openresolver",,1853,"AT","STEIERMARK","GRAZ","4.6190",,,0,0, -"2018-04-14 00:15:01","198.51.100.82","udp",53,"198-51-100-82.example.net","openresolver","9.6-ESV-R7-P2",20811,"AT","TIROL","INNSBRUCK","4.6190",,,0,0, -"2018-04-14 00:15:01","198.51.100.105","udp",53,"198-51-100-105.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:02","198.51.100.173","udp",53,"198-51-100-173.example.net","openresolver",,8445,"AT","NIEDEROSTERREICH","WALD","1.3810",,,0,0, -"2018-04-14 00:15:03","198.51.100.82","udp",53,"198-51-100-82.example.net","openresolver","PowerDNS Recursor tele2/sil",1257,"AT","NIEDEROSTERREICH","MODLING","1.3810",,,518111,737401, -"2018-04-14 00:15:05","198.51.100.39","udp",53,,"openresolver",,8437,"AT","VORARLBERG","LUSTENAU","1.3810",,,0,0, -"2018-04-14 00:15:09","198.51.100.33","udp",53,,"openresolver","dnsmasq-2.55",8447,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:15:09","198.51.100.248","udp",53,"198-51-100-248.example.net","openresolver",,39912,"AT","NIEDEROSTERREICH","HOLLABRUNN","3.8095",,,0,0, -"2018-04-14 00:15:10","198.51.100.119","udp",53,"198-51-100-172.example.net","openresolver","dnsmasq-2.62",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:12","198.51.100.135","udp",53,"198-51-100-135.example.net","openresolver","no access.",43848,"AT","NIEDEROSTERREICH","WIESELBURG","3.8095",,,0,0, -"2018-04-14 00:15:15","198.51.100.64","udp",53,"198-51-100-64.example.net","openresolver",,6830,"AT","VORARLBERG","UBERSAXEN","1.3810",,,0,0, -"2018-04-14 00:15:17","198.51.100.80","udp",53,"198-51-100-80.example.net","openresolver",,42473,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:15:18","198.51.100.60","udp",53,"198-51-100-60.example.net","openresolver","198-51-100-60.example.net",35369,"AT","OBEROSTERREICH","LINZ","1.3810",,,0,0, -"2018-04-14 00:15:21","198.51.100.50","udp",53,,"openresolver","ZyWALL DNS",8447,"AT","STEIERMARK","TAUPLITZ","4.6667",,,0,0, -"2018-04-14 00:15:23","198.51.100.93","udp",53,,"openresolver","Microsoft DNS 6.1.7601 (1DB15D39)",8447,"AT","NIEDEROSTERREICH","SCHWADORF","1.3810",,,0,0, -"2018-04-14 00:15:24","198.51.100.33","udp",53,,"openresolver",,8447,"AT","STEIERMARK","FURSTENFELD","4.6190",,,0,0, -"2018-04-14 00:15:31","198.51.100.45","udp",53,,"openresolver","dnsmasq-2.52",8245,"AT","BURGENLAND","EISENSTADT","1.3810",,,0,0, -"2018-04-14 00:15:34","198.51.100.13","udp",53,"198-51-100-13.example.net","openresolver",,8447,"AT","WIEN","VIENNA","6.4048",,,518210,737415, -"2018-04-14 00:15:36","198.51.100.190","udp",53,,"openresolver",,8447,"AT","BURGENLAND","PINKAFELD","1.3810",,,0,0, -"2018-04-14 00:15:41","198.51.100.104","udp",53,,"openresolver",,21013,"AT","OBEROSTERREICH","LEONDING","2.6667",,,0,0, -"2018-04-14 00:15:42","198.51.100.101","udp",53,"198-51-100-101.example.net","openresolver",,8447,"AT","STEIERMARK","KAINACH BEI VOITSBERG","1.3810",,,0,0, -"2018-04-14 00:15:44","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver",,1901,"AT","OBEROSTERREICH","GMUNDEN","1.3810",,,518210,737415, -"2018-04-14 00:15:46","198.51.100.186","udp",53,"198-51-100-186.example.net","openresolver",,31239,"AT","WIEN","VIENNA","6.4048",,,0,0, -"2018-04-14 00:15:46","198.51.100.197","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","KIRCHDORF AN DER KREMS","4.6190",,,0,0, -"2018-04-14 00:15:49","198.51.100.16","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","LAAKIRCHEN","4.6190",,,0,0, -"2018-04-14 00:15:50","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver",,6830,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","4.6190",,,0,0, -"2018-04-14 00:15:53","198.51.100.7","udp",53,"198-51-100-7.example.net","openresolver",,198950,"AT","TIROL","REUTTE","4.6190",,,518210,737415, -"2018-04-14 00:15:53","198.51.100.177","udp",53,"198-51-100-177.example.net","openresolver","Microsoft DNS 6.1.7601 (1DB1446A)",12605,"AT","OBEROSTERREICH","LINZ","1.3810",,,0,0, -"2018-04-14 00:15:57","198.51.100.47","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","KOTTINGBRUNN","1.3810",,,0,0, -"2018-04-14 00:15:59","198.51.100.95","udp",53,"198-51-100-67.example.net","openresolver","GNS DNS Version 3",57169,"AT","WIEN","VIENNA","1.3810",,,0,0,"Information Technology" -"2018-04-14 00:16:02","198.51.100.104","udp",53,"198-51-100-104.example.net","openresolver",,6830,"AT","OBEROSTERREICH","BAD WIMSBACH-NEYDHARTING","1.3810",,,0,0, -"2018-04-14 00:16:04","198.51.100.106","udp",53,,"openresolver",,8447,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:16:05","198.51.100.204","udp",53,"198-51-100-204.example.net","openresolver",,12605,"AT","OBEROSTERREICH","LINZ","4.6190",,,0,0, -"2018-04-14 00:16:05","198.51.100.111","udp",53,"198-51-100-111.example.net","openresolver",,8447,"AT","OBEROSTERREICH","LINZ","1.3810",,,518210,737415, -"2018-04-14 00:16:06","198.51.100.131","udp",53,"198-51-100-139.example.net","openresolver","p.4.0",25255,"AT","OBEROSTERREICH","TRAUN","1.3810",,,0,0, -"2018-04-14 00:16:10","198.51.100.240","udp",53,"198-51-100-240.example.net","openresolver",,6830,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:13","198.51.100.9","udp",53,"198-51-100-42.example.net","openresolver",,13026,"AT","STEIERMARK","LEIBNITZ","6.4048",,,0,0, -"2018-04-14 00:16:15","198.51.100.231","udp",53,"198-51-100-74.example.net","openresolver",,25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:17","198.51.100.228","udp",53,"198-51-100-227.example.net","openresolver","u.1.0",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:19","198.51.100.152","udp",53,"198-51-100-152.example.net","openresolver",,34694,"AT","TIROL","WORGL","4.6190",,,0,0, -"2018-04-14 00:16:21","198.51.100.88","udp",53,,"openresolver",,8447,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:22","198.51.100.97","udp",53,"198-51-100-97.example.net","openresolver",,8447,"AT","TIROL","INNSBRUCK","1.3810",,,518210,737415, -"2018-04-14 00:16:23","198.51.100.208","udp",53,"198-51-100-208.example.net","openresolver","dnsmasq-2.62",8447,"AT","TIROL","OTZTAL-BAHNHOF","1.3810",,,0,0, -"2018-04-14 00:16:33","198.51.100.113","udp",53,"198-51-100-121.example.net","openresolver","dnsmasq-2.62",25255,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:35","198.51.100.34","udp",53,"198-51-100-44.example.net","openresolver",,25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:37","198.51.100.236","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","ST. ANDRAE-WOERDERN","4.6190",,,0,0, -"2018-04-14 00:16:40","198.51.100.46","udp",53,"198-51-100-46.example.net","openresolver",,21013,"AT","OBEROSTERREICH","LEONDING","2.6667",,,0,0, -"2018-04-14 00:16:45","198.51.100.72","udp",53,"198-51-100-5.example.net","openresolver",,25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:50","198.51.100.179","udp",53,"198-51-100-179.example.net","openresolver",,31125,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:16:50","198.51.100.107","udp",53,"198-51-100-107.example.net","openresolver","dnsmasq-2.66",18845,"AT","WIEN","VIENNA","1.3810",,,0,0,"Information Technology" -"2018-04-14 00:16:51","198.51.100.188","udp",53,,"openresolver","9.9.4-RedHat-9.9.4-51.el7_4.2",49322,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:16:54","198.51.100.232","udp",53,"198-51-100-232.example.net","openresolver",,6830,"AT","SALZBURG","SALZBURG","1.3810",,,0,0, -"2018-04-14 00:16:55","198.51.100.102","udp",53,"198-51-100-102.example.net","openresolver","ZyWALL DNS",6830,"AT","KARNTEN","WERNBERG","3.4762",,,0,0, -"2018-04-14 00:16:59","198.51.100.162","udp",53,"198-51-100-162.example.net","openresolver",,8445,"AT","SALZBURG","SALZBURG","1.3810",,,0,0, -"2018-04-14 00:17:00","198.51.100.110","udp",53,"198-51-100-110.example.net","openresolver",,31543,"AT","TIROL","SOLDEN","4.6190",,,0,0, -"2018-04-14 00:17:02","198.51.100.193","udp",53,"198-51-100-193.example.net","openresolver",,8447,"AT","STEIERMARK","FOHNSDORF","1.3810",,,0,0, -"2018-04-14 00:17:06","198.51.100.45","udp",53,"198-51-100-45.example.net","openresolver",,61201,"AT","KARNTEN","KLAGENFURT AM WORTHERSEE","1.3810",,,0,0, -"2018-04-14 00:17:06","198.51.100.219","udp",53,"198-51-100-219.example.net","openresolver",,8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:10","198.51.100.47","udp",53,"198-51-100-47.example.net","openresolver","unsupported query",8412,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.8095",,,0,0, -"2018-04-14 00:17:13","198.51.100.87","udp",53,"198-51-100-87.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:16","198.51.100.121","udp",53,"198-51-100-121.example.net","openresolver",,8447,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:20","198.51.100.115","udp",53,,"openresolver",,8447,"AT","TIROL","WAIDRING","1.3810",,,0,0, -"2018-04-14 00:17:22","198.51.100.235","udp",53,,"openresolver",,8447,"AT","OBEROSTERREICH","GRIESKIRCHEN","1.3810",,,0,0, -"2018-04-14 00:17:33","198.51.100.154","udp",53,,"openresolver",,8447,"AT","STEIERMARK","GRAZ","4.6190",,,0,0, -"2018-04-14 00:17:36","198.51.100.36","udp",53,"198-51-100-36.example.net","openresolver","BIND",12605,"AT","OBEROSTERREICH","LINZ","4.6190",,,0,0, -"2018-04-14 00:17:38","198.51.100.100","udp",53,"198-51-100-100.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:41","198.51.100.181","udp",53,"198-51-100-181.example.net","openresolver","Microsoft DNS 6.0.6002 (17724D35)",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:41","198.51.100.242","udp",53,"198-51-100-242.example.net","openresolver","Microsoft DNS 6.0.6002 (17724D35)",34767,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.2857",,,0,0, -"2018-04-14 00:17:42","198.51.100.38","udp",53,,"openresolver","PowerDNS Recursor tele2/sil",8437,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:43","198.51.100.132","udp",53,"198-51-100-132.example.net","openresolver","PowerDNS Recursor tele2/sil",8437,"AT","STEIERMARK","GRAZ","1.3810",,,0,0, -"2018-04-14 00:17:49","198.51.100.166","udp",53,"198-51-100-166.example.net","openresolver","9.8.4-rpz2+rl005.12-P1",13292,"AT","STEIERMARK","KINDBERG","4.6190",,,0,0, -"2018-04-14 00:17:49","198.51.100.212","udp",53,"198-51-100-212.example.net","openresolver","dnsmasq-2.40",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:51","198.51.100.225","udp",53,,"openresolver",,8220,"AT","WIEN","VIENNA","1.3810",,,518210,737415, -"2018-04-14 00:17:53","198.51.100.161","udp",53,"198-51-100-161.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:54","198.51.100.12","udp",53,,"openresolver",,8447,"AT","NIEDEROSTERREICH","LANGENLOIS","1.3810",,,0,0, -"2018-04-14 00:17:55","198.51.100.113","udp",53,"198-51-100-113.example.net","openresolver",,6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:17:57","198.51.100.175","udp",53,"198-51-100-175.example.net","openresolver","PowerDNS Recursor tele2/sil",1257,"AT","NIEDEROSTERREICH","MODLING","1.3810",,,518111,737401, -"2018-04-14 00:17:59","198.51.100.107","udp",53,"198-51-100-107.example.net","openresolver",,50719,"AT","STEIERMARK","TIESCHEN","3.8095",,,0,0, -"2018-04-14 00:17:59","198.51.100.51","udp",53,"198-51-100-68.example.net","openresolver","dnsmasq-2.66",25255,"AT","WIEN","VIENNA","4.6190",,,0,0, -"2018-04-14 00:18:04","198.51.100.131","udp",53,,"openresolver","ZyWALL DNS",8447,"AT","TIROL","OBERPERFUSS","3.4762",,,0,0, -"2018-04-14 00:18:05","198.51.100.138","udp",53,"198-51-100-138.example.net","openresolver","unsupported query",8412,"AT","NIEDEROSTERREICH","WIENER NEUSTADT","3.8095",,,0,0, -"2018-04-14 00:18:06","198.51.100.62","udp",53,"198-51-100-62.example.net","openresolver","viezcnsat13, Customer DNS",6830,"AT","WIEN","VIENNA","1.3810",,,0,0, -"2018-04-14 00:18:07","198.51.100.109","udp",53,"198-51-100-109.example.net","openresolver",,1901,"AT","OBEROSTERREICH","LINZ","6.9524",,,518210,737415, -"2018-04-14 00:18:10","198.51.100.205","udp",53,"198-51-100-205.example.net","openresolver",,8437,"AT","WIEN","VIENNA","1.3810",,,0,0, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv deleted file mode 100644 index 535dc4ea8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","http","http_code","http_reason","content_type","server","date","experimental","api_version","arch","go_version","os","kernel_version","git_commit","min_api_version","build_time","pkg_version" -"2010-02-10 00:00:00",192.168.0.1,tcp,2375,node01.example.com,docker,18.05.0-ce,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",HTTP/1.1,200,OK,"application/json; charset=UTF-8","Docker/18.05.0-ce (linux)","Fri, 06 May 2022 14:06:30 GMT",false,1.37,amd64,go1.9.5,linux,3.10.0-514.26.2.el7.x86_64,f150324,1.12,2018-05-09T22:18:36.000000000+00:00, -"2010-02-10 00:00:01",192.168.0.2,tcp,2375,node02.example.com,docker,1.13.1,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,"Docker/1.13.1 (linux)","Fri, 06 May 2022 14:08:07 GMT",false,1.26,amd64,go1.10.3,linux,3.10.0-693.2.2.el7.x86_64,7d71120/1.13.1,1.12,2022-03-02T15:25:43.414574467+00:00,docker-1.13.1-209.git7d71120.el7.centos.x86_64 -"2010-02-10 00:00:02",192.168.0.3,tcp,2375,node03.example.com,docker,18.05.0-ce,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,"application/json; charset=UTF-8","Docker/18.05.0-ce (linux)","Fri, 06 May 2022 14:08:06 GMT",false,1.37,amd64,go1.9.5,linux,3.10.0-514.26.2.el7.x86_64,f150324,1.12,2018-05-09T22:18:36.000000000+00:00, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_docker.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv deleted file mode 100644 index 60c711973..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_id","device_serial","machine_name","manufacturer","method","http_port","internal_port","video_input_channels","alarm_input_channels","video_output_channels","alarm_output_channels","remote_video_input_channels","mac_address","ipv4_address","ipv4_gateway","ipv4_subnet_mask","ipv4_dhcp_enable","ipv6_address","ipv6_link_local","ipv6_gateway","ipv6_dhcp_enable","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,37810,node01.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",General,IPC,BCS-TIP3401IR-E-V,2.800.106F004.0.R,,6J0E022PAG35073,6J0E022PAG35073,General,client.notifyDevInfo,80,37777,1,0,0,0,0,38:c4:e8:03:b3:e2,192.168.0.1,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::1,fe80::3ac4:e8ff:fe03:b3e2/64,fd09:4ab5:dae9:b078::ff,0,794,794.00 -"2010-02-10 00:00:01",192.168.0.2,udp,37810,node02.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,,Private,HCVR,HCVR,3.210.1.4,,2K0488CPAGS0ND6,HCVR,Private,client.notifyDevInfo,80,37777,3,0,0,0,9,3c:ef:8c:18:a5:07,192.168.0.2,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::2,fe80::3eef:8cff:fe18:a507/64,fd09:4ab5:dae9:b078::ff,,761,761.00 -"2010-02-10 00:00:02",192.168.0.3,udp,37810,node03.example.com,dvrdhcpdiscover,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",General,HCVR,BCS-XVR0401-IV,4.000.0000002.11,,5L034FAPAZA0E30,XVR,General,client.notifyDevInfo,80,37777,4,0,0,0,0,38:c4:e8:02:74:da,192.168.0.3,192.168.0.240,255.255.255.0,0,fd09:4ab5:dae9:b078::3,fe80::3ac4:e8ff:fe02:74da/64,fd09:4ab5:dae9:b078::ff,,711,711.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_dvr_dhcpdiscover.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv deleted file mode 100644 index c681a8595..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","ok","name","cluster_name","http_code","build_hash","build_timestamp","build_snapshot","lucene_version","tagline","sector" -"2010-02-10 00:00:00",192.168.0.1,tcp,9200,node01.example.com,elasticsearch,2.3.5,64512,ZZ,Region,City,0,0,,"Red Skull",elasticsearch,,90f439ff60a3c0f497f91663701e64ccd01edbb4,2016-07-27T10:36:52Z,false,5.5.0,"You Know, for Search","Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:01",192.168.0.2,tcp,9200,node02.example.com,elasticsearch,7.17.0,64512,ZZ,Region,City,0,0,,allinonepod,docker-cluster,,bee86328705acaa9a6daede7140defd4d9ec56bd,,false,8.11.1,"You Know, for Search","Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,tcp,9200,node03.example.com,elasticsearch,7.15.0,64512,ZZ,Region,City,0,0,,f547c2952610,docker-cluster,,79d65f6e357953a5b3cbcc5e2c7c21073d89aa29,,false,8.9.0,"You Know, for Search","Communications, Service Provider, and Hosting Service" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_elasticsearch.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv deleted file mode 100644 index 4e375a9b4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv +++ /dev/null @@ -1,8 +0,0 @@ -"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","sector","version","servername","url" -"2021-05-14 00:11:30","12.237.1.2",443,"afs-exch-cas2.xxx.com","exchange;cve-2021-26855",7018,"US","CALIFORNIA","TURLOCK",517311,,"Communications, Service Provider, and Hosting Service","15.2.721","AFS-EXCH2019", -"2021-05-14 00:11:37","98.153.3.4",443,"rrcs-98-153-x-x.west.biz.rr.com","exchange;webshell",20001,"US","CALIFORNIA","LOS ANGELES",517311,,"Communications, Service Provider, and Hosting Service","15.0.847","SSAMAIL", -"2021-05-14 00:11:38","206.210.5.6",443,"webmail.xxx.com","exchange;webshell",17054,"US","PENNSYLVANIA","PITTSBURGH",518210,,,"15.0.1178","OMNYXEXCH02", -"2021-05-14 00:11:38","12.33.7.8",443,"mail.xxx.org","exchange;cve-2021-26855",7018,"US","ARKANSAS","LITTLE ROCK",921120,,"Communications, Service Provider, and Hosting Service","15.1.2176","MHASVR02", -"2021-05-14 00:11:38","41.204.9.10",443,"mail.xxx.mg","exchange;cve-2021-26855",21042,"MG","ANTANANARIVO","ANTANANARIVO",,,,,"SABMHQE0232", -"2021-05-14 00:11:38","62.33.11.12",443,,"exchange;cve-2021-26855",20485,"RU","ALTAYSKIY KRAY","BARNAUL",,,,"15.2.659","PV-SRV04", -"2021-05-14 00:11:43","199.33.13.14",443,"mail.xxx.tv","exchange;cve-2021-26855",26481,"US","CALIFORNIA","LOS ANGELES",,,,"15.1.1779","MAIL", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license deleted file mode 100644 index f4e16ec67..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_exchange.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Birger Schacht -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv deleted file mode 100644 index 912e73d84..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","auth_tls_response","auth_ssl_response","tlsv13_support","tlsv13_cipher","jarm","device_vendor","device_type","device_model","device_version","device_sector" -"2019-03-06 06:37:00","61.126.3.70","tcp",21,"arcus-net.co.jp","ftp",4713,"JP","TOKYO","TOKYO",517311,737401,"220 FTP Server ready.|","TLSv1.2","TLS_RSA_WITH_AES_128_CBC_SHA",2048,"*.bizmw.com","GlobalSign Organization Validation CA - SHA256 - G2","Jan 14 08:04:50 2015 GMT","Jan 14 08:04:50 2020 GMT","D9:98:3F:2E:F9:D1:BE:9A:10:1E:DE:51:2C:C1:DF:01:18:0A:20:65","1121DC7421AB7924C3B1D396AEA3707E9E29",2,"sha256WithRSAEncryption","rsaEncryption","NTT Communications Corporation",,"JP","Tokyo","Minato-ku",,,,,,,,"GlobalSign nv-sa",,"BE",,,,,,,,,,"27:4A:8A:3A:A7:DF:82:D0:43:03:0E:6F:48:30:30:C9:24:77:11:1A:08:EF:F7:B9:74:0C:CE:40:87:03:D2:51","E5:93:8B:72:84:0F:35:52:8E:7A:6C:E3:EF:36:90:4C:F2:86:A7:4D:B2:DD:C0:C6:23:83:18:EF:DD:86:34:92:91:57:22:29:75:45:71:8B:3A:CD:F1:27:A9:CA:5F:70:5E:AC:15:A5:E6:63:FD:6F:BB:C5:E2:45:99:73:E9:E6","D1:A7:BC:96:78:1D:16:D0:24:A8:62:7C:3A:95:5A:4A","N","N","N","OV","234 AUTH TLS successful",,,,,,,,, -"2019-03-06 06:37:00","62.48.156.65","tcp",21,"dial-62-48-156-65.ptprime.net","ftp",15525,"PT","LISBOA","FRIELAS",0,0,"220-================================================================| PT Empresas| Acesso Reservado| Acesso nao autorizado punido por lei: 109/91; 67/98| ----------------------------------------------------------------| HENNES & MAURITZ LDA - 149093| SITE: PT303 - Cascais Shopping| MORADA: | NIR: EWS1822940| ================================================================|220 FTP server ready, 1 active clients of 4 simultaneous clients allowed.|",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"500 Syntax error, command unrecognized.","500 Syntax error, command unrecognized.",,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ftp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv deleted file mode 100644 index 26f8ccbcf..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","version","asn","geo","region","city","naics","sic","server_type","clusterid","total_disk","used_disk","free_disk","livenodes","namenodeaddress","volumeinfo" -"2017-09-13 02:06:05","199.116.235.200",50070,,"2.7.3, rbaa91f7c6bc9cb92be5982de4719c1c8af91ccff",15296,"CA","ALBERTA","CALGARY",0,0,"namenode","CID-64471a53-60cb-4302-9832-92f321f111fe",41567956992,53248,25160089600,"edmonton:50010",, -"2017-09-13 02:07:48","104.43.235.92",50075,,"2.7.1.2.4.0.0-169",8075,"US","IOWA","DES MOINES",334111,357101,"datanode","CID-771bae52-9e4f-4ec4-bc1a-c867585751f0",,,,,"sandbox.hortonworks.com","/hadoop/hdfs/data/current" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license deleted file mode 100644 index f8f131c2c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_hadoop.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Sebastian Wagner -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv deleted file mode 100644 index a7e3eb707..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","http_date" -"2018-04-19 00:02:26","75.74.78.113","tcp",8080,"c-75-74-78-113.hsd1.fl.comcast.net","http",7922,"US","FLORIDA","MIAMI",518111,737401,"HTTP/1.1",200,"OK","text/html",,,,"lighttpd",,"chunked","Thu, 19 Apr 2018 00:02:28 GMT" -"2018-04-19 00:02:26","88.162.174.130","tcp",8080,"sto95-3-88-162-174-130.fbx.proxad.net","http",12322,"FR",,"SAINT-OUEN-LAUMONE",518210,737415,"HTTP/1.1",200,"OK","text/html",,,,,17729,,"Thu, 19 Apr 2018 02:02:28 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv deleted file mode 100644 index b1f2330f1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","proxy_authenticate","via","server","content_length","transfer_encoding","http_date" -"2010-02-10 00:00:00",192.168.0.1,tcp,3128,node01.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,,,,,"Wed, 10 Feb 2010 00:00:00 GMT" -"2010-02-10 00:00:01",192.168.0.2,tcp,3128,node02.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,"HTTP/1.1 s_proxy_den1",,,,"Wed, 10 Feb 2010 00:00:01 GMT" -"2010-02-10 00:00:02",192.168.0.3,tcp,3128,node03.example.com,http-connect-proxy,64512,ZZ,Region,City,0,0,HTTP/1.1,200,"Connection established",,,,"HTTP/1.1 s_proxy_yvr",,,,"Wed, 10 Feb 2010 00:00:02 GMT" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_proxy.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv deleted file mode 100644 index 195342533..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","http","http_code","http_reason","content_type","connection","www_authenticate","set_cookie","server","content_length","transfer_encoding","http_date","version","build_date","detail" -"2010-02-10 00:00:00",192.168.0.1,tcp,8080,node01.example.com,"basic-auth,http",64512,ZZ,Region,City,0,0,HTTP/1.1,401,Unauthorized,"text/html; charset=utf-8",,"Basic realm=\"\"OpenWebif\"\"",TWISTED_SESSION=5473ad3faa3de66685fb3a53bffb390b4fcec2039893009a06caf38e1bec8aa8,TwistedWeb/19.7.0,149,,"Wed, 10 Feb 2010 00:00:00 GMT",,, -"2010-02-10 00:00:01",192.168.0.2,tcp,80,node02.example.com,"basic-auth,http",64512,ZZ,Region,City,0,0,HTTP/1.1,401,Unauthorized,"text/html; charset=utf-8",,"Basic realm=\"\"OpenWebif\"\"",TWISTED_SESSION=d2460d37b7fdbdd6c27dd74423ead5704e553d4f2c230672313edc5602059e33,TwistedWeb/19.7.0,149,,"Wed, 10 Feb 2010 00:00:01 GMT",,, -"2010-02-10 00:00:02",192.168.0.3,tcp,443,node03.example.com,git-config-file,64512,ZZ,Region,City,0,0,,,,,,,,,,,"Wed, 10 Feb 2010 00:00:02 GMT",,,"repositoryformatversion = 0;filemode = false;bare = false;logallrefupdates = true;symlinks = false;ignorecase = true" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_http_vulnerable.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv deleted file mode 100644 index d327f1f3b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_id","response_size","raw_response" -2022-03-02 00:34:22,192.168.0.1,tcp,502,host1.example.net,modbus,64512,ZZ,REGION,CITY,0,0,Sector,Vendor 1,device_type,device_model,device_version,0,5,dGVzdDE= -2022-03-02 00:34:22,192.168.0.2,tcp,502,host2.example.net,modbus,64513,ZZ,REGION,CITY,0,0,Sector,Vendor 2,device_type,device_model,device_version,0,5,dGVzdDI= -2022-03-02 00:34:22,192.168.0.3,tcp,502,host3.example.net,modbus,64514,ZZ,REGION,CITY,0,0,Sector,Vendor 3,device_type,device_model,device_version,0,5,dGVzdDM= diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ics.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv deleted file mode 100644 index 87a98157f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv +++ /dev/null @@ -1,96 +0,0 @@ -"timestamp","ip","port","hostname","tag","ipmi_version","asn","geo","region","city","none_auth","md2_auth","md5_auth","passkey_auth","oem_auth","defaultkg","permessage_auth","userlevel_auth","usernames","nulluser","anon_login","error","deviceid","devicerev","firmwarerev","version","manufacturerid","manufacturername","productid","productname","naics","sic","sector" -"2016-07-24 00:09:42","198.51.100.4",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.182",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:42","198.51.100.4",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.182",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.176",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:43","198.51.100.221",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:44","198.51.100.176",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:45","198.51.100.174",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:45","198.51.100.167",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:46","198.51.100.60",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:47","198.51.100.7",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:48","198.51.100.24",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.86",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.231",623,,"ipmi","1.5",20686,"DE","BAYERN","HAPPURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:49","198.51.100.197",623,,"ipmi","2.0",3320,"DE","BERLIN","BERLIN","no","no","yes","yes","yes","default","enabled","enabled","yes","no","yes",,,,,,,,,,541690,874899, -"2016-07-24 00:09:49","198.51.100.87",623,,"ipmi","1.5",3209,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:49","198.51.100.6",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:51","198.51.100.193",623,,"ipmi","2.0",15598,"DE","BAYERN","NUREMBERG","no","yes","yes","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:51","198.51.100.63",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:52","198.51.100.179",623,,"ipmi","2.0",3320,"DE","BAYERN","DENKLINGEN","no","no","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,518210,737415, -"2016-07-24 00:09:53","198.51.100.112",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:53","198.51.100.189",623,,"ipmi","2.0",30134,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0,"Communications" -"2016-07-24 00:09:54","198.51.100.44",623,"198-51-100-44.example.net","ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","yes","yes","no","no","default","disabled","disabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:54","198.51.100.215",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.231",623,"198-51-100-231.example.net","ipmi","2.0",6805,"DE","HAMBURG","HAMBURG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.234",623,,"ipmi","2.0",31103,"DE","THURINGEN","ERFURT","no","no","yes","no","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:55","198.51.100.165",623,,"ipmi","2.0",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:09:56","198.51.100.170",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:09:56","198.51.100.66",623,,"ipmi","2.0",41412,"DE","BAYERN","REGENSBURG","no","yes","yes","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:56","198.51.100.150",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:57","198.51.100.222",623,,"ipmi","2.0",34309,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:57","198.51.100.19",623,,"ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:09:58","198.51.100.83",623,,"ipmi","1.5",3209,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:00","198.51.100.61",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:00","198.51.100.94",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:01","198.51.100.242",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:03","198.51.100.251",623,,"ipmi","2.0",553,"DE","BADEN-WURTTEMBERG","HEIDELBERG","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:03","198.51.100.41",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:04","198.51.100.160",623,"198-51-100-160.example.net","ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:04","198.51.100.243",623,,"ipmi","1.5",2914,"DE","BAYERN","MUNICH","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.190",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.29",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:05","198.51.100.224",623,,"ipmi","2.0",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:06","198.51.100.143",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","HEMER","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.120",623,,"ipmi","2.0",13003,"DE","SACHSEN","LEIPZIG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.196",623,,"ipmi","1.5",20686,"DE","BAYERN","HAPPURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:07","198.51.100.123",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.122",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.192",623,,"ipmi","2.0",34171,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,518210,737415, -"2016-07-24 00:10:08","198.51.100.146",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:08","198.51.100.127",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:09","198.51.100.112",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:09","198.51.100.45",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:09","198.51.100.46",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","NEUSS","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:10","198.51.100.202",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:11","198.51.100.6",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:11","198.51.100.34",623,,"ipmi","2.0",3320,"DE","HESSEN","LEUN","no","yes","yes","no","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,518210,737415, -"2016-07-24 00:10:12","198.51.100.210",623,,"ipmi","2.0",3320,"DE","BADEN-WURTTEMBERG","AALEN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,541690,874899, -"2016-07-24 00:10:12","198.51.100.97",623,,"ipmi","2.0",42730,"DE","BERLIN","BERLIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:12","198.51.100.172",623,,"ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.20",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.181",623,,"ipmi","2.0",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE","no","yes","yes","no","no","default","disabled","disabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.244",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.85",623,,"ipmi","2.0",34309,"DE","HESSEN","FRANKFURT AM MAIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.150",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.154",623,,"ipmi","2.0",196763,"DE","SAARLAND","ST. INGBERT","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:13","198.51.100.83",623,,"ipmi","2.0",31342,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.6",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.228",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:14","198.51.100.150",623,,"ipmi","2.0",44066,"DE","BAYERN","MUNICH","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:15","198.51.100.71",623,,"ipmi","2.0",44066,"DE","BAYERN","MUNICH","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:15","198.51.100.239",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:17","198.51.100.46",623,"198-51-100-53.example.net","ipmi","2.0",29083,"DE","BRANDENBURG","MAHLOW","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:17","198.51.100.78",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:18","198.51.100.164",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,812990,489999, -"2016-07-24 00:10:18","198.51.100.142",623,,"ipmi","2.0",34568,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:18","198.51.100.85",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.173",623,,"ipmi","1.5",2914,"DE","BERLIN","BERLIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.180",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.119",623,,"ipmi","2.0",12843,"DE","RHEINLAND-PFALZ","SPEYER","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:19","198.51.100.183",623,,"ipmi","1.5",12348,"DE","BAYERN","NUREMBERG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:20","198.51.100.108",623,,"ipmi","2.0",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:20","198.51.100.221",623,"198-51-100-156.example.net","ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","yes","yes","yes","yes","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0,"Information Technology" -"2016-07-24 00:10:21","198.51.100.200",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.162",623,,"ipmi","1.5",30766,"DE","HESSEN","BENSHEIM","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.140",623,,"ipmi","2.0",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:21","198.51.100.121",623,,"ipmi","2.0",34549,"DE","HESSEN","FRANKFURT AM MAIN","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:22","198.51.100.33",623,,"ipmi","2.0",47215,"DE","NORDRHEIN-WESTFALEN","GUTERSLOH","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:22","198.51.100.203",623,,"ipmi","2.0",201011,"DE","BAYERN","NUREMBERG","no","yes","yes","yes","no","default","enabled","enabled","yes","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:23","198.51.100.16",623,,"ipmi","2.0",28753,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:24","198.51.100.166",623,,"ipmi","2.0",24940,"DE","BAYERN","GUNZENHAUSEN","no","no","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.135",623,,"ipmi","1.5",2914,"DE","BAYERN","REGENSBURG","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.154",623,"198-51-100-154.example.net","ipmi","1.5",2914,"DE","HESSEN","FRANKFURT AM MAIN","yes","no","yes","yes","no","-","enabled","enabled","no","yes","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.237",623,,"ipmi","2.0",12586,"DE","HESSEN","FRANKFURT AM MAIN","no","no","no","no","no","default","disabled","enabled","yes","no","no",,,,,,,,,,0,0, -"2016-07-24 00:10:25","198.51.100.45",623,,"ipmi","2.0",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF","no","yes","yes","yes","no","default","enabled","enabled","yes","no","no",,,,,,,,,,0,0, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipmi.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv deleted file mode 100644 index a585db6eb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv +++ /dev/null @@ -1,2 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","ipp_version","cups_version","printer_uris","printer_name","printer_info","printer_more_info","printer_make_and_model","printer_firmware_name","printer_firmware_string_version","printer_firmware_version","printer_organization","printer_organization_unit","printer_uuid","printer_wifi_ssid","device_vendor","device_type","device_model","device_version","device_sector" -"2020-06-08 11:30:14","123.45.67.89","tcp",631,"some.host.com","ipp",12345,"AA","REGION","CITY",517311,0,"IPP/2.1","CUPS/2.0","ipp://123.45.67.89:631/ipp/print","NPI3F0D22","HP Color LaserJet MFP M277dw","http://123.45.67.89:631/hp/device/info_config_AirPrint.html?tab=Networking&menu=AirPrintStatus","HP Color LaserJet MFP M277dw",20191203,20191203,20191203,"org","unit","urn:uuid:456e4238-4a44-4643-4c42-10e1813f0a18","wifissid",,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license deleted file mode 100644 index 476908eeb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ipp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv deleted file mode 100644 index cef6b027c..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","initiator_spi","responder_spi","next_payload","exchange_type","flags","message_id","next_payload2","domain_of_interpretation","protocol_id","spi_size","notify_message_type" -"2019-09-04 00:17:25","198.123.245.42","udp",500,"example.local","isakmp-vulnerable",5678,"AA","LOCATION","LOCATION",517311,0,"3e35c70729dfedef","253acab7cbfda607",11,05,00,00000000,00,00,,0,14 -"2019-09-04 00:17:28","198.123.245.67","udp",500,"example.local","isakmp-vulnerable",20255,"AA","LOCATION","LOCATION",0,0,"3e35c70729dfedef","b274460e7adc1bf0",11,05,00,00000000,00,00,,0,14 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_isakmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv deleted file mode 100644 index ab71b9a15..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","http","http_code","http_reason","content_type","server","date","major","minor","git_version","git_commit","git_tree_state","build_date","go_version","compiler","platform","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,6443,node01.example.com,kubernetes,,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:13 GMT",1,20,v1.20.13,2444b3347a2c45eb965b182fb836e1f51dc61b70,clean,2021-11-17T13:00:29Z,go1.15.15,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:01",192.168.0.2,tcp,6443,node02.example.com,kubernetes,,64512,ZZ,Region,City,0,0,"Retail Trade",HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:12 GMT",1,23,v1.23.3+e419edf,6f5a5295923a614a4202a7ad274b38b69f9ca8c0,clean,2022-02-25T06:26:46Z,go1.17.5,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:02",192.168.0.3,tcp,6443,node03.example.com,kubernetes,,64512,ZZ,Region,City,0,0,,HTTP/1.1,200,OK,application/json,,"Tue, 10 May 2022 14:24:12 GMT",1,16+,v1.16.9-aliyun.1,4f7ea78,,2020-05-08T07:29:59Z,go1.13.9,gc,linux/amd64,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,N,Y,unknown,N,"x509: failed to load system roots and no roots provided",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_kubernetes.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv deleted file mode 100644 index 54121fd3b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","size","configuration_naming_context","current_time","default_naming_context","dns_host_name","domain_controller_functionality","domain_functionality","ds_service_name","forest_functionality","highest_committed_usn","is_global_catalog_ready","is_synchronized","ldap_service_name","naming_contexts","root_domain_naming_context","schema_naming_context","server_name","subschema_subentry","supported_capabilities","supported_control","supported_ldap_policies","supported_ldap_version","supported_sasl_mechanisms","amplification" -"2010-02-10 00:00:00",192.168.0.1,tcp,389,node01.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",,"DC=ad,DC=example,DC=com",node01.example.com,7,,"CN=Configuration,DC=ad,DC=example,DC=com",2,,,,node01.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5, -"2010-02-10 00:00:01",192.168.0.2,tcp,389,node02.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",20220821124435.0Z,"DC=ad,DC=example,DC=com",node02.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,25029662,TRUE,TRUE,node02.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5, -"2010-02-10 00:00:02",192.168.0.3,tcp,389,node03.example.com,ldap-tcp,64512,ZZ,Region,City,0,0,0,"CN=Configuration,DC=ad,DC=example,DC=com",20220821124539.0Z,"DC=ad,DC=example,DC=com",node03.example.com,,,"CN=Configuration,DC=ad,DC=example,DC=com",,,,,node03.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_tcp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv deleted file mode 100644 index 3cd5021c5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","size","configuration_naming_context","current_time","default_naming_context","dns_host_name","domain_controller_functionality","domain_functionality","ds_service_name","forest_functionality","highest_committed_usn","is_global_catalog_ready","is_synchronized","ldap_service_name","naming_contexts","root_domain_naming_context","schema_naming_context","server_name","subschema_subentry","supported_capabilities","supported_control","supported_ldap_policies","supported_ldap_version","supported_sasl_mechanisms","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,389,node01.example.com,ldap-udp,64512,ZZ,Region,City,0,0,3038,"CN=Configuration,DC=ad,DC=example,DC=com",20220821044533.0Z,"DC=ad,DC=example,DC=com",node01.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,222537,TRUE,TRUE,node01.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5,58.42 -"2010-02-10 00:00:01",192.168.0.2,udp,389,node02.example.com,ldap-udp,64512,ZZ,Region,City,0,0,3062,"CN=Configuration,DC=ad,DC=example,DC=com",20220821044948.0Z,"DC=ad,DC=example,DC=com",node02.example.com,7,7,"CN=Configuration,DC=ad,DC=example,DC=com",7,1478714,TRUE,TRUE,node02.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",1.2.840.113556.1.4.800|1.2.840.113556.1.4.1670|1.2.840.113556.1.4.1791|1.2.840.113556.1.4.1935|1.2.840.113556.1.4.2080|1.2.840.113556.1.4.2237,1.2.840.113556.1.4.319|1.2.840.113556.1.4.801|1.2.840.113556.1.4.473|1.2.840.113556.1.4.528|1.2.840.113556.1.4.417|1.2.840.113556.1.4.619|1.2.840.113556.1.4.841|1.2.840.113556.1.4.529|1.2.840.113556.1.4.805|1.2.840.113556.1.4.521|1.2.840.113556.1.4.970|1.2.840.113556.1.4.1338|1.2.840.113556.1.4.474|1.2.840.113556.1.4.1339|1.2.840.113556.1.4.1340|1.2.840.113556.1.4.1413|2.16.840.1.113730.3.4.9|2.16.840.1.113730.3.4.10|1.2.840.113556.1.4.1504|1.2.840.113556.1.4.1852|1.2.840.113556.1.4.802|1.2.840.113556.1.4.1907|1.2.840.113556.1.4.1948|1.2.840.113556.1.4.1974|1.2.840.113556.1.4.1341|1.2.840.113556.1.4.2026|1.2.840.113556.1.4.2064|1.2.840.113556.1.4.2065|1.2.840.113556.1.4.2066|1.2.840.113556.1.4.2090|1.2.840.113556.1.4.2205|1.2.840.113556.1.4.2204|1.2.840.113556.1.4.2206|1.2.840.113556.1.4.2211|1.2.840.113556.1.4.2239|1.2.840.113556.1.4.2255|1.2.840.113556.1.4.2256|1.2.840.113556.1.4.2309|1.2.840.113556.1.4.2330|1.2.840.113556.1.4.2354,MaxPoolThreads|MaxPercentDirSyncRequests|MaxDatagramRecv|MaxReceiveBuffer|InitRecvTimeout|MaxConnections|MaxConnIdleTime|MaxPageSize|MaxBatchReturnMessages|MaxQueryDuration|MaxDirSyncDuration|MaxTempTableSize|MaxResultSetSize|MinResultSets|MaxResultSetsPerConn|MaxNotificationPerConn|MaxValRange|MaxValRangeTransitive|ThreadMemoryLimit|SystemMemoryLimitPercent,3|2,GSSAPI|GSS-SPNEGO|EXTERNAL|DIGEST-MD5,58.88 -"2010-02-10 00:00:02",192.168.0.3,udp,389,node03.example.com,ldap-udp,64512,ZZ,Region,City,0,0,36,"CN=Configuration,DC=ad,DC=example,DC=com",,"DC=ad,DC=example,DC=com",node03.example.com,,,"CN=Configuration,DC=ad,DC=example,DC=com",,,,,node03.example.com,"DC=ad,DC=example,DC=com|CN=Configuration,DC=example,DC=com|CN=Schema,CN=Configuration,DC=example,DC=com","DC=example,DC=com","CN=Schema,CN=Configuration,DC=example,DC=com","CN=Servers,CN=Default-First-Site-Name,CN=Sites,CN=Configuration,DC=example,DC=com","CN=Aggregate,CN=Schema,CN=Configuration,DC=example,DC=com",,,,,,0.69 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ldap_udp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv deleted file mode 100644 index 4a97121e7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","mdns_name","mdns_ipv4","mdns_ipv6","services","workstation_name","workstation_ipv4","workstation_ipv6","workstation_info","http_name","http_ipv4","http_ipv6","http_ptr","http_info","http_target","http_port","spotify_name","spotify_ipv4","spotify_ipv6","opc_ua_discovery" -"2010-02-10 00:00:00",192.168.0.1,udp,5353,node01.example.com,mdns,64512,ZZ,Region,City,0,0,,,,"_smb._tcp.local.; _device-info._tcp.local.; _http._tcp.local.; _dacp._tcp.local.;",,192.168.0.1,fd09:4ab5:dae9:b078::1,,,192.168.0.1,fd09:4ab5:dae9:b078::1,,,,,,,, -"2010-02-10 00:00:01",192.168.0.2,udp,5353,node02.example.com,mdns,64512,ZZ,Region,City,0,0,,,,_home-assistant._tcp.local.;,,192.168.0.2,fd09:4ab5:dae9:b078::2,,,192.168.0.2,fd09:4ab5:dae9:b078::2,,,,,,,, -"2010-02-10 00:00:02",192.168.0.3,udp,5353,node03.example.com,"mdns,iot",64512,ZZ,Region,City,0,0,,,,"_webdav._tcp.local.; _adisk._tcp.local.; _smb._tcp.local.; _http._tcp.local.; _dacp._tcp.local.; _afpovertcp._tcp.local.; _device-info._tcp.local.;",,192.168.0.3,fd09:4ab5:dae9:b078::3,,snmeijer.local.,192.168.0.3,fd09:4ab5:dae9:b078::3,snmeijer._http._tcp.local.,"\"\"vendor=Synology\"\" \"\"model=DS218+\"\" \"\"serial=17A0PCN482002\"\" \"\"version_major=6\"\" \"\"version_minor=2\"\" \"\"version_build=25556\"\" \"\"admin_port=5000\"\" \"\"secure_admin_port=5001\"\" \"\"mac_address=00:11:32:80:fd:b5\"\"",snmeijer.local.,5000,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mdns.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv deleted file mode 100644 index 6a1d445e7..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","pid","pointer_size","uptime","time","curr_connections","total_connections","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,50260,node01.example.com,memcached,1.4.15,64512,ZZ,Region,City,0,0,1010,64,32908114,"2022-08-21 10:34:06",243,6106,"Communications, Service Provider, and Hosting Service",1144,81.71 -"2010-02-10 00:00:01",192.168.0.2,udp,11211,node02.example.com,memcached,1.4.13,64512,ZZ,Region,City,0,0,5316,64,9618498,"2022-08-21 10:39:21",9,2962,"Communications, Service Provider, and Hosting Service",1053,75.21 -"2010-02-10 00:00:02",192.168.0.3,udp,11211,node03.example.com,memcached,1.2.6,64512,ZZ,Region,City,0,0,1460,32,1375159,"2022-08-21 10:39:39",2,534,,442,31.57 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_memcached.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv deleted file mode 100644 index 1228dcfc6..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv +++ /dev/null @@ -1,11 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","gitversion","sysinfo","opensslversion","allocator","javascriptengine","bits","maxbsonobjectsize","ok","visible_databases","sector" -"2016-07-24 00:40:07","198.51.100.203","tcp",27017,"198-51-100-203.example.net","mongodb","2.4.5",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"a2ddc68ba7c9cee17bfe69ed840383ec3506602b","Linux ip-198-51-100-100 198.51.100.103-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49",,"tcmalloc","V8",64,16777216,1,"local | countly | admin", -"2016-07-24 00:40:07","198.51.100.42","tcp",27017,"198-51-100-208.example.net","mongodb","2.6.12",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"d73c92b1c85703828b55c2916a5dd4ad46535f6a","Linux build5.ny.cbi.10gen.cc 2.6.32-431.3.1.el6.x86_64 #1 SMP Fri Jan 3 21:39:27 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49",,"tcmalloc","V8",64,16777216,1,"none visible","Information Technology" -"2016-07-24 00:40:07","198.51.100.225","tcp",27017,"198-51-100-225.example.net","mongodb","3.0.6",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,"1ef45a23a4c5e3480ac919b28afcba3c615488f2","Linux ip-198-51-100-100 3.4.43-43.43.amzn1.x86_64 #1 SMP Mon May 6 18:04:41 UTC 2013 x86_64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.0-fips 29 Mar 2010","tcmalloc","V8",64,16777216,1,"bluu | local","Communications" -"2016-07-24 00:40:07","198.51.100.144","tcp",27017,"198-51-100-144.example.net","mongodb","2.2.2",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"d1b43b61a5308c4ad0679d34b262c5af9d664267","Linux ip-198-51-100-100 198.51.100.252-2.ec2.v1.2.fc8xen #1 SMP Fri Nov 20 17:48:28 EST 2009 x86_64 BOOST_LIB_VERSION=1_49",,,,64,16777216,1,"errbit_production | DELETED_BECAUSE_YOU_DIDNT_PASSWORD_PROTECT_YOUR_MONGODB | admin | local", -"2016-07-24 00:40:07","198.51.100.68","tcp",27017,,"mongodb","3.2.6",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,"05552b562c7a0b3143a729aaa0838e558dc49b25","deprecated",,"tcmalloc","mozjs",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.101","tcp",27017,,"mongodb","3.0.9",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,"20d60d3491908f1ae252fe452300de3978a040c7","Linux ip-198-51-100-100 3.13.0-24-generic #46-Ubuntu SMP Thu Apr 10 19:11:08 UTC 2014 x86_64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.1f 6 Jan 2014","tcmalloc","V8",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.53","tcp",27017,"198-51-100-162.example.net","mongodb","3.2.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"05552b562c7a0b3143a729aaa0838e558dc49b25","deprecated",,"tcmalloc","mozjs",64,16777216,1,"none visible", -"2016-07-24 00:40:07","198.51.100.206","tcp",27017,"198-51-100-206.example.net","mongodb","2.4.10",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,"e3d78955d181e475345ebd60053a4738a4c5268a","Linux bs-linux32.10gen.cc 198.51.100.34-2.fc8xen #1 SMP Fri Feb 15 12:39:36 EST 2008 i686 BOOST_LIB_VERSION=1_49",,"system","V8",32,16777216,1,"sharelatex | test1 | local | tmp | lococms_production", -"2016-07-24 00:40:10","198.51.100.157","tcp",27017,"198-51-100-157.example.net","mongodb","2.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"nogitversion","Linux biber 2.6.32-5-amd64 #1 SMP Mon Feb 25 00:26:11 UTC 2013 i686 BOOST_LIB_VERSION=1_49",,,,32,16777216,1,"none visible", -"2016-07-24 00:40:10","198.51.100.173","tcp",27017,"198-51-100-173.example.net","mongodb","2.6.12",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,"nogitversion","FreeBSD 101amd64-default-job-24 10.1-RELEASE-p33 FreeBSD 10.1-RELEASE-p33 amd64 BOOST_LIB_VERSION=1_49","OpenSSL 1.0.1l-freebsd 15 Jan 2015","system","V8",64,16777216,1,"none visible", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mongodb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv deleted file mode 100644 index cfe4f0061..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv +++ /dev/null @@ -1,2 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","anonymous_access","raw_response","hex_code","code","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serialNumber" -"2022-02-07 12:56:53","18.220.0.0","tcp",8883,"18-220-0-0.example.com","mqtt",12345,"US","OHIO","COLUMBUS",454110,,"N",20020005,05,"Connection Refused, not authorized","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"*.tracesafe.io","Sectigo RSA Domain Validation Secure Server CA","2020-08-12 00:00:00","2022-11-14 00:00:00","70:84:F1:6D:28:DA:B6:E6:27:60:13:8B:2C:93:52:B6:7B:4B:13:7B","D2:D7:54:52:EB:86:4E:2D:34:4D:FC:CE:CD:CF:39:41:E1:06:5C:8B:B8:54:E6:0C:DF:FD:6E:E3:F1:B5:41:00","17:57:FB:88:9D:BE:A7:F0:29:A5:31:FC:79:DF:F7:8A:1C:D6:4A:DF:1B:4A:DC:BF:05:E7:E8:2F:79:9A:FA:FE:F7:E8:66:22:CB:B9:4C:72:F7:FB:6C:1D:59:8C:54:63:70:05:DE:7F:3C:2F:BA:B8:37:18:CE:29:6F:11:E8:AB","DE:2C:98:30:27:2E:7D:C9:ED:A3:9D:AF:9E:CE:14:CC","085699743A23114C9B6B8DC975A8AF42",2,"sha256WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,"Sectigo Limited",,"GB","Greater Manchester","Salford",,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license deleted file mode 100644 index 476908eeb..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 Thomas Hungenberg -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv deleted file mode 100644 index e0ab4b929..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","raw_response","hex_code","code","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serialNumber" -"2022-01-10 00:59:34","47.106.0.0","tcp",8883,,"mqtt,mqtt-anon",37963,"CN","GUANGDONG SHENG","SHENZHEN",518210,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"Server","RootCA","2020-05-08 08:07:05","2030-05-06 08:07:05","70:1A:1E:1F:EC:5F:7E:A9:12:32:B2:C9:8A:C9:EE:91:8E:0B:82:45","85:26:A2:F2:A2:50:CD:96:33:19:A6:2D:12:2E:97:6B:D3:06:3C:11:EA:01:B4:B7:25:2A:B7:4F:0A:8F:45:40","72:50:07:30:9A:6F:CB:FD:E2:80:69:02:65:62:77:16:C3:B4:0C:98:44:4E:D4:2C:AC:6B:AF:F8:9E:AB:51:C2:FA:A8:72:A3:45:DF:81:09:50:08:18:EB:03:34:FC:92:33:A7:12:46:FE:90:20:91:86:C5:4D:89:48:86:4C:CD","AB:A8:E0:2C:EF:AE:BF:9D:DD:FA:70:BA:2F:F2:CA:5C",02,2,"sha256WithRSAEncryption","rsaEncryption","EMQ",,"CN","hangzhou",,,,,,,,,"EMQ",,"CN","hangzhou",,,,,,,, -"2022-01-10 00:59:34","144.76.0.0","tcp",8883,,"mqtt,mqtt-anon",24940,"DE","SACHSEN-ANHALT","WERNIGERODE",518210,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",2048,"example.com","R3","2021-12-06 13:48:04","2022-03-06 13:48:03","20:0E:AC:E7:AF:07:8D:D3:16:7C:63:D1:B9:12:AD:1D:2C:F0:46:86","DD:7A:4C:8A:1D:66:1D:7C:F5:17:04:5B:A0:B4:C4:E0:80:58:44:B4:DB:A7:5E:61:AE:43:9D:85:4C:9E:DC:83","55:B6:3D:56:A4:39:6E:99:B6:AF:72:AF:4D:3C:7C:C5:A8:C5:4F:A1:79:92:D0:46:8A:A2:9B:2A:48:0D:00:68:39:F0:B8:67:B4:E0:88:51:2A:D7:55:46:83:BD:ED:1E:09:6E:DB:3D:21:E2:AA:DB:42:6A:33:45:1A:2A:DB:4C","23:99:39:C6:77:D8:9F:55:90:FC:A5:FB:BA:72:8B:42","06B25BEAD1F43266ABCFCDDE408D3544D04B",2,"sha256WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,"Lets Encrypt",,"US",,,,,,,,, -"2022-01-10 00:59:34","173.0.0.0","tcp",8883,"example.com","mqtt,mqtt-anon",5555,"US","CALIFORNIA","BURBANK",,,20020000,00,"Connection Accepted","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA",2048,"foo.example.com","ClearView2Dev","2020-08-07 16:51:57","2030-08-05 16:51:57","32:4B:66:98:FA:5B:D2:D1:F2:53:83:21:19:11:5A:A9:BE:85:56:16","AE:0D:65:34:2F:51:F7:32:1E:DF:B1:DA:12:C7:6A:DE:42:B5:4B:FF:80:2C:E5:EF:99:F6:CC:01:4B:C9:77:68","44:C4:B8:19:FA:39:55:51:EC:E4:6D:C4:6D:0F:A5:46:BB:D5:F9:FD:A6:8D:DF:F3:2D:D2:92:6C:0B:D5:D3:25:CB:19:50:9D:A6:A4:D4:D3:2E:53:10:F5:8D:77:F7:90:F8:65:A7:79:AB:14:62:72:01:F3:EA:38:E2:68:C7:25","43:0D:A7:89:9E:76:8D:6E:D5:AD:95:CC:F2:91:87:56","A71541EFAE529B03",0,"sha256WithRSAEncryption","rsaEncryption","Sohonet",,,,"<",,,,,,,,"Sohonet","ClearView2Dev",,,,,,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mqtt_anon.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv deleted file mode 100644 index c12a6063e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","server_name","instance_name","tcp_port","named_pipe","response_size","amplification","sector" -"2010-02-10 00:00:00",192.168.0.1,udp,1434,node01.example.com,mssql,13.2.5026.0,64512,ZZ,Region,City,0,0,ERPOPTIMA,OPTIMA,49729,"\\\\ERPOPTIMA\\pipe\\MSSQL$OPTIMA\\sql\\query",310,310.00, -"2010-02-10 00:00:01",192.168.0.2,udp,1434,node02.example.com,mssql,13.0.1601.5,64512,ZZ,Region,City,0,0,SERWER,MSSQLSERVER,1433,,226,226.00,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,udp,1434,node03.example.com,mssql,10.50.2500.0,64512,ZZ,Region,City,0,0,ILONY,INSERTGT,49358,"\\\\ILONY\\pipe\\MSSQL$INSERTGT\\sql\\query",304,304.00, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mssql.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv deleted file mode 100644 index 25fed2166..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","mysql_protocol_version","server_version","error_code","error_id","error_message","client_can_handle_expired_passwords","client_compress","client_connect_attrs","client_connect_with_db","client_deprecated_eof","client_found_rows","client_ignore_sigpipe","client_ignore_space","client_interactive","client_local_files","client_long_flag","client_long_password","client_multi_results","client_multi_statements","client_no_schema","client_odbc","client_plugin_auth","client_plugin_auth_len_enc_client_data","client_protocol_41","client_ps_multi_results","client_reserved","client_secure_connection","client_session_track","client_ssl","client_transactions","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,3306,node01.example.com,mysql,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",10,5.7.37-0ubuntu0.18.04.1,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:01",192.168.0.2,tcp,3306,node02.example.com,mysql,,64512,ZZ,Region,City,0,0,,10,5.7.30-0ubuntu0.18.04.1-log,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, -"2010-02-10 00:00:02",192.168.0.3,tcp,3306,node03.example.com,mysql,,64512,ZZ,Region,City,0,0,"Retail Trade",10,8.0.23,1,1,1,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,Y,N,N,N,TLSv1.2,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,2,sha256WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,unknown,,"x509: failed to load system roots and no roots provided",, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_mysql.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv deleted file mode 100644 index e8a1108d5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","opcode","uptime","external_ip","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,5351,node01.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,291278940,192.168.0.1,,12,6.00 -"2010-02-10 00:00:01",192.168.0.2,udp,5351,node02.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,768416,192.168.0.2,,12,6.00 -"2010-02-10 00:00:02",192.168.0.3,udp,5351,node03.example.com,nat-pmp,0,64512,ZZ,Region,City,0,0,128,19629454,192.168.0.3,,12,6.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_nat_pmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv deleted file mode 100644 index 932225b0b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","mac_address","asn","geo","region","city","workgroup","machine_name","username","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,137,node01.example.com,netbios,00-00-00-00-00-00,64512,ZZ,Region,City,,NBG6503,NBG6503,0,0,,229,4.58 -"2010-02-10 00:00:01",192.168.0.2,udp,137,node02.example.com,netbios,00-00-00-00-00-00,64512,ZZ,Region,City,PRACOWNIAELN.,NAS-OLD,NAS-OLD,0,0,,193,3.86 -"2010-02-10 00:00:02",192.168.0.3,udp,137,node03.example.com,netbios,00-25-90-F0-64-64,64512,ZZ,Region,City,HRSIGMA,HR-SRV01,,0,0,Government,157,3.14 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netbios.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv deleted file mode 100644 index 4e9159356..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","response","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,53413,node01.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 -"2010-02-10 00:00:01",192.168.0.2,53413,node02.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 -"2010-02-10 00:00:02",192.168.0.3,53413,node03.example.com,netis_vulnerability,Login:,64512,ZZ,Region,City,0,0,,18,18.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_netis_router.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv deleted file mode 100644 index cc3cf6fc2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","version","clk_wander","clock","error","frequency","jitter","leap","mintc","noise","offset","peer","phase","poll","precision","processor","refid","reftime","rootdelay","rootdispersion","stability","state","stratum","system","tai","tc","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,123,node01.example.com,ntpversion,64512,ZZ,Region,City,4,,0xe6ac3809.363028e7,,2.018,0.977,0,,0.984,0.557,18986,,10,-10,unknown,81.15.252.130,0xe6ac35ba.2d2e8f2b,17.685,61.254,0.027,4,4,UNIX,,,0,0,,324,27.00 -"2010-02-10 00:00:01",192.168.0.2,udp,123,node02.example.com,ntpversion,64512,ZZ,Region,City,4,0.007,0xE6AC3806.7DF3B7A0,,-20.407,8.776,0,3,,-14.502,19244,,,-10,unknown,10.48.21.21,0xE6AC3431.B3B64790,32.25,105.778,,,8,UNIX,,10,0,0,"Transportation and Warehousing",328,27.33 -"2010-02-10 00:00:02",192.168.0.3,udp,123,node03.example.com,ntpversion,64512,ZZ,Region,City,4,0.001,0xE6AC380A.5A1CAD00,,-24.01,2.343,0,3,,0.49,51892,,,-10,unknown,172.28.0.1,0xE6AC3020.0C49BA80,7.749,81.612,,,4,UNIX,,10,0,0,,324,27.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv deleted file mode 100644 index dca5386d9..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","packets","size","asn","geo","region","city","naics","sic","sector","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,123,node01.example.com,2,664,64512,ZZ,Region,City,0,0,,55.33 -"2010-02-10 00:00:01",192.168.0.2,udp,123,node02.example.com,100,44000,64512,ZZ,Region,City,0,0,,3666.67 -"2010-02-10 00:00:02",192.168.0.3,udp,123,node03.example.com,100,44000,64512,ZZ,Region,City,0,0,,3666.67 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ntpmonitor.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv deleted file mode 100644 index c32bc3d4d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","programs","mountd_port","exports","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,111,node01.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0","Communications, Service Provider, and Hosting Service",148,3.70 -"2010-02-10 00:00:01",192.168.0.2,udp,111,node02.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0","Communications, Service Provider, and Hosting Service",148,3.70 -"2010-02-10 00:00:02",192.168.0.3,udp,111,node03.example.com,portmapper,64512,ZZ,Region,City,0,0,"100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp; 100000 4 111/udp; 100000 3 111/udp; 100000 2 111/udp;",,"/mnt/export 192.168.0.0",Government,148,3.70 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_portmapper.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv deleted file mode 100644 index 8c1d6f725..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","sector","supported_protocols","protocol_error_code","protocol_error_file","protocol_error_line","protocol_error_message","protocol_error_routine","protocol_error_severity","protocol_error_severity_v","startup_error_code","startup_error_file","startup_error_line","startup_error_message","startup_error_routine","startup_error_severity","startup_error_severity_v","client_ssl","handshake","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","cert_valid","self_signed","cert_expired","validation_level","browser_trusted","browser_error","raw_cert","raw_cert_chain" -"2010-02-10 00:00:00",192.168.0.1,tcp,5432,node01.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, -"2010-02-10 00:00:01",192.168.0.2,tcp,5432,node02.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, -"2010-02-10 00:00:02",192.168.0.3,tcp,5432,node03.example.com,postgres,,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",1.0-3.0,0A000,postmaster.c,1798,"unsupported frontend protocol 255.255: server supports 1.0 to 3.0",ProcessStartupPacket,FATAL,,28000,postmaster.c,1893,"no PostgreSQL user name specified in startup packet",ProcessStartupPacket,FATAL,,N,,TLS_AES_256_GCM_SHA384,2048,example.com,example.com,"2012-11-14 11:18:27","2021-11-12 11:18:27",03:39:9E:5D:77:19:38:C4:49:DE:C3:3D:9B:E6:13:ED:5A:F1:42:55,B3F13DFBDBA2D8B2,,,rsaEncryption,,,US,,,,,,,,,,,,,,,,,,,,,,E1:D1:6E:87:49:B9:AC:74:B4:AC:9B:77:85:27:69:97:0D:16:B1:F6:63:F0:26:51:AA:89:42:39:66:BD:47:D0,1C:E9:04:22:90:46:68:0B:8B:54:33:38:C6:20:5F:EE:A6:73:A6:B5:2C:7D:12:94:DE:F1:CC:11:2E:72:0B:97:C2:7D:19:BF:E0:6B:98:A9:21:D9:9D:5A:CB:38:0B:D8:7E:E2:8E:2B:EA:15:EC:60:11:1E:41:E3:FB:4C:20:9F,F1:8A:02:48:3C:6B:F4:00:CC:5C:D5:B0:71:E4:FA:00,N,,Y,,,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_postgres.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv deleted file mode 100644 index 857699376..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","quote","asn","geo","region","city","naics","sic","sector","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,17,node01.example.com,qotd,"_The secret of being miserable is to have leisure to bother about whether?? you are happy or not. The cure for it is occupation._?? George Bernard Shaw (1856-1950)?",64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",166,166.00 -"2010-02-10 00:00:01",192.168.0.2,udp,17,node02.example.com,qotd,"_Oh the nerves, the nerves; the mysteries of this machine called man!?? Oh the little that unhinges it, poor creatures that we are!_?? Charles Dickens (1812-70)?",64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",162,162.00 -"2010-02-10 00:00:02",192.168.0.3,udp,17,node03.example.com,qotd,"_Oh the nerves, the nerves; the mysteries of this machine called man!?? Oh the little that unhinges it, poor creatures that we are!_?? Charles Dickens (1812-70)?",64512,ZZ,Region,City,0,0,,162,162.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_qotd.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv deleted file mode 100644 index c9fb18896..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","version_field_1","version_field_2","version_field_3","version_field_4" -"2022-01-10 14:31:17","176.255.0.0","udp",443,"test1.example.com","quic",5607,"UK","LONDON","LONDON",517311,,"Q050",,"Q046","Q043" -"2022-01-10 14:31:17","24.244.0.0","udp",443,,"quic",6327,"CA","SASKATCHEWAN","MEACHAM",517311,,"Q050","Q046",,"Q043" -"2022-01-10 14:31:17","23.60.0.0","udp",443,"test3.example.com","quic",20940,"JP","OSAKA","OSAKA",517919,,,"Q050","Q046","Q043" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_quic.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv deleted file mode 100644 index 76b388aca..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv +++ /dev/null @@ -1,10 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic" -"2020-07-06 13:55:26","74.101.218.75","tcp",4899,"static-74-101-218-75.nycmny.fios.verizon.net","radmin","Radmin (Details Unknown)",701,"US","NEW YORK","BROOKLYN",517312, -"2020-07-06 13:55:27","192.162.189.171","tcp",4899,"rubin.an.ru","radmin","Radmin v3.X Radmin Authentication",56618,"RU","MURMANSKAYA OBLAST","MURMANSK",0, -"2020-07-06 13:55:27","111.197.143.69","tcp",4899,,"radmin","Radmin (Details Unknown)",4808,"CN","BEIJING SHI","BEIJING",517311, -"2020-07-06 13:55:27","121.147.215.220","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",4766,"KR","GWANGJU-GWANGYEOKSI","DAEIN-DONG",517311, -"2020-07-06 13:55:27","121.147.215.178","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",4766,"KR","GWANGJU-GWANGYEOKSI","DAEIN-DONG",517311, -"2020-07-06 13:55:27","183.230.5.219","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",9808,"CN","CHONGQING SHI","CHONGQING",517312, -"2020-07-06 13:55:27","85.93.154.74","tcp",4899,,"radmin","Radmin v3.X Radmin Authentication",34300,"RU","MOSKVA","MOSCOW",0, -"2020-07-06 13:55:27","81.246.135.247","tcp",4899,"247.135-246-81.adsl-dyn.isp.belgacom.be","radmin","Radmin v3.X Radmin Authentication",5432,"BE","ANTWERPEN","BRASSCHAAT",517311, -"2020-07-06 13:55:27","46.27.146.22","tcp",4899,"static-22-146-27-46.ipcom.comunitel.net","radmin","Radmin v3.X Radmin Authentication",12430,"ES","LAS PALMAS","LAS PALMAS DE GRAN CANARIA",517312, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license deleted file mode 100644 index 833024a75..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_radmin.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2020 sinus-x -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv deleted file mode 100644 index 4bac90f19..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","rdp_protocol","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","naics","sic","sector","tlsv13_support","tlsv13_cipher","cve20190708_vulnerable","bluekeep_vulnerable","jarm" -"2019-09-04 15:45:51","198.123.245.178",5678,,"rdp",,5678,"AA","LOCATION","LOCATION","RDP",5678,"KABESRV.KABE.local","KABESRV.KABE.local","2019-04-29 02:22:06","2019-10-29 02:22:06","EC:BB:4D:DB:9F:0C:D3:FF:5B:49:EA:B1:56:62:B6:A7:5D:60:54:42","1EF2B37AF850C9BF4E88F18177001D6B",2,"sha256WithRSAEncryption","rsaEncryption","B7:C9:F4:07:D5:C0:75:1D:EA:0C:40:E7:26:39:C2:30:C6:13:83:7E:18:46:D8:E9:4C:45:3F:88:1B:0B:70:76","08:AC:75:FA:EB:A3:2B:44:15:DE:6D:A7:0B:C0:AE:17:94:F3:55:D9:EC:70:AC:5B:B7:94:79:F0:D7:84:83:89:CB:A9:11:E0:08:D7:54:4D:33:85:89:D2:A8:DD:9D:15:F4:CC:95:DE:6A:E3:DF:6B:FA:8B:27:E3:DA:16:AF:0A","BC:6E:C3:E2:98:22:EC:BA:5B:30:E2:53:FD:4A:9D:FF",517311,0,,,,"N","N" -"2019-09-04 15:45:51","198.123.245.233",5678,,"rdp",,5678,"AA","LOCATION","LOCATION","RDP",5678,"RAMBLA01.rambla.local","RAMBLA01.rambla.local","2019-04-16 06:15:20","2019-10-16 06:15:20","7A:67:1F:F8:87:C6:B0:AC:A9:84:15:B7:40:EC:CB:19:AA:E3:19:52","3FF3EBC5CF154BA54D128A8548C8AAF5",2,"sha1WithRSAEncryption","rsaEncryption","8F:CD:7D:C4:80:2D:8D:9B:06:A0:40:18:9F:ED:73:7A:BA:83:55:BE:1B:56:83:A2:97:DF:BB:B4:06:57:CB:F1","E8:9B:9A:93:69:B4:58:01:D8:46:C2:DC:01:20:1E:DD:93:E1:EB:E3:9D:6B:65:A0:C5:00:6C:A4:44:08:FE:A4:A6:19:FF:55:79:F2:AA:61:68:C8:1C:B0:CE:78:EB:84:DD:29:9D:64:2F:4E:25:31:3A:6C:B8:02:C9:AF:F5:1F","38:73:6A:B3:AA:41:69:C9:BA:E7:3D:D7:40:16:F8:AA",517311,0,"Information Technology",,,"N","N" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv deleted file mode 100644 index 73d0d55ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sessionid","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,3389,node01.example.com,rdpeudp,64512,ZZ,Region,City,0,0,05b28c0c,1232,77.00 -"2010-02-10 00:00:01",192.168.0.2,udp,3389,node02.example.com,rdpeudp,64512,ZZ,Region,City,0,0,053d355f,1232,77.00 -"2010-02-10 00:00:02",192.168.0.3,udp,3389,node03.example.com,rdpeudp,64512,ZZ,Region,City,0,0,0567a8cb,1232,77.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rdpeudp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv deleted file mode 100644 index dc9760cf2..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv +++ /dev/null @@ -1,94 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","version","asn","geo","region","city","naics","sic","git_sha1","git_dirty_flag","build_id","mode","os","architecture","multiplexing_api","gcc_version","process_id","run_id","uptime","connected_clients","sector" -"2016-07-24 00:42:33","198.51.100.152","tcp",6379,,"redis","2.8.19",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"26069fb482f6334b","standalone","Linux 3.2.0-4-amd64 x86_64",,"epoll","4.7.2",2127,"d440b0b2fb3d1db655ad607e11e6f38011a0f599",27946314,50, -"2016-07-24 00:42:43","198.51.100.67","tcp",6379,"198-51-100-67.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310556,25376, -"2016-07-24 00:42:43","198.51.100.125","tcp",6379,"198-51-100-125.example.net","redis","2.8.17",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5937320cdd80c1e4","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.9.2",11573,"0d58143df099738a7ce9330ee5ec2367d11b1187",25888041,4, -"2016-07-24 00:42:43","198.51.100.203","tcp",6379,"198-51-100-203.example.net","redis","2.8.4",31103,"DE","THURINGEN","ERFURT",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-83-generic x86_64",,"epoll","4.8.2",3847,"4f7765dee91d8c4b1b24604cc5f0c29fca1a4f32",3068554,38, -"2016-07-24 00:42:43","198.51.100.240","tcp",6379,"198-51-100-30.example.net","redis","3.0.7",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.4",1011,"864c8d7df1e72c662a4edd77b6df6cd30161af6e",2476542,2,"Information Technology" -"2016-07-24 00:42:49","198.51.100.69","tcp",6379,"198-51-100-69.example.net","redis","3.0.6",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"315c8c74805fca88","standalone","Linux 3.2.0-98-generic x86_64",,"epoll","4.6.3",28961,"bc705102c854ea1818213e4740a3c6fd9b9f1716",4633191,1, -"2016-07-24 00:42:53","198.51.100.50","tcp",6379,"198-51-100-50.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"6afb1e1f0d80abd0","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.4.5",1717,"f729595b3642b48f3ac9e098bcccab1d6ef82e3e",6345372,3, -"2016-07-24 00:43:49","198.51.100.113","tcp",6379,,"redis","3.0.6",24961,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310623,24628, -"2016-07-24 00:43:49","198.51.100.228","tcp",6379,"198-51-100-131.example.net","redis","2.8.210",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"83ad777ec89a946b","standalone","Windows",,"winsock_IOCP",,1948,"f5d6ad26e423039636afaf3918ee7e6a7e0b5b68",2214134,4,"Information Technology" -"2016-07-24 00:43:59","198.51.100.155","tcp",6379,,"redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"f09a0843cc9876c3","standalone","Linux 3.13.0-79-generic x86_64",,"epoll","4.9.2",1,"5f4f5b7158f928cc96e3ae6af6092a163ace15eb",2897902,24, -"2016-07-24 00:43:59","198.51.100.171","tcp",6379,,"redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310633,25031, -"2016-07-24 00:44:09","198.51.100.230","tcp",6379,"198-51-100-230.example.net","redis","2.8.11",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"f26bfdf4b8265fc","standalone","Linux 3.8.0-29-generic x86_64",,"epoll","4.6.3",14551,"0d001175cf26cee88486d814b4f0c972a5aa89b9",21038337,9, -"2016-07-24 00:44:09","198.51.100.182","tcp",6379,"198-51-100-182.example.net","redis","3.0.7",197540,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"fd24f54fec00684b","standalone","Linux 3.13.0-85-generic x86_64",,"epoll","4.8.4",949,"b11fdf2b95251b8e6c3e9e782409ef82fc8b89aa",8643389,11, -"2016-07-24 00:44:10","198.51.100.23","tcp",6379,"198-51-100-116.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 4.2.0-27-generic x86_64",,"epoll","4.8.2",335,"90079d58e970a1ae94aa91bc0ea0236a0e55269c",4930922,2,"Information Technology" -"2016-07-24 00:44:19","198.51.100.51","tcp",6379,"198-51-100-51.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310652,26257, -"2016-07-24 00:44:22","198.51.100.88","tcp",6379,,"redis","3.0.6",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310656,26371, -"2016-07-24 00:44:22","198.51.100.107","tcp",6379,"octopus-dev","redis","2.8.14",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"78be6d5e32e34139","standalone","Linux 2.6.32-042stab108.2 x86_64",,"epoll","4.8.2",21205,"b98a41b6ea690c207527587f60bff1f1d24236b4",9364864,4, -"2016-07-24 00:44:22","198.51.100.75","tcp",6379,,"redis","3.0.0",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"2b5201a6bfd5f75e","standalone","Linux 3.11.0-19-generic x86_64",,"epoll","4.8.2",832,"2bdcda8b3b59cef244785b58935d68daf48645be",6745479,5, -"2016-07-24 00:44:25","198.51.100.12","tcp",6379,,"redis","3.0.6",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.13.0-79-generic x86_64",,"epoll","4.8.4",899,"94550e510bf770aa315cc3983ce9958853c77cfe",7816856,9, -"2016-07-24 00:44:27","198.51.100.13","tcp",6379,"198-51-100-13.example.net","redis","3.0.7",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"6f8b503a2787e3a6","standalone","Linux 4.4.5-15.26.amzn1.x86_64 x86_64",,"epoll","4.9.2",1,"e050f40e755a739ffecdb2468e1333f371e2abca",7124048,6,"Communications" -"2016-07-24 00:44:29","198.51.100.12","tcp",6379,"198-51-100-12.example.net","redis","2.8.3",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"992c97be25a6b6d2","standalone","Linux 2.6.32-042stab111.12 x86_64",,"epoll","4.4.5",12340,"d7cda18212cf4bcdfd7c42fff33e506a4e9a2614",16874891,8, -"2016-07-24 00:44:38","198.51.100.66","tcp",6379,"198-51-100-66.example.net","redis","3.2.1",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ed627d97d5dc311e","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"4a6beb721ddbaa411f53e5268e6112127903cae3",2029470,3,"Chemical" -"2016-07-24 00:44:38","198.51.100.170","tcp",6379,,"redis","3.0.6",8881,"DE","SACHSEN","RADEBEUL",0,0,00000000,0,"1b14d17ce6fea422","standalone","Linux 4.2.6-1-pve x86_64",,"epoll","4.9.2",728,"c423ba856285690a2fae350b03514cec80db9d5e",1679635,1, -"2016-07-24 00:44:38","198.51.100.67","tcp",6379,"198-51-100-67.example.net","redis","2.8.23",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"8e819a849ea2d7f8","standalone","Linux 4.2.0-23-generic x86_64",,"epoll","4.9.2",1,"7ee1dc403540ff4d1fc0a80d9f0b2910857b6c1b",9451832,68,"Information Technology" -"2016-07-24 00:44:44","198.51.100.238","tcp",6379,,"redis","2.8.4",44066,"DE","BAYERN","MUNICH",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 2.6.32-19-pve x86_64",,"epoll","4.8.2",2207,"6a079396cc44c1aca745edab13f4014c394da3ab",10338949,3, -"2016-07-24 00:44:44","198.51.100.84","tcp",6379,"198-51-100-84.example.net","redis","3.0.2",51862,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"4795df119e2d77fe","standalone","Linux 3.13.0-91-generic x86_64",,"epoll","4.7.2",1,"c120481a551c232b8e1a9cff20d9e0968a402dd9",1040551,7, -"2016-07-24 00:44:44","198.51.100.23","tcp",6379,"198-51-100-23.example.net","redis","3.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"98c227055d7fa7b6","standalone","Linux 3.10.0-327.10.1.el7.x86_64 x86_64",,"epoll","4.8.5",35198,"424b15e04ce09f26299ff19b252a920916d4e4be",8875355,2, -"2016-07-24 00:44:47","198.51.100.160","tcp",6379,"198-51-100-160.example.net","redis","2.8.210",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"83ad777ec89a946b","standalone","Windows",,"winsock_IOCP",,2284,"9bde76afda6f81acfb241ea5ee3a9e878ad53881",742778,2, -"2016-07-24 00:44:47","198.51.100.111","tcp",6379,"198-51-100-98.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e19bb8c3d1c28291","standalone","Linux 3.10.0-327.22.2.el7.x86_64 x86_64",,"epoll","5.3.0",1,"c951371f430c1d94299bfc93759f6940d8bfce78",208557,2, -"2016-07-24 00:44:48","198.51.100.227","tcp",6379,"198-51-100-227.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310681,26496, -"2016-07-24 00:44:54","198.51.100.18","tcp",6379,"198-51-100-18.example.net","redis","2.8.9",8972,"DE","NORDRHEIN-WESTFALEN","COLOGNE",0,0,00000000,0,"52c7b9284559eb20","standalone","Linux 2.6.32-5-amd64 x86_64",,"epoll","4.4.5",31887,"e5b1da35862482c4df8d4fce635ec89a36476a4d",14393072,6, -"2016-07-24 00:44:54","198.51.100.248","tcp",6379,"198-51-100-248.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",310687,26112, -"2016-07-24 00:44:57","198.51.100.228","tcp",6379,"198-51-100-228.example.net","redis","3.0.7",8972,"DE","NORDRHEIN-WESTFALEN","COLOGNE",0,0,00000000,0,"5e03212a543f54f8","standalone","Linux 3.13.0-042stab116.1 x86_64",,"epoll","4.8.4",719,"537e3e824a45414c3199ef20201b4362b752eeb5",1263367,2, -"2016-07-24 00:45:04","198.51.100.227","tcp",6379,"198-51-100-227.example.net","redis","2.8.12",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ff040dde4a39b4ff","standalone","Windows",,"winsock_IOCP","0.0.0",1872,"c78751c65793a9a72f6fb0318efa532eb4fc87de",277953,18,"Chemical" -"2016-07-24 00:45:07","198.51.100.132","tcp",6379,,"redis","3.0.5",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"30405cba8f6c2d55","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",2500,"10b4084b930d5a77e5f09e89cf0b21702027bd60",10028956,695, -"2016-07-24 00:46:10","198.51.100.47","tcp",6379,"198-51-100-185.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"6a943c0b5bf37fa1","standalone","Linux 4.4.0-24-generic x86_64",,"epoll","5.3.1",1023,"de9c9c0da3d971f689bd7366c1edc93a00fd1506",2791106,1, -"2016-07-24 01:23:27","198.51.100.246","tcp",6379,"198-51-100-190.example.net","redis","2.8.19",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"665519ce00ddac9b","standalone","Linux 2.6.32-43-pve x86_64",,"epoll","4.9.2",2310,"94595838457eddb30a60184a9db66212268e6f82",9481199,4, -"2016-07-24 01:23:29","198.51.100.187","tcp",6379,"198-51-100-63.example.net","redis","2.8.19",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"c0359e7aa3798aa2","standalone","Linux 3.10.0-229.7.2.el7.x86_64 x86_64",,"epoll","4.8.3",14050,"e67a19de4bd2dc485b98ca353eb6fdc65e8fed4a",14051444,10, -"2016-07-24 01:23:29","198.51.100.228","tcp",6379,"198-51-100-228.example.net","redis","2.8.4",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-32-generic x86_64",,"epoll","4.8.2",22837,"daf5dba760d3db12716c6dc1d0bfe6d5e7b33749",10916038,8, -"2016-07-24 01:23:43","198.51.100.180","tcp",6379,"198-51-100-180.example.net","redis","3.2.1",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"ed627d97d5dc311e","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"569881874d8d5e1508d584a3fd9dff0ac3515839",1677711,1,"Chemical" -"2016-07-24 01:23:56","198.51.100.5","tcp",6379,"198-51-100-207.example.net","redis","3.0.7",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.4",1011,"864c8d7df1e72c662a4edd77b6df6cd30161af6e",2479015,2,"Information Technology" -"2016-07-24 01:24:03","198.51.100.226","tcp",6379,"198-51-100-226.example.net","redis","3.0.5",8972,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"b33bc3e2f8ad13f6","standalone","Linux 2.6.32-573.12.1.el6.x86_64 x86_64",,"epoll","4.4.7",1801,"7f4bb7ed008cdbd665672e88d57fc55616b6dbf2",13189200,9, -"2016-07-24 01:24:14","198.51.100.253","tcp",6379,"198-51-100-136.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.19.0-39-generic x86_64",,"epoll","4.8.2",28272,"13a889aa846c6302dc8f5453e35e051a6f359e9a",14046610,185, -"2016-07-24 01:24:28","198.51.100.206","tcp",6379,,"redis","3.0.6",13301,"DE","NORDRHEIN-WESTFALEN","DUSSELDORF",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313061,26695, -"2016-07-24 01:24:35","198.51.100.73","tcp",6379,,"redis","3.0.2",28753,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"6be7fc9e6b88f79","standalone","Linux 2.6.32-71.29.1.el6.x86_64 x86_64",,"epoll","4.4.7",811,"bbd4dce247ab51d029a64243810aeb900e00d1d6",10082205,15, -"2016-07-24 01:24:35","198.51.100.83","tcp",6379,"198-51-100-174.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"7e7b61a55b95e8e7","standalone","Linux 4.2.0-41-generic x86_64",,"epoll","4.8.4",1076,"48f5f780ca53553fc4c0bbdbb32a5cb06a0551cd",814255,88,"Information Technology" -"2016-07-24 01:25:30","198.51.100.182","tcp",6379,,"redis","3.0.7",31400,"DE","RHEINLAND-PFALZ","FREINSHEIM",0,0,00000000,0,"d9ceac045f7983a9","standalone","FreeBSD 10.1-RELEASE-p26 amd64",,"kqueue","4.2.1",957,"48f37d15b3f5169f11aa5d7194fdfccc7f8df20b",6364747,1, -"2016-07-24 01:25:30","198.51.100.211","tcp",6379,"198-51-100-118.example.net","redis","2.8.17",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e4968abcd4b78b2e","standalone","Linux 3.13.0-36-generic x86_64",,"epoll","4.8.2",1643,"665565b1b1fb6e773039707a0f680bbc417186be",20180649,4,"Information Technology" -"2016-07-24 01:25:35","198.51.100.249","tcp",6379,,"redis","3.0.2",28753,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"6be7fc9e6b88f79","standalone","Linux 2.6.32-71.29.1.el6.x86_64 x86_64",,"epoll","4.4.7",811,"bbd4dce247ab51d029a64243810aeb900e00d1d6",10082265,15, -"2016-07-24 01:25:40","198.51.100.55","tcp",6379,,"redis","3.2.1",3320,"DE","NORDRHEIN-WESTFALEN","SOLINGEN",518210,737415,00000000,0,"e19bb8c3d1c28291","standalone","Linux 4.4.0-24-generic x86_64",,"epoll","5.3.0",1,"49687ba2a5be7f7b6cdf0c837e06307442f6a369",494739,1, -"2016-07-24 01:25:42","198.51.100.62","tcp",6379,"198-51-100-62.example.net","redis","3.0.7",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"2b87841ee28adfc3","standalone","Linux 3.13.0-042stab113.11 x86_64",,"epoll","4.8.4",525,"4045d68fd2e59a1135bb303206d7cd0439ba7ffd",6971251,4, -"2016-07-24 01:25:55","198.51.100.127","tcp",6379,"198-51-100-25.example.net","redis","2.8.4",20473,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-87-generic x86_64",,"epoll","4.8.2",11492,"3de3e977405eef9392a77db4a50d99a5caa2f2d9",2194103,3,"Information Technology" -"2016-07-24 01:26:08","198.51.100.92","tcp",6379,"198-51-100-92.example.net","redis","2.8.10",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5fce0c4aab65e01","standalone","Linux 2.6.32-042stab113.11 x86_64",,"epoll","4.6.3",490,"15abe68a10b011972f50d0abb3bb18f1735994a5",7505621,4, -"2016-07-24 01:26:17","198.51.100.218","tcp",6379,,"redis","3.0.7",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"dc142e699f115c40","standalone","Linux 3.2.60-grsec-x86_64 x86_64",,"epoll","4.7.3",8006,"53a093bd4d0a7b72b2d084ec3767d23b18b8b947",4024979,7, -"2016-07-24 01:26:29","198.51.100.168","tcp",6379,"198-51-100-168.example.net","redis","3.0.6",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.13.0-37-generic x86_64",,"epoll","4.8.4",1279,"8218bd77a0dcb0e00bd77dbb9478115757c70ba5",2405965,1, -"2016-07-24 01:26:29","198.51.100.155","tcp",6379,"198-51-100-155.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"d9155128f7b25ea0","standalone","Linux 3.19.0-25-generic x86_64",,"epoll","4.8.4",27030,"0ede623cb268643672abc04d0267f684a5ee7a0d",6880190,5,"Information Technology" -"2016-07-24 01:26:34","198.51.100.185","tcp",6379,,"redis","2.8.4",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-43-generic x86_64",,"epoll","4.8.2",1196,"ae80fcbb54017f521212caf257418885cd6836a0",5412584,5, -"2016-07-24 01:26:34","198.51.100.1","tcp",6379,"198-51-100-1.example.net","redis","3.2.0",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"5382f69a4e75566b","standalone","Linux 4.2.0-19-generic x86_64",,"epoll","4.9.2",1,"ff8990f109ff5b2d4e0eee47e5ebc66acc43f9e3",4615889,4,"Chemical" -"2016-07-24 01:26:39","198.51.100.51","tcp",6379,"198-51-100-164.example.net","redis","3.0.0",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"9526f4809583faaa","standalone","Linux 2.6.32-042stab113.21 x86_64",,"epoll","4.4.5",14528,"d7271feff55175f434ace92d199f332ad35776a9",7440370,16, -"2016-07-24 01:26:44","198.51.100.138","tcp",6379,,"redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313197,26452, -"2016-07-24 01:26:47","198.51.100.16","tcp",6379,,"redis","2.8.17",25074,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"5937320cdd80c1e4","standalone","Linux 2.6.32-43-pve x86_64",,"epoll","4.9.2",266,"e1d403f2daff849a64b178f74c672db6712f217a",351253,1, -"2016-07-24 01:26:54","198.51.100.171","tcp",6379,"198-51-100-171.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313207,26601, -"2016-07-24 01:27:14","198.51.100.89","tcp",6379,"198-51-100-89.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313227,26358, -"2016-07-24 01:27:24","198.51.100.65","tcp",6379,,"redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"869e89100d5ea8c2","standalone","Linux 3.13.0-85-generic x86_64",,"epoll","4.8.4",21575,"3ec40168300e14f5776d82a48ba873a3999caec1",1897530,1, -"2016-07-24 01:27:24","198.51.100.248","tcp",6379,"198-51-100-248.example.net","redis","3.0.6",12586,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313237,25902, -"2016-07-24 01:27:33","198.51.100.17","tcp",6379,,"redis","2.8.17",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"43dd9e14444e6aea","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",556,"3e8fc2878511cc72f79b765fca86cefe21346912",2607965,72, -"2016-07-24 01:27:33","198.51.100.134","tcp",6379,"198-51-100-134.example.net","redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"6f8b503a2787e3a6","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",1,"b85b2419cf35dd81ff5b9ba6e8bf802cf1d439f6",128621,33, -"2016-07-24 01:27:42","198.51.100.186","tcp",6379,"198-51-100-186.example.net","redis","2.8.13",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"d588bf57ea0dfa69","standalone","Linux 4.4.8-jb1 i686",,"epoll","4.6.3",2460,"97b8d49e62d340d94a38c96c5104abfcacbfa4cb",181557,1, -"2016-07-24 01:27:42","198.51.100.21","tcp",6379,"198-51-100-21.example.net","redis","2.8.19",34011,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"920d7eda78149e99","standalone","Linux 4.4.8-x86_64-jb1 x86_64",,"epoll","4.7.2",3722,"74dfd8a7d87cbb9ecc590ceafd438c85d5073903",183984,1, -"2016-07-24 01:27:43","198.51.100.128","tcp",6379,"198-51-100-203.example.net","redis","3.0.5",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"f3bd5bc2b8b4c486","standalone","Linux 2.6.32-573.8.1.el6.x86_64 x86_64",,"epoll","4.4.7",1968,"0d92b1323fea791ba4b0a43435a156b6ec0aac1c",2967611,2,"Information Technology" -"2016-07-24 01:27:44","198.51.100.216","tcp",6379,"198-51-100-229.example.net","redis","2.8.4",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.16.0-30-generic x86_64",,"epoll","4.8.2",1470,"e76cd0cf25eec5d254c880965189ae011a119220",302420,1, -"2016-07-24 01:27:53","198.51.100.242","tcp",6379,"198-51-100-242.example.net","redis","3.0.2",20773,"DE","NORDRHEIN-WESTFALEN","WEEZE",0,0,00000000,0,"6a04b5ede30cd4cd","standalone","Linux 3.13.0-32-generic x86_64",,"epoll","4.8.4",29725,"1b7e8dc53dec8fb29a8a2d76f516fd3dcb8df652",5815739,7, -"2016-07-24 01:27:53","198.51.100.54","tcp",6379,"198-51-100-54.example.net","redis","2.8.4",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-91-generic x86_64",,"epoll","4.8.2",2903,"0e02514dec6031018eb148b13a4a9639cab3e8aa",905886,1, -"2016-07-24 01:27:54","198.51.100.225","tcp",6379,"198-51-100-225.example.net","redis","3.0.6",12586,"DE","BERLIN","BERLIN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 3.18.24-sirzion x86_64",,"epoll","4.8.4",343519,"53d63f23511dc0080b49aaa8e8203d65619f1c8c",313267,25281, -"2016-07-24 01:27:57","198.51.100.38","tcp",6379,"198-51-100-38.example.net","redis","3.0.5",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"3b863f97501297e9","standalone","Linux 3.13.0-042stab111.12 x86_64",,"epoll","4.8.4",2088,"31a8cececad2e4a33310a741143d85cdef3479b4",11906868,10, -"2016-07-24 01:27:58","198.51.100.22","tcp",6379,"198-51-100-22.example.net","redis","2.8.9",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"2ac6afaedfd3ea15","standalone","Linux 3.13.0-86-generic x86_64",,"epoll","4.8.4",9082,"8e5d9d74c86a9f148a7012733eb52a21938c3c04",5833880,5, -"2016-07-24 01:28:05","198.51.100.106","tcp",6379,"198-51-100-106.example.net","redis","2.8.19",36351,"DE","HESSEN","FRANKFURT AM MAIN",0,0,00000000,0,"9968db13395be4aa","standalone","Windows",,"winsock_IOCP","0.0.0",4372,"89716352a10cd53b5c10e6d5e6cd1d46f5f53a30",485031,4,"Information Technology" -"2016-07-24 01:28:06","198.51.100.130","tcp",6379,"198-51-100-130.example.net","redis","2.8.3",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"542faa6f897d2236","standalone","Linux 2.6.32-573.3.1.el6.x86_64 x86_64",,"epoll","4.4.7",25531,"9d7606a883f764e744d766b7bf0036ba61f7fb6e",496133,5, -"2016-07-24 01:28:08","198.51.100.37","tcp",6379,"198-51-100-37.example.net","redis","2.8.23",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"50630e46be5feb4f","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.9.2",1,"62d16be721c3c62d6c4d080a9bdbe9502c57ca86",3481683,9,"Communications" -"2016-07-24 01:28:32","198.51.100.148","tcp",6379,"198-51-100-148.example.net","redis","3.0.5",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"83dc15dcf8ee3eb8","standalone","Linux 4.1.7-15.23.amzn1.x86_64 x86_64",,"epoll","4.8.3",2304,"883accf76dc364c60902b4eab7861dd1a7eac71d",10981957,10,"Communications" -"2016-07-24 01:28:49","198.51.100.247","tcp",6379,"198-51-100-247.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"3e971e94fbe2eaa6","standalone","Linux 3.2.0-4-amd64 x86_64",,"epoll","4.7.2",2535,"d223aab0621cdd2e4ab752978ad3009ad3814d8b",7715188,57, -"2016-07-24 02:08:46","198.51.100.220","tcp",6379,"198-51-100-220.example.net","redis","3.0.6",51167,"DE","BAYERN","MUNICH",0,0,00000000,0,"1f8e4c92f1ca309","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.8.4",3355,"dd517756bb6ee81e1929fa605972318b2baebb93",5211978,10, -"2016-07-24 02:08:46","198.51.100.239","tcp",6379,"198-51-100-239.example.net","redis","2.8.23",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"83a5616190c5a1aa","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",711,"4117960b13fa313b823c79b0e9f188d8ec6aa3ac",10156283,6, -"2016-07-24 02:08:50","198.51.100.233","tcp",6379,,"redis","2.8.11",8560,"DE","BADEN-WURTTEMBERG","KARLSRUHE",0,0,00000000,0,"f26bfdf4b8265fc","standalone","Linux 3.8.0-29-generic x86_64",,"epoll","4.6.3",14551,"0d001175cf26cee88486d814b4f0c972a5aa89b9",21043417,9, -"2016-07-24 02:08:51","198.51.100.208","tcp",6379,"198-51-100-181.example.net","redis","3.0.6",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"e41bf84a0cecf09d","standalone","Linux 4.2.0-38-generic x86_64",,"epoll","4.8.4",809,"14c5ec7f9669e42ea45a40ff26a6501d593695c0",2405839,19, -"2016-07-24 02:08:51","198.51.100.60","tcp",6379,"198-51-100-60.example.net","redis","3.0.7",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"4ed99bd9c45dfc14","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",1144,"9e28c29ff40017e2fbe32fb97755caf801f95793",843538,2, -"2016-07-24 02:08:51","198.51.100.107","tcp",6379,"198-51-100-39.example.net","redis","3.2.0",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"82b2619163aabc80","standalone","Linux 4.2.0-25-generic x86_64",,"epoll","4.9.2",1,"98f6640bbde04b1214730937212e1fd4e58d03a8",2195657,12, -"2016-07-24 02:08:54","198.51.100.31","tcp",6379,,"redis","2.8.4",6724,"DE","BERLIN","BERLIN",0,0,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-042stab111.12 x86_64",,"epoll","4.8.2",1112,"9c4e55b5ebd06045c5d89d43fa202e219ec8b42c",8839783,7, -"2016-07-24 02:08:56","198.51.100.221","tcp",6379,,"redis","3.0.7",44066,"DE","BAYERN","MUNICH",0,0,00000000,0,"49f951dce0725d71","standalone","FreeBSD 10.0-RELEASE-p7 amd64",,"kqueue","4.2.1",932,"28c6af3c4dedcd9b71cf51a7ebc4e84899196aee",8000949,1, -"2016-07-24 02:09:01","198.51.100.155","tcp",6379,"198-51-100-155.example.net","redis","2.8.22",201229,"DE","HESSEN","FRANKFURT AM MAIN",541512,737999,00000000,0,"fcdf45e47686c89b","standalone","Linux 3.13.0-57-generic x86_64",,"epoll","4.8.4",7,"946ec6b96fe9925d2b677ce02b6c56097c5e69a8",8449694,6, -"2016-07-24 02:09:02","198.51.100.219","tcp",6379,"198-51-100-219.example.net","redis","2.8.4",16509,"DE","HESSEN","FRANKFURT AM MAIN",454113,596101,00000000,0,"a44a05d76f06a5d9","standalone","Linux 3.13.0-74-generic x86_64",,"epoll","4.8.2",1047,"9b83d6a6e7a6ffe50e75dac88cdc5e06f6203c9c",966148,1,"Chemical" -"2016-07-24 02:09:02","198.51.100.193","tcp",6379,"198-51-100-193.example.net","redis","3.0.7",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"fd640d8ef55a22dd","standalone","Linux 4.2.0-42-generic x86_64",,"epoll","4.8.4",1397,"ed5ec17d78d089af53afd4abc339f7decf4641d4",651175,2,"Information Technology" -"2016-07-24 02:09:20","198.51.100.120","tcp",6379,"198-51-100-120.example.net","redis","3.2.1",24940,"DE","BAYERN","GUNZENHAUSEN",0,0,00000000,0,"ed627d97d5dc311e","standalone","Linux 3.16.0-4-amd64 x86_64",,"epoll","4.9.2",1,"f524508ad29334eee2fcf7bdda5c80b9f99d3dfe",987580,167, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_redis.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv deleted file mode 100644 index a61e4573e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","module","motd","has_password" -"2010-02-10 00:00:00",192.168.0.1,tcp,873,node01.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N -"2010-02-10 00:00:01",192.168.0.2,tcp,873,node02.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N -"2010-02-10 00:00:02",192.168.0.3,tcp,873,node03.example.com,rsync,64512,ZZ,Region,City,0,0,"system|Backup system;system_full|Backup full system;mysql|Backup virtual mysql;netadmin|Backup virtual netadmin;",,N diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_rsync.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv deleted file mode 100644 index ee0a625e5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","sip","sip_code","sip_reason","user_agent","sip_via","sip_to","sip_from","content_length","content_type","server","contact","cseq","call_id","allow","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,5060,node01.example.com,sip,64512,ZZ,Region,City,SIP/2.0,489,"Event Package Not Supported",,,,,0,,,,,,"INVITE,ACK,BYE,CANCEL,REGISTER",15.57,109 -"2010-02-10 00:00:01",192.168.0.2,udp,5060,node02.example.com,sip,64512,ZZ,Region,City,SIP/2.0,400,"Bad Request",,,,,364,text/plain,,,,,,62.57,438 -"2010-02-10 00:00:02",192.168.0.3,udp,5060,node03.example.com,sip,64512,ZZ,Region,City,SIP/2.0,400,"Bad Request",,,,,0,,,,,,,6.57,46 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license deleted file mode 100644 index 9f58c89ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_sip.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv deleted file mode 100644 index 256dd78f6..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","version","function","function_text","flags","next_extension_offset","xid","language_tag_length","language_tag","error_code","error_code_text","response_size","raw_response" -"2010-02-10 00:00:00",192.168.0.1,tcp,427,node01.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== -"2010-02-10 00:00:01",192.168.0.2,tcp,427,node02.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== -"2010-02-10 00:00:02",192.168.0.3,tcp,427,node03.example.com,slp,64512,ZZ,Region,City,0,0,,2,2,"Service reply",0x0000,0,5,2,en,5,"Unsupported SLP SPI",40,MDIwMjAwMDAxNDAwMDAwMDAwMDAwMDA1MDAwMjY1NmUwMDA1MDAwMA== diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license deleted file mode 100644 index 9f58c89ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_slp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv deleted file mode 100644 index fc7fe2fff..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","smb_implant","arch","key","smbv1_support","smb_major_number","smb_minor_number","smb_revision","smb_version_string" -"2010-02-10 00:00:00",192.168.0.1,445,node01.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" -"2010-02-10 00:00:01",192.168.0.2,445,node02.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" -"2010-02-10 00:00:02",192.168.0.3,445,node03.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smb.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv deleted file mode 100644 index 19eb56053..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" -"2021-07-08 11:58:42","1.2.3.4","tcp",25,"smtp-server.invalid","smtp;21nails",12345,"EE","HARJUMAA","TALLINN",,,"220 smtp-server.invalid ESMTP Exim 4.80 Wed, 11 Jun 2021 10:00:00 +0300|" -"2021-07-08 11:58:44","5.6.7.8","tcp",25,"smtp-out.invalid","smtp;21nails",23456,"EE","HARJUMAA","TALLINN",,,"220 smtp-out.invalid, ESMTP EXIM 4.86_2|" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license deleted file mode 100644 index c1900637f..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_smtp_vulnerable.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2021 Mikk Margus Möll -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv deleted file mode 100644 index f489261c4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","sysdesc","sysname","asn","geo","region","city","version","naics","sic","sector","device_vendor","device_type","device_model","device_version","device_sector","tag","community","response_size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,161,node01.example.com,"Linux localhost 3.18.20 #1 SMP Mon Jul 9 14:11:21 CST 2018 armv7l",,64512,ZZ,Region,City,2,0,0,,,,,,,snmp,public,165,1.94 -"2010-02-10 00:00:01",192.168.0.2,udp,161,node02.example.com,"RouterOS CCR1009-8G-1S-1S+",,64512,ZZ,Region,City,2,0,0,,MikroTik,router,,,consumer,"snmp,iot",public,115,1.35 -"2010-02-10 00:00:02",192.168.0.3,udp,161,node03.example.com,,,64512,ZZ,Region,City,2,0,0,,,,,,,snmp,public,85,1.00 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_snmp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv deleted file mode 100644 index c591a5c09..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector" -"2010-02-10 00:00:00",192.168.0.1,tcp,1080,node01.example.com,socks4,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:01",192.168.0.2,tcp,1080,node02.example.com,socks5,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service" -"2010-02-10 00:00:02",192.168.0.3,tcp,1080,node03.example.com,socks4,64512,ZZ,Region,City,0,0,"Retail Trade" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_socks.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv deleted file mode 100644 index 460be32c5..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","header","asn","geo","region","city","systime","cache_control","location","server","search_target","unique_service_name","host","nts","nt","content_type","naics","sic","sector","server_port","instance","version","updated_at","resource_identifier","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,60194,node01.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,"Sun, 21 Aug 2022 09:51:13 GMT",max-age=100,http://192.168.200.254:49152/description.xml,"Linux/2.6.26, UPnP/1.0, Portable SDK for UPnP devices/1.3.1",upnp:rootdevice,uuid:28802880-2880-1880-a880-001bc502f600::upnp:rootdevice,node01.example.com,,,,0,0,Government,,,,,,3.35,325 -"2010-02-10 00:00:01",192.168.0.2,udp,38732,node02.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,,"max-age = 1800",http://95.160.216.14:52235/dmr/SamsungMRDesc.xml,"Linux/9.0 UPnP/1.0 PROTOTYPE/1.0",upnp:rootdevice,uuid:f144ca92-6816-94b5-b95f-b58180834044::upnp:rootdevice,node02.example.com,,,,0,0,,,,,,,2.71,263 -"2010-02-10 00:00:02",192.168.0.3,udp,57626,node03.example.com,ssdp,"HTTP/1.1 200 OK",64512,ZZ,Region,City,"Sun, 03 Jan 2016 21:37:50 GMT",max-age=1800,http://192.168.1.3:8008/ssdp/device-desc.xml,"Linux/3.10.79, UPnP/1.0, Portable SDK for UPnP devices/1.6.18",upnp:rootdevice,uuid:62fa0fc8-079d-d00f-2e22-59b49fb488f9::upnp:rootdevice,node03.example.com,,,,0,0,Government,,,,,,4.79,465 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssdp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv deleted file mode 100644 index 837adbad1..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","serverid_raw","serverid_version","serverid_software","serverid_comment","server_cookie","available_kex","available_ciphers","available_mac","available_compression","selected_kex","algorithm","selected_cipher","selected_mac","selected_compression","server_signature_value","server_signature_raw","server_host_key","server_host_key_sha256","rsa_prime","rsa_prime_length","rsa_generator","rsa_generator_length","rsa_public_key","rsa_public_key_length","rsa_exponent","rsa_modulus","rsa_length","dss_prime","dss_prime_length","dss_generator","dss_generator_length","dss_public_key","dss_public_key_length","dss_dsa_public_g","dss_dsa_public_p","dss_dsa_public_q","dss_dsa_public_y","ecdsa_curve25519","ecdsa_curve","ecdsa_public_key_length","ecdsa_public_key_b","ecdsa_public_key_gx","ecdsa_public_key_gy","ecdsa_public_key_n","ecdsa_public_key_p","ecdsa_public_key_x","ecdsa_public_key_y","ed25519_curve25519","ed25519_cert_public_key_nonce","ed25519_cert_public_key_bytes","ed25519_cert_public_key_raw","ed25519_cert_public_key_sha256","ed25519_cert_public_key_serial","ed25519_cert_public_key_type_id","ed25519_cert_public_key_type_name","ed25519_cert_public_key_keyid","ed25519_cert_public_key_principles","ed25519_cert_public_key_valid_after","ed25519_cert_public_key_valid_before","ed25519_cert_public_key_duration","ed25519_cert_public_key_sigkey_bytes","ed25519_cert_public_key_sigkey_raw","ed25519_cert_public_key_sigkey_sha256","ed25519_cert_public_key_sigkey_value","ed25519_cert_public_key_sig_raw","banner","userauth_methods","device_vendor","device_type","device_model","device_version","device_sector" -"2022-01-10 02:20:37","18.179.0.0","tcp",22,"ec2-18-179-0-0.ap-northeast-1.compute.amazonaws.com","ssh",16509,"JP","TOKYO","TOKYO",454110,,"SSH-2.0-OpenSSH_7.4","2.0","OpenSSH_7.4",,"bGjsifbPIDWT7tAu8BMjyg==","curve25519-sha256, curve25519-sha256@libssh.org, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, diffie-hellman-group-exchange-sha256, diffie-hellman-group16-sha512, diffie-hellman-group18-sha512, diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha256, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1","chacha20-poly1305@openssh.com, aes128-ctr, aes192-ctr, aes256-ctr, aes128-gcm@openssh.com, aes256-gcm@openssh.com, aes128-cbc, aes192-cbc, aes256-cbc, blowfish-cbc, cast128-cbc, 3des-cbc","umac-64-etm@openssh.com, umac-128-etm@openssh.com, hmac-sha2-256-etm@openssh.com, hmac-sha2-512-etm@openssh.com, hmac-sha1-etm@openssh.com, umac-64@openssh.com, umac-128@openssh.com, hmac-sha2-256, hmac-sha2-512, hmac-sha1","none, zlib@openssh.com","curve25519-sha256@libssh.org","ecdsa-sha2-nistp256","aes128-ctr","hmac-sha2-256","none","AAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=","AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAABKAAAAIQCd+X/B/OEx+FrwJSlVecOvNMuS5w2vTRz0z4prM+5VBwAAACEArU60b9CHs/d5BgyaOd7vmFygTMK5SyL90bS8VIztX/4=","AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDSEHaLacthwB30rtA4xJgN3G9zXkCmm2WhV/TlNBrD20fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=","a6e4e1c16ba25d51bcddc58a6e16797144575dd18d02d9dedf75093d2b15c557",,,,,,,,,,,,,,,,,,,,"1xx7ASut7BF4ED8b592bebZBMBKTCzOsmbH4cjwx/0U=","P-256",256,"WsY12Ko6k+ez671VdpiGvGUdBrDMU7D2O848PifSYEs=","axfR8uEsQkf4vOblY6RA8ncDfYEt6zOg9KE5RdiYwpY=","T+NC4v4af5uO5+tKfA+eFivOM1drMV7Oy7ZAaDe/UfU=","/////wAAAAD//////////7zm+q2nF56E87nKwvxjJVE=","/////wAAAAEAAAAAAAAAAAAAAAD///////////////8=","NIQdotpy2HAHfSu0DjEmA3cb3NeQKabZaFX9OU0GsPY=","0fuNQAZX7XciX2YkqIHtK2dWLBYwVCCqvl//zoM42kI=",,,,,,,,,,,,,,,,,,,,"publickey",,,,, -"2022-01-10 02:20:37","170.10.0.0","tcp",22,"170-10-0-0.example.com","ssh",11976,"US","TEXAS","MARSHALL",,,"SSH-2.0-ARRIS_0.50","2.0","ARRIS_0.50",,"Y4RQS9sdRgEFwNJKVP6bZg==","diffie-hellman-group1-sha1","aes128-cbc, 3des-cbc, aes256-cbc, twofish256-cbc, twofish-cbc, twofish128-cbc, blowfish-cbc","hmac-sha1-96, hmac-sha1, hmac-md5","none","diffie-hellman-group1-sha1","ssh-rsa","aes128-cbc","hmac-sha1","none","LQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==","AAAAB3NzaC1yc2EAAACCLQj+UTJEQqdb/p/c/19yVc63eo+rnedwXKjP6eNNxxijN2cFoOjVMeqT2QTBjyoN7yRWBU2EID+3y2jUYT8mCqmqfyUv1eEbiCfLVlUyQ0X/CY9I5DDb5l6yEjNkuH2xVNNV6R7GFRwyYKAsYzfy+i9o1OORlUh3tozkkPfA9z/NlA==","AAAAB3NzaC1yc2EAAAADAQABAAAAgwCDq1kGqqwdQVryCNcoyDbBpnL/okvM2d9NmR0OjprcToCZ2TZ5WUZt2BGwPE1QLJYskjhv7GwlfQ4qhEqHDg35wMrkO7j9LTQC7KW3xisOLuUil4FmMxPkol6s39945zBGpjw0l/BmJnUDlutxavkdd84fppFMwXNp2vbjxV1SYVc9","d53fedbfe92e631264629882b2e85bfd213ca4b07b824cd31f8de1fcb8d0ddcb",,,,,,,65537,"g6tZBqqsHUFa8gjXKMg2waZy/6JLzNnfTZkdDo6a3E6Amdk2eVlGbdgRsDxNUCyWLJI4b+xsJX0OKoRKhw4N+cDK5Du4/S00Auylt8YrDi7lIpeBZjMT5KJerN/feOcwRqY8NJfwZiZ1A5brcWr5HXfOH6aRTMFzadr248VdUmFXPQ==",1040,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"publickey, password","Arris",,,, -"2022-01-10 02:20:37","72.17.0.0","tcp",22,"072-017-0-0.example.com","ssh",33363,"US","FLORIDA","ORLANDO",517311,,"SSH-1.99-Cisco-1.25","1.99","Cisco-1.25",,"Z2fOfWsrLlh76Y0bOqa1cw==","diffie-hellman-group-exchange-sha1, diffie-hellman-group14-sha1, diffie-hellman-group1-sha1","aes128-cbc, 3des-cbc, aes192-cbc, aes256-cbc","hmac-sha1, hmac-sha1-96, hmac-md5, hmac-md5-96","none","diffie-hellman-group14-sha1","ssh-rsa","aes128-cbc","hmac-sha1","none","lrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=","AAAAB3NzaC1yc2EAAAIAlrzL2DY9fVvwYg6CgB75uf2s8CLo+rL8Mp9tU1Ja3sDfBzj9QJjVDykupiy8s3usHfxMrHS2v3DhTiZjz/b5K6tVTgUBTXL94JfM4lwB+3EbLggPzKnlm1jQgnnU9c+tb7RX3IhBqU9Yj1gqxhErv9NFotgajQOOLgY0Ua5C0Ee+AIaMlLaNZe3LTejMsNUZMN5tl+sEmtutMHkGQsmjJxiJ3feF+Pys0I2+ojiiAfzqlMYar/5xOPl4Dj+HO+h91xVQ1/8nQRBc082fM7+ZJtDbRLtt4G8srlB5gew26jqfVASc/ui5gx4+BR9DG9VH8w+rJWBGfhOAaWqLFE2M3YuEWkjEmQMR1SQK1WFQ/oNiWJO2K5L3rk2LcAmyR6nQMtClVxYZ7CQOwa3uFL+JNXp9AhiiAtVaqhrEK81NJrJNh/+egTBl5STphxIShXd4KI9wyvkGlCIvNIMO94iXPVaWUXXbsGnU03+dsUkBzGf0eJ4DePInCk/RtunlSmOsjGld+rpS9g0VRxPrzbQRWuhpkgpV+CldyrI3C/rOxJRs2vSAKXocRsGwhqEKseAJzHXmiZ5ncsaGKoeB5lUkWLwcKjyok2tHVCDlzDUpE4aA/JHNEhT48det9RqtjC71yz8m0PeK2ySI/I+Qb7eBgevgduBmt+OUxgvfKi2UB6s=","AAAAB3NzaC1yc2EAAAADAQABAAACAQDIVXBwKGhi35gabwHNZi6Bxls1BGtDVVZFhwvhTpJKTKhV4T2HnDFG7+FBpYejc92wH026Wf+uJHIpnKkVQRnnOV98zKXW68Tz+OnwT8aBQdLI+QYDC7wLwGRf+cOiXEAkpMrp2OJme+GwQ97oBccEwdu2j9vcYAFQ0+eCPNfwPrcZhwVb00kt/moLVSxWRdsDMzQiNDZf2zel+FQIAl5cCfaLSAQa1TIXy8SM13B0brnlpdyIqukQS0zUv/PL/6AsfhgLXeQBgjs1XIf6qL+ZdtQss5AKUDuJgrWDcS3nyNZQg/CAt8XdIsLntu3bCn+VGA1O/gUGLS1a9GoGd/lRArlmODNtbds74m7hxaAf/gzg0LFJx6HhwubmVCzTXEHl95KHYHKoDvCtUOgUm7zUugxWjhsLPfT6UfZCwvCY21SGVYsoEPiTT2DhuAFriM+PT83JresFHgZDosbqW0VCi2bzAKSBu/vphaqTbSdDo0xhkW9JCb3zUkW2ge/e/GrjxV4cNXRC9XQ/XYEIWmtF/gHSi0i9KweX4sN5TEkB/41vDvyDOdyPJ8Jta0I9vBolDwJ6qdMHOPlOW5oW83yCgbmUJNYkZ+MivABlc6iS/006qYiIwknHezbY5foYd8kDON7YAssOwCJcG5viII50Z1N9VsGkUv5sZMr2p9ry8Q==","06ff3cce443ed832927576d982b69d5a526d0e63334c72e87201deda61679406",,,,,,,65537,"yFVwcChoYt+YGm8BzWYugcZbNQRrQ1VWRYcL4U6SSkyoVeE9h5wxRu/hQaWHo3PdsB9Nuln/riRyKZypFUEZ5zlffMyl1uvE8/jp8E/GgUHSyPkGAwu8C8BkX/nDolxAJKTK6djiZnvhsEPe6AXHBMHbto/b3GABUNPngjzX8D63GYcFW9NJLf5qC1UsVkXbAzM0IjQ2X9s3pfhUCAJeXAn2i0gEGtUyF8vEjNdwdG655aXciKrpEEtM1L/zy/+gLH4YC13kAYI7NVyH+qi/mXbULLOQClA7iYK1g3Et58jWUIPwgLfF3SLC57bt2wp/lRgNTv4FBi0tWvRqBnf5UQK5ZjgzbW3bO+Ju4cWgH/4M4NCxSceh4cLm5lQs01xB5feSh2ByqA7wrVDoFJu81LoMVo4bCz30+lH2QsLwmNtUhlWLKBD4k09g4bgBa4jPj0/Nya3rBR4GQ6LG6ltFQotm8wCkgbv76YWqk20nQ6NMYZFvSQm981JFtoHv3vxq48VeHDV0QvV0P12BCFprRf4B0otIvSsHl+LDeUxJAf+Nbw78gzncjyfCbWtCPbwaJQ8CeqnTBzj5TluaFvN8goG5lCTWJGfjIrwAZXOokv9NOqmIiMJJx3s22OX6GHfJAzje2ALLDsAiXBub4iCOdGdTfVbBpFL+bGTK9qfa8vE=",4096,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"publickey, keyboard-interactive, password","Cisco",,,,"enterprise" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssh.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv deleted file mode 100644 index 0b125001b..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","ssl_poodle","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","freak_vulnerable","freak_cipher_suite","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","tlsv13_support","tlsv13_cipher","jarm" -"2022-01-10 00:01:42","96.60.0.0",10443,"96-60-0-0.example.com","ssl,vpn","TLSv1.2",4181,"US","WISCONSIN","MILWAUKEE","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","N",1024,"FGT60D4614030700","support","2014-06-23 09:56:32","2038-01-19 03:14:07","5A:3D:FF:06:F9:E9:25:37:57:F9:09:52:33:A4:85:15:24:2D:88:7F","168CAE",2,"sha1WithRSAEncryption","rsaEncryption","Fortinet","FortiGate","US","California","Sunnyvale",,,,,"support@fortinet.com",,,"Fortinet","Certificate Authority","US","California","Sunnyvale",,,,,"support@fortinet.com",,,517311,,"N",,,"35:AB:B6:76:2A:3D:17:B2:FB:40:45:1B:FC:0A:99:0A:6E:48:57:F7:30:0A:3B:B1:1A:E6:99:70:5B:7C:32:41","88:7B:16:DB:39:44:0C:47:0E:4A:8F:0B:C5:FB:4D:45:BC:93:5A:00:43:A1:D9:7F:05:1D:86:33:02:F8:FC:57:67:A6:1D:C0:FF:F7:D2:40:D8:9A:21:AE:4E:6D:DC:E7:FF:72:BF:13:CB:EE:A7:5F:CD:83:EA:8A:5E:FB:87:DD","99:45:1F:2E:AE:EB:88:91:27:43:33:79:FA:93:7D:CA","HTTP/1.1",200,"OK","text/html",,,,"xxxxxxxx-xxxxx",131,,"Mon, 10 Jan 2022 00:01:44 GMT","Y","N","N","N","unknown","x509: unknown error",,, -"2022-01-10 00:01:42","113.160.0.0",10443,"","ssl","TLSv1.2",45899,"VN","THAI BINH","THAI BINH","TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","N",2048,"1078-btb-tbi-HungHa-61d39c6d5a7e2","1078-btb-tbi-HungHa-61d39c6d5a7e2","2022-01-04 01:01:34","2023-02-06 01:01:34","A9:00:BB:E1:54:4D:56:54:59:F1:B7:EA:F1:1A:D5:36:5C:63:90:8E","36974C4C6B1B3785",2,"sha256WithRSAEncryption","rsaEncryption","pfSense webConfigurator Self-Signed Certificate",,,,,,,,,,,,"pfSense webConfigurator Self-Signed Certificate",,,,,,,,,,,,517311,,"N",,,"38:85:F0:44:1E:AD:84:B8:2F:43:68:BA:AC:EE:17:13:A4:BF:86:1D:48:75:7E:22:FA:08:4C:28:5F:AC:3E:5F","AE:1B:4F:D1:E4:C0:35:9D:2A:4F:7A:37:B8:7B:11:9D:84:25:23:21:AB:EF:B2:0F:DC:C9:F2:A3:72:28:92:E1:74:72:FA:E1:09:6C:E1:F6:B6:E3:A7:61:1C:58:89:34:D7:06:5C:3D:0A:A7:F6:CC:8A:D6:24:D0:04:4C:03:02","16:93:9A:F4:35:7F:9A:85:45:71:91:C7:7C:80:88:00","HTTP/1.1",200,"OK","text/html; charset=UTF-8","keep-alive",,"PHPSESSID=e15bdfa5739c36877608eb4cf46cc388; path=/; secure; HttpO","nginx",,"chunked","Mon, 10 Jan 2022 00:01:44 GMT","N","Y","N","N","unknown","x509: unknown error",,, -"2022-01-10 00:01:42","34.224.0.0",10443,"","ssl,vpn","TLSv1.2",14618,"US","VIRGINIA","ASHBURN","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","N",2048,"","Entrust Certification Authority - L1K","2021-10-07 15:30:28","2022-11-06 15:30:28","AD:19:B2:1C:CB:88:70:9B:DB:8E:7E:F5:65:50:13:D6:43:6C:BE:6E","7B388364A24B88E77E5553B5C6748100",2,"sha256WithRSAEncryption","rsaEncryption","Ciena Corporation",,"US","Maryland","Hanover",,,,,,,,"Entrust, Inc.","(c) 2012 Entrust, Inc. - for authorized use only","US",,,,,,,,,,454110,,"N",,"Retail Trade","9A:64:73:0B:8A:FA:DE:22:D4:6D:5A:C6:C4:6F:D4:A4:2A:28:FA:41:1E:FF:81:DC:D4:D9:00:FD:78:DF:C4:DD","9A:B7:BD:68:7D:F3:E7:C1:B7:D3:F4:2F:01:B6:C4:77:90:A3:2B:1E:C0:89:F5:08:EC:43:87:35:60:36:D4:87:61:AA:B8:A8:B3:8A:E9:F1:04:AA:5B:67:12:FF:63:D5:14:80:77:6E:8F:7D:C3:E2:3A:F3:13:DF:08:43:6C:B0","E7:34:BC:92:84:FA:39:DE:E1:46:6C:27:DA:5A:01:F4","HTTP/1.1",200,"OK","text/html",,,,"xxxxxxxx-xxxxx",131,,"Mon, 10 Jan 2022 00:01:44 GMT","Y","N","N","Y","OV",,,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv deleted file mode 100644 index ab28456b4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv +++ /dev/null @@ -1,46 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","freak_vulnerable","freak_cipher_suite","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","device_model","device_sector","device_type","device_vendor","device_version","jarm","page_sha256fp","raw_cert","raw_cert_chain","tlsv13_cipher","tlsv13_support" -"2018-04-23 13:25:21","198.51.100.232","443",,"ssl-freak","TLSv1.0","8447","AT","WIEN","VIENNA","TLS_RSA_WITH_RC4_128_SHA","1024","usg50_B0B2DC2FA69D","usg50_B0B2DC2FA69D","2012-05-10 00:01:19","2032-05-05 00:01:19","14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2","4FAB054F","sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,"0","0","Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5",,"57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1","E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87","1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE","HTTP/1.1","200","OK","text/html",,,,,,"chunked","Mon, 23 Apr 2018 13:25:26 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -"2018-04-23 13:25:26","198.51.100.224","443","198-51-100-224.example.net","ssl-freak","TLSv1.0","12577","AT","NIEDEROSTERREICH","BADEN","TLS_RSA_WITH_RC4_128_SHA","1024","usg20w_C86C870287EC","usg20w_C86C870287EC","2010-01-01 00:00:53","2029-12-27 00:00:53","14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2","4B3D3B35","sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,"0","0","Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5",,"57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1","E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87","1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE","HTTP/1.1","200","OK","text/html",,,,,,"chunked","Mon, 23 Apr 2018 13:25:29 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -2018-04-23 13:25:21,198.51.100.232,443,,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC2FA69D,usg50_B0B2DC2FA69D,2012-05-10 00:01:19,2032-05-05 00:01:19,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FAB054F,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:26 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:26,198.51.100.224,443,198-51-100-224.example.net,ssl-freak,TLSv1.0,12577,AT,NIEDEROSTERREICH,BADEN,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_C86C870287EC,usg20w_C86C870287EC,2010-01-01 00:00:53,2029-12-27 00:00:53,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4B3D3B35,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:29 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:33,198.51.100.67,443,,ssl-freak,TLSv1.0,8447,AT,NIEDEROSTERREICH,WAIDHOFEN AN DER THAYA,TLS_RSA_WITH_RC4_128_SHA,1024,Technicolor TG670,Technicolor TG670,2005-01-01 00:00:00,2024-12-31 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-7A2C610E,sha1WithRSAEncryption,rsaEncryption,Technicolor,1112WT0YK,,,,,,,,,,,Technicolor,1112WT0YK,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,,keep-alive,,xAuth_SESSION_ID=bm90aGluZyBoZXJlCg==; path=/;,,0,,"Mon, 23 Apr 2018 14:25:37 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:36,198.51.100.3,443,,ssl-freak,TLSv1.2,8445,AT,SALZBURG,HINTERGLEMM,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,2048,uag2100_04BF6D22A5A9,uag2100_04BF6D22A5A9,2016-03-08 20:27:08,2026-03-06 20:27:08,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B0F07D300BDB4FC4,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:39 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:38,198.51.100.198,443,198-51-100-198.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,2048,198-51-100-198.example.net,Go Daddy Secure Certificate Authority - G2,2016-12-29 08:51:00,2019-12-29 08:51:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,AEA6D3637023B56B,sha256WithRSAEncryption,rsaEncryption,,Domain Control Validated,,,,,,,,,,,198-51-100-198.example.net," Inc.""",http://certs.godaddy.com/repository/,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden ( The server,text/html,close,,,,2024,,,Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-23 13:25:38,198.51.100.98,443,198-51-100-98.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_FCF528743754,usg50_FCF528743754,2013-04-29 00:00:26,2033-04-24 00:00:26,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,517DB81A,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:41,198.51.100.156,443,198-51-100-156.example.net,ssl-freak,TLSv1.0,8339,AT,NIEDEROSTERREICH,SCHWECHAT,TLS_RSA_WITH_AES_128_CBC_SHA,1024,usg200_404A036775FC,usg200_404A036775FC,2010-05-01 00:04:04,2030-04-26 00:04:04,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4BDB6FF4,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:43 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:25:53,198.51.100.200,443,,ssl-freak,TLSv1.2,8447,AT,NIEDEROSTERREICH,KREMS AN DER DONAU,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB67FC6F,usg20_5CF4AB67FC6F,2015-12-02 00:00:47,2035-11-27 00:00:47,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,565E34AF,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:25:56 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:02,198.51.100.83,443,198-51-100-83.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_FCF5286F5972,usg20w_FCF5286F5972,2013-03-23 00:00:43,2033-03-18 00:00:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,514CF0AB,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:05 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:03,198.51.100.155,443,198-51-100-155.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-155.example.net,198-51-100-155.example.net,2018-03-19 19:47:07,2023-03-19 19:47:07,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2DF52AA905C7A2B44C2B9F0012FD5745,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden,text/html,,,,Microsoft-IIS/6.0,1939,,"Mon, 23 Apr 2018 13:11:52 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:03,198.51.100.129,443,198-51-100-129.example.net,ssl-freak,TLSv1.0,29654,AT,SALZBURG,SALZBURG,TLS_RSA_WITH_RC4_128_SHA,1024,localhost,localhost,2007-01-31 19:00:29,2008-01-31 19:00:29,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2,sha1WithRSAEncryption,rsaEncryption,Apache HTTP Server,Test Certificate,,,,,,,,,,,Apache HTTP Server,For testing purposes only,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,318,,"Mon, 23 Apr 2018 17:42:37 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:12,198.51.100.7,443,198-51-100-7.example.net,ssl-freak,TLSv1.0,8445,AT,SALZBURG,ALTENMARKT IM PONGAU,TLS_RSA_WITH_RC4_128_SHA,2048,IMM2-5cf3fcaf3abd,IMM2-5cf3fcaf3abd,2013-03-22 14:32:06,2023-03-20 14:32:06,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,D8C631398B585F10,sha1WithRSAEncryption,rsaEncryption,System X,,US,SomeState,SomeCity,,,,,,,,System X,,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,301,Moved Permanently,application/x-appweb-php,keep-alive,,,Mbedthis-Appweb/2.4.2,0,,"Mon, 23 Apr 2018 13:37:08 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:15,198.51.100.93,443,,ssl-freak,TLSv1.2,8447,AT,KARNTEN,SPITTAL AN DER DRAU,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC3308EF,usg50_B0B2DC3308EF,2012-05-25 00:00:39,2032-05-20 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FBECBA7,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:17 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:16,198.51.100.81,443,198-51-100-81.example.net,ssl-freak,TLSv1.0,5385,AT,VORARLBERG,FELDKIRCH,TLS_RSA_WITH_RC4_128_SHA,1024,usg100_5067F03642A5,usg100_5067F03642A5,2010-10-01 00:04:48,2030-09-26 00:04:48,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4CA525A0,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:19 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:17,198.51.100.162,443,198-51-100-162.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,rc1,Peppercon CA,2003-05-08 16:30:05,2008-05-06 16:30:05,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,18,md5WithRSAEncryption,rsaEncryption,,R&D,DE,SomeState,,,,,,198-51-100-162.example.net,,,,Security Department,DE,SomeState,SomeCity,,,,,198-51-100-162.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Redirect,,,,,,,,,N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:22,198.51.100.57,443,,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,GLEISDORF,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB661192,usg20_5CF4AB661192,2015-09-22 00:00:46,2035-09-17 00:00:46,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,56009A2E,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:25 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:28,198.51.100.146,443,198-51-100-146.example.net,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_RC4_128_SHA,1024,zywall_110_A0E4CB7CE5AF,zywall_110_A0E4CB7CE5AF,2015-01-26 17:19:56,2025-01-23 17:19:56,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,54C6773C,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:31 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:34,198.51.100.233,443,,ssl-freak,TLSv1.0,8447,AT,KARNTEN,KLAGENFURT AM WORTHERSEE,TLS_RSA_WITH_RC4_128_SHA,1024,198.51.100.174,198-51-100-174.example.net,2009-04-14 07:26:09,2025-04-15 07:26:09,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,A0571920C03C9EE0DA1168E586E0E8D440E42EA69D898AC829,sha1WithRSAEncryption,rsaEncryption,,General,DE,SomeState,SomeCity,,,,,198-51-100-174.example.net,,,,General,DE,SomeState,SomeCity,,,,,198-51-100-174.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,Close,,,LANCOM 1781A 8.50.0161 / 09.08.2011,,,,Y,N,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:35,198.51.100.106,443,198-51-100-106.example.net,ssl-freak,TLSv1.0,12793,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-106.example.net,SHT-Gruppe CA,2004-07-20 07:28:10,2006-07-20 07:38:10,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,190DBE75000000000007,sha1WithRSAEncryption,rsaEncryption,,,AT,SomeState,SomeCity,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/6.0,1508,,"Mon, 23 Apr 2018 13:26:37 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:37,198.51.100.191,443,,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,LEBRING,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB669448,usg20_5CF4AB669448,2015-10-01 00:00:38,2035-09-26 00:00:38,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,560C77A6,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:42,198.51.100.235,443,198-51-100-235.example.net,ssl-freak,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_107BEF33651A,usg50_107BEF33651A,2014-04-24 00:00:27,2034-04-19 00:00:27,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,5358541B,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:45 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:43,198.51.100.167,443,198-51-100-167.example.net,ssl-freak,TLSv1.0,8412,AT,BURGENLAND,ELTENDORF,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-167.example.net,198-51-100-167.example.net,2008-08-19 06:57:11,2010-08-19 06:57:11,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2,sha1WithRSAEncryption,rsaEncryption,SuSE Linux Web Server,web server,XY,unknown,unknown,,,,,198-51-100-167.example.net,,,SuSE Linux Web Server,CA,XY,SomeState,unknown,,,,,198-51-100-167.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache/2.2.3 (Linux/SUSE),80,,"Mon, 23 Apr 2018 13:26:45 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:47,198.51.100.42,443,198-51-100-42.example.net,ssl-freak,TLSv1.0,8437,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-42.example.net,iLO Default Issuer (Do not trust),2013-11-05 00:00:00,2028-11-04 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,72FD09EF,sha1WithRSAEncryption,rsaEncryption,,,US,SomeState,SomeCity,,,,,,,,,,US,SomeState,Houston,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,,,,,,,,,N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.177,443,198-51-100-177.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB625772,usg20_5CF4AB625772,2015-03-04 00:00:39,2035-02-27 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,54F64B27,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:50 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.66,443,198-51-100-66.example.net,ssl-freak,TLSv1.0,5385,AT,VORARLBERG,DORNBIRN,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-66.example.net,198-51-100-66.example.net,2009-10-06 11:23:48,2015-03-29 11:23:48,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,98B18BCD61B0CD5D,sha1WithRSAEncryption,rsaEncryption,,??,??,??,??,,,,,??,,,,??,??,??,??,,,,,??,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,close,,DSSignInURL=/; path=/; secure,,,,,Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:48,198.51.100.29,443,198-51-100-29.example.net,ssl-freak,TLSv1.0,6830,AT,NIEDEROSTERREICH,GUNTRAMSDORF,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_FCF52878354B,usg20_FCF52878354B,2013-05-20 00:00:39,2033-05-15 00:00:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,519967A7,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:50 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:49,198.51.100.235,443,,ssl-freak,TLSv1.0,8447,AT,TIROL,KITZBUHEL,TLS_RSA_WITH_RC4_128_SHA,1024,usg50_B0B2DC3AEFE7,usg50_B0B2DC3AEFE7,2012-10-30 00:02:36,2032-10-25 00:02:36,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,508F191C,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:52 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:50,198.51.100.159,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-159.example.net,198-51-100-159.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-159.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-159.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:26:52 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:51,198.51.100.138,443,198-51-100-138.example.net,ssl-freak,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_B0B2DC34A1F6,usg20_B0B2DC34A1F6,2012-06-16 00:00:58,2032-06-11 00:00:58,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FDBCCBA,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:54 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:52,198.51.100.64,443,,ssl-freak,TLSv1.0,1853,AT,OBEROSTERREICH,WILHERING,TLS_RSA_WITH_RC4_128_SHA,1024,198.51.100.171,198.51.100.117,2017-08-10 10:48:40,2020-08-09 10:48:40,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,598C3A08,sha1WithRSAEncryption,rsaEncryption,,,,,SomeCity,,,,,,,,,,,,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,5597,,"Mon, 23 Apr 2018 13:26:54 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:55,198.51.100.189,443,198-51-100-62.example.net,ssl-freak,TLSv1.0,25255,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20w_107BEF3A4C9E,usg20w_107BEF3A4C9E,2014-07-04 00:00:43,2034-06-29 00:00:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,53B5EEAB,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:57 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.17,443,198-51-100-17.example.net,ssl-freak,TLSv1.0,8447,AT,STEIERMARK,SOEDING,TLS_RSA_WITH_AES_256_CBC_SHA,1024,Vimar By-Web,Vimar By-Web,2011-10-27 09:19:55,2016-10-25 09:19:55,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B82B13ED1FB0FD71,sha1WithRSAEncryption,rsaEncryption,,R&D,IT,SomeState,SomeCity,,,,,,,,,R&D,IT,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_DES40_CBC_SHA,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,text/html,keep-alive,,,nginx/0.6.32,,chunked,"Mon, 23 Apr 2018 13:26:56 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.111,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-111.example.net,198-51-100-111.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-111.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-111.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:26:58 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:56,198.51.100.179,443,198-51-100-179.example.net,ssl-freak,TLSv1.0,12605,AT,OBEROSTERREICH,LINZ,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB665FB9,usg20_5CF4AB665FB9,2015-09-25 00:00:42,2035-09-20 00:00:42,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,56048EAA,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:26:58 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:58,198.51.100.143,443,,ssl-freak,TLSv1.0,8447,AT,KARNTEN,GLAN,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_FCF5285DEDC4,usg20_FCF5285DEDC4,2012-11-09 00:00:44,2032-11-04 00:00:44,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,509C47AC,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:00 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:58,198.51.100.111,443,198-51-100-111.example.net,ssl-freak,TLSv1.0,1901,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,*.*,198-51-100-111.example.net,2009-01-16 12:51:43,2010-01-16 12:51:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6,md5WithRSAEncryption,rsaEncryption,,,IL,SomeState,,,,,,,,,,Visonic CA,IL,SomeState,,,,,,198-51-100-111.example.net,,,518210,737415,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html,close,,PowerLink=226002836046b4bddcd2d16b809f76d9; path=/,Apache/1.3.31 (Unix) PHP/4.3.9 mod_ssl/2.8.20 Open,,chunked,"Wed, 23 Jan 2002 10:17:09 GMT",N,N,Y,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:26:59,198.51.100.79,443,,ssl-freak,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,usg20_5CF4AB65A17C,usg20_5CF4AB65A17C,2015-09-01 00:00:51,2035-08-27 00:00:51,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,55E4EAB3,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:02 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:26:59,198.51.100.90,443,,ssl-freak,TLSv1.0,8218,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-90.example.net,198-51-100-90.example.net,2002-01-09 20:22:25,2003-01-09 20:22:25,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,md5WithRSAEncryption,rsaEncryption,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-90.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-90.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,,Apache/1.3.33 (Unix) (Red-Hat/Linux) FrontPage/4.,31,,"Mon, 23 Apr 2018 13:27:02 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.186,443,198-51-100-186.example.net,ssl-freak,TLSv1.0,31125,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-186.example.net,198-51-100-186.example.net,2013-07-11 12:20:19,2021-07-09 12:20:19,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,D947ED19BEAB28E6,sha1WithRSAEncryption,rsaEncryption,,IT,PL,SomeState,SomeCity,,,,,198-51-100-186.example.net,,,,IT,PL,SomeState,SomeCity,,,,,198-51-100-186.example.net,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,401,Unauthorized,text/plain,close,"Basic realm=""example.com""",,Microsoft-IIS/7.5,0,,"Mon, 23 Apr 2018 14:03:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.150,443,198-51-100-150.example.net,ssl-freak,TLSv1.0,8559,AT,BURGENLAND,NEUSIEDL AM SEE,TLS_ECDHE_RSA_WITH_RC4_128_SHA,2048,198-51-100-150.example.net,COMODO RSA Domain Validation Secure Server CA,2017-02-08 00:00:00,2019-05-09 23:59:59,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B6EF6CF436532F0252627393BD7311FD,sha256WithRSAEncryption,rsaEncryption,,Domain Control Validated,,,,,,,,,,,,,GB,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Mon, 23 Apr 2018 13:27:06 GMT",N,N,N,N,DV,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:27:03,198.51.100.141,443,198-51-100-141.example.net,ssl-freak,TLSv1.0,39372,AT,OBEROSTERREICH,HINTERSTODER,TLS_RSA_WITH_RC4_128_SHA,1024,198-51-100-141.example.net,iLO Default Issuer (Do not trust),2014-01-14 00:00:00,2029-01-13 00:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7852761B,sha1WithRSAEncryption,rsaEncryption,,,US,SomeState,SomeCity,,,,,,,,,,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,,,,,,,,,N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-23 13:27:04,198.51.100.194,443,198-51-100-194.example.net,ssl-freak,TLSv1.0,8447,AT,KARNTEN,GLAN,TLS_RSA_WITH_RC4_128_SHA,1024,iDRAC6 default certificate,iDRAC6 default certificate,2009-09-17 22:47:28,2019-09-15 22:47:28,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,1,sha1WithRSAEncryption,rsaEncryption,,Remote Access Group,US,SomeState,SomeCity,,,,,,,,,Remote Access Group,US,SomeState,SomeCity,,,,,,,,0,0,Y,TLS_RSA_EXPORT_WITH_RC4_40_MD5,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Moved Temporarily,,keep-alive,,,Mbedthis-Appweb/2.4.2,0,,"Mon, 23 Apr 2018 13:25:57 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -"2022-02-07 00:01:41","2.136.0.0",10443,"2-136-0-0.example.com","ssl,ssl-freak,ssl-poodle,vpn","TLSv1.0",12345,"ES","MADRID","MADRID","TLS_RSA_WITH_RC4_128_SHA",1024,"usg50_107BEF336340","usg50_107BEF336340","2014-04-24 00:00:32","2034-04-19 00:00:32","F5:04:98:CD:D4:67:13:E1:77:B7:38:D4:B9:43:C0:72:50:6C:0D:58",53585420,"sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,517311,,"Y","TLS_RSA_EXPORT_WITH_RC4_40_MD5","Communications, Service Provider, and Hosting Service","AF:3A:71:B7:1B:A2:62:4E:87:22:FF:19:3F:84:1F:7F:CC:DC:06:E0:AF:80:E2:5D:33:A5:68:9A:E3:81:25:45","14:92:CC:6B:C7:B3:09:31:50:8C:1C:8D:5B:FD:D1:BE:41:78:80:97:E0:10:11:48:1F:EE:D6:CB:4F:F0:13:D5:05:56:AC:BA:12:12:02:F7:0F:03:40:95:17:8A:5F:79:98:E1:44:EF:E6:5A:44:E3:AC:3A:F8:49:F7:AC:B6:52","E8:5F:96:16:3F:76:35:F0:07:4F:4C:2C:38:FC:27:6B","HTTP/1.1",200,"OK","text/html",,,,"",,"chunked","Mon, 07 Feb 2022 00:01:43 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,"Zyxel","firewall","ZyWALL USG 50",,"enterprise", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_freak.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv deleted file mode 100644 index 4bcc6758a..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv +++ /dev/null @@ -1,32 +0,0 @@ -"timestamp","ip","port","hostname","tag","handshake","asn","geo","region","city","cipher_suite","ssl_poodle","cert_length","subject_common_name","issuer_common_name","cert_issue_date","cert_expiration_date","sha1_fingerprint","cert_serial_number","ssl_version","signature_algorithm","key_algorithm","subject_organization_name","subject_organization_unit_name","subject_country","subject_state_or_province_name","subject_locality_name","subject_street_address","subject_postal_code","subject_surname","subject_given_name","subject_email_address","subject_business_category","subject_serial_number","issuer_organization_name","issuer_organization_unit_name","issuer_country","issuer_state_or_province_name","issuer_locality_name","issuer_street_address","issuer_postal_code","issuer_surname","issuer_given_name","issuer_email_address","issuer_business_category","issuer_serial_number","naics","sic","sector","sha256_fingerprint","sha512_fingerprint","md5_fingerprint","http_response_type","http_code","http_reason","content_type","http_connection","www_authenticate","set_cookie","server_type","content_length","transfer_encoding","http_date","cert_valid","self_signed","cert_expired","browser_trusted","validation_level","browser_error","tlsv13_support","tlsv13_cipher","device_model","device_sector","device_type","device_vendor","device_version","jarm","page_sha256fp","raw_cert","raw_cert_chain" -"2018-08-08 00:51:42","203.0.113.85",8443,"example.com","ssl-poodle","TLSv1.0",65540,"AT","WIEN","VIENNA","TLS_RSA_WITH_RC4_128_SHA","Y",1024,"usg20_107BEF394BA5","usg20_107BEF394BA5","2014-06-25 00:00:42","2034-06-20 00:00:42","04:FA:DE:1D:BD:4A:05:25:61:FB:F3:D6:64:74:66:44:01:22:D7:C3","53AA112A",2,"sha1WithRSAEncryption","rsaEncryption",,,,,,,,,,,,,,,,,,,,,,,,,0,0,,"16:25:9F:C7:A1:8D:64:1F:D9:25:42:BF:87:5C:4F:F3:63:14:97:21:EC:B6:67:10:F2:CA:52:37:C9:FE:49:2E","0B:2D:48:8C:4B:55:8B:F3:AB:F8:45:ED:E0:A0:63:F4:84:2F:4C:19:DC:A8:6F:7D:6A:AF:61:D7:98:AA:58:0F:CB:CA:87:D2:C3:0B:C5:DF:49:A7:84:7C:47:58:89:7D:92:B6:7B:98:7D:B1:64:4B:DC:DD:BE:9D:11:2A:D1:AE","33:E3:61:3F:5D:AA:96:99:38:A5:D6:F1:11:C7:ED:FC","HTTP/1.1",200,"OK","text/html",,,,,,"chunked","Wed, 08 Aug 2018 00:51:44 GMT","Y","Y","N","N","unknown","x509: unknown error",,,,,,,,,,, -2018-04-19 13:32:27,198.51.100.147,443,,ssl-poodle,TLSv1.0,8445,AT,SALZBURG,SALZBURG,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-147.example.net,some_issuer,2017-09-18 08:22:17,2019-09-18 08:22:17,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,746481F100000000000C,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,Tirol,Ehrwald,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:32 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:32,198.51.100.207,443,198-51-100-94.example.net,ssl-poodle,TLSv1.0,25255,AT,SALZBURG,SALZBURG,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2004-06-03 11:11:43,2024-05-29 11:11:43,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0,2,md5WithRSAEncryption,rsaEncryption,,,US,,,,,,,,,,,,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,close,,"rg_cookie_session_id=1654544029; path=/; expires=Fri, 01 Jan 2038",,,,"Thu, 19 Apr 2018 13:32:34 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:32,198.51.100.200,443,198-51-100-200.example.net,ssl-poodle,TLSv1.2,8445,AT,SALZBURG,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-200.example.net,198-51-100-200.example.net,2016-10-01 14:09:12,2020-10-02 14:09:12,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,2E8C9E4A2C7D3EDC,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,,AT,,,,,,,,,,some_org_name,,AT,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=utf-8,close,,,,,,,N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:33,198.51.100.239,443,198-51-100-239.example.net,ssl-poodle,TLSv1.0,8437,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198-51-100-239.example.net,198-51-100-239.example.net,2011-07-27 13:30:18,2012-07-26 13:30:18,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7C91,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,--,SomeState,SomeCity,,,,,198-51-100-239.example.net,,,SomeOrganization,SomeOrganizationalUnit,--,SomeState,SomeCity,,,,,198-51-100-239.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,403,Forbidden,text/html; charset=UTF-8,close,,,Apache/2.2.3 (CentOS),4958,,"Thu, 19 Apr 2018 13:32:35 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:35,198.51.100.156,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2010-01-01 00:00:52,2029-12-27 00:00:52,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4B3D3B34,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Thu, 19 Apr 2018 13:32:37 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:36,198.51.100.122,443,198-51-100-122.example.net,ssl-poodle,TLSv1.2,36351,AT,AUSTRIA,?,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,2048,198-51-100-122.example.net,COMODO RSA Organization Validation Secure Server CA,2017-04-06 00:00:00,2019-04-06 23:59:59,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,CAB81F32F3FF4766BC545A2C14DF34B5,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,Wien,Wien,,1130,,,,,,COMODO CA Limited,,GB,Greater Manchester,Salford,,,,,,,,518210,737401,Information Technology,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,0,,"Thu, 19 Apr 2018 13:32:20 GMT",Y,N,N,Y,OV,,,,,,,,,,,, -2018-04-19 13:32:37,198.51.100.58,443,198-51-100-58.example.net,ssl-poodle,TLSv1.2,12605,AT,OBEROSTERREICH,LINZ,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,example,some_issuer,2015-01-17 16:11:24,2020-01-17 16:11:24,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6D9E2D4443F1D69E4A8865CC1C5B6963,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/8.5,701,,"Thu, 19 Apr 2018 13:34:53 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:38,198.51.100.18,443,198-51-100-18.example.net,ssl-poodle,TLSv1.2,6830,AT,OBEROSTERREICH,LINZ,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,2048,198-51-100-18.example.net,TERENA SSL CA 3,2017-07-14 00:00:00,2020-07-22 12:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0386AD387BEC13878473D23C8C786ECE,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,,Linz,,,,,,,,TERENA,,NL,Noord-Holland,Amsterdam,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,,Close,,BNIS_ChallengeState=Bqyd+IQebjQwiiYNKBJkA5Ta0spL1gX5; Path=/; Exp,,61,,,Y,N,N,Y,OV,,,,,,,,,,,, -2018-04-19 13:32:38,198.51.100.246,443,,ssl-poodle,TLSv1.2,8447,AT,SALZBURG,SALZBURG,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,example,some_issuer,2014-09-01 16:18:46,2054-08-24 16:18:46,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,129FA64A4BE039B54E850F1AA65AD835,2,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,,,ASP.NET_SessionId=e3qfk1dfz2mtqwzoym3gul3r; path=/; HttpOnly,Microsoft-IIS/8.5,145,,"Thu, 19 Apr 2018 13:32:40 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.35,443,198-51-100-35.example.net,ssl-poodle,TLSv1.0,12605,AT,OBEROSTERREICH,LINZ,TLS_RSA_WITH_AES_128_CBC_SHA,Y,2048,198-51-100-35.example.net,Go Daddy Secure Certificate Authority - G2,2017-08-28 13:29:01,2018-09-10 06:28:49,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,90B22B4CEF57C0FC,2,sha256WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,198-51-100-35.example.net,,,,,US,Arizona,Scottsdale,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,266,,"Thu, 19 Apr 2018 13:35:03 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.142,443,,ssl-poodle,TLSv1.0,8447,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,Y,2048,198.51.100.19,198-51-100-19.example.net,2014-12-11 09:57:33,2024-12-08 09:57:33,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,A0571DCBE5E1A2C062D8FB7001271581B5F69824157E385563FA23527E0B,2,sha256WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,DE,,,,,,,198-51-100-19.example.net,,,some_org_name,Engineering,DE,NRW,Wuerselen,,,,,198-51-100-19.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,Close,,,LANCOM,,,"Thur, 19 Apr 2018 13:32:41 GMT",Y,N,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:39,198.51.100.178,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2012-05-30 00:00:44,2032-05-25 00:00:44,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,4FC5632C,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,,chunked,"Thu, 19 Apr 2018 13:32:41 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:40,198.51.100.99,443,198-51-100-99.example.net,ssl-poodle,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-99.example.net,RapidSSL RSA CA 2018,2018-03-30 00:00:00,2019-04-29 12:00:00,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0BDCB5D6D4C22BD2A1CF55584B6DE09C,2,sha256WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,DigiCert Inc,198-51-100-99.example.net,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,404,Not Found,text/html; charset=us-ascii,close,,,Microsoft-HTTPAPI/2.0,315,,"Thu, 19 Apr 2018 13:32:43 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:40,198.51.100.235,443,198-51-100-235.example.net,ssl-poodle,TLSv1.0,25255,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,1024,Nextcloud,Nextcloud,2016-12-13 20:28:39,2017-01-12 20:28:39,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,CDE5769D28C80B6B,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,,AU,Some-State,,,,,,,,,Internet Widgits Pty Ltd,,AU,Some-State,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,400,Bad Request,text/html; charset=UTF-8,close,,nc_sameSiteCookiestrict=true; path=/; httponly;secure; expires=Fr,Apache/2.4.10 (FreeBSD) OpenSSL/0.9.8zd-freebsd PH,6939,,"Thu, 19 Apr 2018 13:32:42 GMT",Y,Y,Y,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:41,198.51.100.187,443,198-51-100-187.example.net,ssl-poodle,TLSv1.2,28760,AT,OBEROSTERREICH,,TLS_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-187.example.net,Go Daddy Secure Certificate Authority - G2,2018-02-12 17:56:01,2020-02-12 17:56:01,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,03BA30FF4972177C,2,sha256WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,198-51-100-187.example.net,,,,,US,Arizona,Scottsdale,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,400,No parameters passed t,text/html,,,,Microsoft-IIS/10.0,11,,"Thu, 19 Apr 2018 13:32:42 GMT",Y,N,N,Y,DV,,,,,,,,,,,, -2018-04-19 13:32:42,198.51.100.213,443,198-51-100-213.example.net,ssl-poodle,TLSv1.2,8447,AT,OBEROSTERREICH,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-213.example.net,some_issuer,2016-09-22 08:12:17,2018-09-22 08:12:17,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,770000000EBB9429663601BAB700000000000E,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,AT,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,301,Moved Permanently,,close,,,Microsoft-IIS/8.5,0,,"Thu, 19 Apr 2018 13:32:44 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:42,198.51.100.74,443,198-51-100-74.example.net,ssl-poodle,TLSv1.0,62363,AT,STEIERMARK,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,DC,DC,2016-12-30 17:15:38,2021-12-30 17:15:38,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,7753CCEB55990A834E15DAC5707D403A,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:44 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:43,198.51.100.145,443,198-51-100-145.example.net,ssl-poodle,TLSv1.0,8447,AT,KARNTEN,KLAGENFURT AM WORTHERSEE,TLS_RSA_WITH_RC4_128_SHA,Y,1024,localdomain,localdomain,2008-10-07 20:12:54,2018-10-07 20:12:54,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,91B04FFCF174CCFF,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,,CA,,,,,,,198-51-100-145.example.net,,,some_org_name,,CA,Quebec,Gatineau,,,,,198-51-100-145.example.net,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,302,Found,text/html; charset=UTF-8,close,,"HOMEBASEID=658512b32961b9b6f8df7a3d4de7fa01; expires=Tue, 19-Jan-",Apache/2.2.3 (Red Hat),0,,"Thu, 19 Apr 2018 12:52:32 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:44,198.51.100.48,443,198-51-100-48.example.net,ssl-poodle,TLSv1.0,1901,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198-51-100-48.example.net,198-51-100-48.example.net,2013-06-15 20:10:49,2023-06-15 20:10:49,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,013F49762DAE,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,198-51-100-48.example.net,,,Western Digital,Branded Products,US,CS,Mountain View,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache,225,,"Thu, 19 Apr 2018 03:08:06 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:45,198.51.100.94,443,198-51-100-94.example.net,ssl-poodle,TLSv1.2,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-94.example.net,RapidSSL CA,2013-04-03 17:02:33,2014-04-07 03:32:33,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,0B697D,2,sha1WithRSAEncryption,rsaEncryption,,some_org_name,,,,,,,,,,KtAjvog6HgAsml0cyxE4hpc9kv8dhgWZ,"GeoTrust, Inc.",,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,302,Found,text/html; charset=utf-8,,,ASP.NET_SessionId=z5lph4ufefkvg1xzmd4q2m33; path=/; HttpOnly,Microsoft-IIS/8.0,144,,"Thu, 19 Apr 2018 13:32:48 GMT",Y,N,Y,N,unknown,x509: certificate has expired or is not yet valid,,,,,,,,,,, -2018-04-19 13:32:45,198.51.100.53,443,198-51-100-53.example.net,ssl-poodle,TLSv1.0,8447,AT,TIROL,,TLS_RSA_WITH_RC4_128_SHA,Y,1024,example,some_issuer,2008-11-13 13:47:18,2028-11-08 13:47:18,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,BE2B43544C0AFF2E,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,DE,,,,,,,198-51-100-53.example.net,,,some_org_name,some_org_name,DE,Niedersachsen,38162 Cremlingen (OT Schandelah),,,,,198-51-100-53.example.net,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=iso-8859-1;,,,,GoAhead-Webs,,,,Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:46,198.51.100.56,443,198-51-100-56.example.net,ssl-poodle,TLSv1.0,8445,AT,TIROL,,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-56.example.net,some_issuer,2016-11-28 08:05:12,2018-11-28 08:05:12,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,637D34F100010000000E,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Microsoft-IIS/7.5,689,,"Thu, 19 Apr 2018 13:32:49 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -2018-04-19 13:32:46,198.51.100.82,443,198-51-100-82.example.net,ssl-poodle,TLSv1.0,6830,AT,OBEROSTERREICH,,TLS_RSA_WITH_AES_128_CBC_SHA,Y,1024,123AFG,7426AC8186F3,2011-01-01 00:00:06,2020-12-29 00:00:06,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,8186F3,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,,,,"Cisco Systems, Inc.",some_org_name,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html,close,,,Embedded HTTP Server.,107,,"Sat, 01 Jan 2011 00:00:45 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:48,198.51.100.29,443,198-51-100-29.example.net,ssl-poodle,TLSv1.0,6830,AT,STEIERMARK,GRAZ,TLS_RSA_WITH_RC4_128_SHA,Y,1024,198.51.100.43,198.51.100.22,2018-04-18 13:32:09,2038-01-15 13:32:09,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,862D98F4B99D0042,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html; charset=utf-8,,,,,,,,Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.114,443,198-51-100-114.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_AES_256_CBC_SHA,Y,1024,198-51-100-114.example.net,198-51-100-114.example.net,2009-08-25 17:47:57,2019-05-25 17:47:57,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,FB09C7848A7F4D77,0,sha1WithRSAEncryption,rsaEncryption,some_org_name,,AT,Vienna,Vienna,,,,,198-51-100-114.example.net,,,Digispectrum,,AT,Vienna,Vienna,,,,,198-51-100-114.example.net,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html; charset=utf-8,close,,b69223925949d45306d32f1a3d23c011=6a01vehilfpml41pl3pq3oth52; path,Apache/2.2.3 (CentOS),,chunked,"Thu, 19 Apr 2018 13:32:52 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.11,443,198-51-100-11.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_RSA_WITH_RC4_128_SHA,Y,2048,FGT60C3G12019794,FGT60C3G12019794,2012-08-10 07:17:11,2022-08-11 07:17:11,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-6CD83A89,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,,,,,,,,,,,,Fortinet Ltd.,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,,79,,"Thu, 19 Apr 2018 13:32:08 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:49,198.51.100.49,443,198-51-100-49.example.net,ssl-poodle,TLSv1.2,8447,AT,NIEDEROSTERREICH,,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,Y,1024,localhost,localhost,2009-11-10 23:48:47,2019-11-08 23:48:47,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,B5C752C98781B503,0,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,518210,737415,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,200,OK,text/html,,,,Apache/2.4.10 (Win32) OpenSSL/1.0.1i PHP/5.5.15,2190,,"Thu, 19 Apr 2018 13:32:55 GMT",Y,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:50,198.51.100.236,443,,ssl-poodle,TLSv1.0,8447,AT,NIEDEROSTERREICH,,TLS_RSA_WITH_AES_128_CBC_SHA,Y,1024,example,some_issuer,2013-01-30 12:00:08,2023-01-28 12:00:08,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,-462A1420,2,sha1WithRSAEncryption,rsaEncryption,some_org_name,some_org_name,US,,,,,,,,,,Netgear Inc.,Netgear Prosafe,US,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.0,200,OK,text/html,close,,,Embedded HTTP Server.,107,,"Sat, 01 Jan 2011 00:00:21 GMT",N,Y,N,N,unknown,x509: unknown error,,,,,,,,,,, -2018-04-19 13:32:50,198.51.100.224,443,198-51-100-224.example.net,ssl-poodle,TLSv1.0,6830,AT,WIEN,VIENNA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,Y,2048,198-51-100-224.example.net,some_issuer,2017-08-03 10:21:50,2019-08-03 10:21:50,14:09:8C:6E:64:5F:50:C9:E9:A3:62:5E:02:BB:33:67:E1:05:D3:D2,6126D181000300000041,2,sha1WithRSAEncryption,rsaEncryption,,,,,,,,,,,,,,,,,,,,,,,,,0,0,,57:7A:FC:7C:A1:0F:79:11:67:E0:31:AC:66:F5:84:22:28:4E:AC:9D:27:A6:3E:93:84:D9:65:8C:FC:21:BF:A1,E9:AE:EE:6C:D1:D1:9C:08:A5:8E:00:07:40:39:60:A0:CF:6D:A0:14:F0:A4:4C:47:28:9D:43:2E:A5:F6:45:66:3A:6F:5A:A4:CC:20:9A:FC:93:88:9B:BD:0B:EF:79:AF:EA:17:0A:08:6A:8A:98:9C:16:EC:94:1E:E7:C4:C7:87,1C:96:78:29:AA:E2:2E:11:AC:61:E5:AA:56:E1:91:BE,HTTP/1.1,401,Unauthorized,text/html,,NTLM,,Microsoft-IIS/7.5,1344,,"Thu, 19 Apr 2018 13:32:52 GMT",N,N,N,N,unknown,x509: certificate signed by unknown authority,,,,,,,,,,, -"2022-02-07 00:01:41","206.162.0.0",10443,,"ssl,ssl-poodle,vpn","TLSv1.2",12345,"CA","BRITISH COLUMBIA","BURNABY","TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA","Y",1024,"FWF60D4615000455","support","2015-01-28 18:14:33","2038-01-19 03:14:07","C9:B0:4E:B7:79:94:B4:DD:A7:15:21:86:43:F9:6E:4B:C9:A2:87:D9","1CA40F",2,"sha1WithRSAEncryption","rsaEncryption","Fortinet","FortiGate","US","California","Sunnyvale",,,,,"support@fortinet.com",,,"Fortinet","Certificate Authority","US","California","Sunnyvale",,,,,"support@fortinet.com",,,517311,,"Communications, Service Provider, and Hosting Service","38:F7:E0:92:24:8C:CB:28:43:93:0B:91:17:30:B1:41:8F:4E:2D:E5:A8:93:AE:4D:FE:53:00:D3:0E:53:02:16","0C:F0:37:3F:A8:93:AE:4D:FE:53:00:D3:2A:E6:6D:0B:02:9D:B9:46:58:A6:9E:5A:35:40:FB:62:9C:81:47:0A:4F:15:5D:53:D9:2F:36:4A:0B:3B:10:61:A9:07:EE:94:EC:00:B8:9C:F7:E0:92:24:8C:CB:28:2C:DD:E7:07:C6","8A:B3:08:20:34:79:94:B4:DD:A7:36:D7:14:6E:33:50","HTTP/1.1",200,"OK","text/html",,,,,131,,"Mon, 07 Feb 2022 00:01:43 GMT","Y","N","N","N","unknown","x509: unknown error",,,,,,"Fortinet","firewall","FortiGate",,"enterprise", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ssl_poodle.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv deleted file mode 100644 index fd671ec90..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","transaction_id","magic_cookie","message_length","message_type","mapped_family","mapped_address","mapped_port","xor_mapped_family","xor_mapped_address","xor_mapped_port","software","fingerprint","amplification","response_size" -"2010-02-10 00:00:00",192.168.0.1,udp,3478,node01.example.com,stun,64512,ZZ,Region,City,0,0,,000000000000000000000000,2112a442,88,0101,01,192.168.0.1,3243,01,192.168.0.1,3243,"Coturn-4.5.1.1 'dan Eider'",0xfaedd06e,5.40,108 -"2010-02-10 00:00:01",192.168.0.2,udp,3478,node02.example.com,stun,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",000000000000000000000000,2112a442,88,0101,01,51.77.39.195,45877,01,192.168.0.2,45877,"Coturn-4.5.1.1 'dan Eider'",0x21128641,5.40,108 -"2010-02-10 00:00:02",192.168.0.3,udp,3478,node03.example.com,stun,64512,ZZ,Region,City,0,0,,000000000000000000000000,2112a442,76,0101,01,192.168.0.3,16321,01,188.68.240.32,16321,"ApolloProxy-1.20.1.28 'sunflower'",,4.80,96 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_stun.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv deleted file mode 100644 index 8f6355491..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sequence_number","ack_number","window_size","urgent_pointer","tcp_flags","raw_packet","sector" -"2022-01-10 09:18:23","66.9.0.0","tcp",80,,"synfulknock",18885,"US","NEW JERSEY","JERSEY CITY",,,0,791102,8192,0,4608,"3cfdfec601e4700f6a9a2000080045000034c3780000f706789442099555b869f7ee0050b20800000000000c123e8012200002aa0000020405b40101040201030305", -"2022-01-10 09:19:17","213.131.0.0","tcp",80,"host-213-131-55-210-customer.wanex.net","synfulknock",35805,"GE","TBILISI","TBILISI",,,0,791102,8192,0,4608,"90e2baaf0b84700f6a9a200008004500003434100000f2064382d58337d2b8698b720050916200000000000c123e8012200059d50000020405b40101040201030305", -"2022-01-10 09:27:39","213.178.0.0","tcp",80,,"synfulknock",29256,"SY","DIMASHQ","DAMASCUS",,,0,791102,8192,0,4608,"90e2bab9cfd4700f6a9a20000800450000340f1d0000ea068bdad5b2e6914a522f360050eb5200000000000c123e801220001b4a0000020405b40101040201030305" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_synfulknock.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv deleted file mode 100644 index 3309e9a3d..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" -"2019-09-04 12:27:34","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.5|Login:" -"2019-09-04 12:27:40","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.45.3 (stable)|Login:" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_telnet.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv deleted file mode 100644 index 3dde133d4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","opcode","errorcode","error","errormessage","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,35067,node01.example.com,tftp,64512,ZZ,Region,City,0,0,5,0,"Not defined","Get not supported",22,1.57 -"2010-02-10 00:00:01",192.168.0.2,udp,56709,node02.example.com,tftp,64512,ZZ,Region,City,0,0,5,1,"File not found","File not found",19,1.36 -"2010-02-10 00:00:02",192.168.0.3,udp,32785,node03.example.com,tftp,64512,ZZ,Region,City,0,0,5,2,"Access violation","Access violation",21,1.50 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_tftp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv deleted file mode 100644 index efeab02c4..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","mac","radioname","essid","modelshort","modelfull","firmware","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,10001,node01.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,00156db98c3a,kachine.meta.lidia.tereixa,Kachine-Meta-Lidia-Tereixa,NS5,,XS5.ar2313.v3.5.4494.091109.1459,148,37.00 -"2010-02-10 00:00:01",192.168.0.2,udp,10001,node02.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,00156d7c9188,adana.mason.lanikai.ozaner,Adana-Mason-Lanikai-Ozaner,LM5,"NanoStation Loco M5",XM.ar7240.v5.6.3.28591.151130.1749,156,39.00 -"2010-02-10 00:00:02",192.168.0.3,udp,10001,node03.example.com,"ubiquiti,iot",64512,ZZ,Region,City,0,0,0418d6000fd5,tailynn.kadija.noreen.dinkar,Tailynn-Kadija-Noreen-Dinkar,P2B-400,"PowerBeam M2 400",XW.ar934x.v5.6.5.29033.160515.2108,145,36.25 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ubiquiti.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv deleted file mode 100644 index 000f5ed42..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv +++ /dev/null @@ -1,3 +0,0 @@ -"timestamp","ip","port","hostname","asn","geo","region","city","naics","sic","product","banner","sector" -"2019-09-04 14:51:44","198.123.245.53",5678,,5678,"AA","LOCATION","LOCATION",0,0,"Apple remote desktop vnc","RFB 003.889", -"2019-09-04 14:51:44","198.123.245.112",5678,"localhost.localdomain",5678,"AA","LOCATION","LOCATION",517311,0,"RealVNC Enterprise v5.3 or later","RFB 005.000", diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_vnc.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv deleted file mode 100644 index 7e279ca3e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","sector","response_size","amplification","error","raw_response" -"2010-02-10 00:00:00",192.168.0.1,udp,3702,node01.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",989,164.83,"Validation constraint violation: SOAP message expected",c2FtcGxlIHJlc3BvbnNlIGRhdGEK -"2010-02-10 00:00:01",192.168.0.2,udp,3702,node02.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",918,183.60,"Validation constraint violation: missing root element",c2FtcGxlIHJlc3BvbnNlIGRhdGEK -"2010-02-10 00:00:02",192.168.0.3,udp,3702,node03.example.com,ws-discovery,64512,ZZ,Region,City,0,0,"Communications, Service Provider, and Hosting Service",989,197.80,"Validation constraint violation: SOAP message expected",c2FtcGxlIHJlc3BvbnNlIGRhdGEK diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license deleted file mode 100644 index 9f58c89ef..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_ws_discovery.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv deleted file mode 100644 index 7e83bbaf8..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","opcode","reported_hostname","status","size","amplification" -"2010-02-10 00:00:00",192.168.0.1,udp,177,node01.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node01.example.com,"Linux 3.0.101-100-default",44,6.29 -"2010-02-10 00:00:01",192.168.0.2,udp,47074,node02.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node02.example.com,"Linux 2.6.9-103.ELsmp",48,6.86 -"2010-02-10 00:00:02",192.168.0.3,udp,177,node03.example.com,xdmcp,64512,ZZ,Region,City,0,0,Willing,node03.example.com,"1 user, load: 6,5, 6,6, 6,6",46,6.57 diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/scan_xdmcp.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license deleted file mode 100644 index 942a94035..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/sinkhole_http_drone.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2019 Guillermo Rodriguez -SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv deleted file mode 100644 index 2e7b59158..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv +++ /dev/null @@ -1,4 +0,0 @@ -"timestamp","ip","port","protocol","asn","geo","region","city","hostname","naics","sector","tag","public_source","status","detail","method","device_vendor" -"2010-02-10 00:00:00",192.168.0.1,,,64512,ZZ,Region,City,node01.example.com,0,"Communications, Service Provider, and Hosting Service",cyclops-blink,,"likely compromised",,, -"2010-02-10 00:00:01",192.168.0.2,,,64512,ZZ,Region,City,node02.example.com,0,"Communications, Service Provider, and Hosting Service",cyclops-blink,,"likely compromised",,, -"2010-02-10 00:00:02",192.168.0.3,,,64512,ZZ,Region,City,node03.example.com,0,"Professional, Scientific, and Technical Services",cyclops-blink,,"likely compromised",,, diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license deleted file mode 100644 index f512a890e..000000000 --- a/intelmq/tests/bots/parsers/shadowserver/testdata/special.csv.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: 2022 The Shadowserver Foundation -SPDX-License-Identifier: AGPL-3.0-or-later From a33fa64569426ee47d039e1dd69bb2a76db52de7 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:57:12 +0000 Subject: [PATCH 43/67] remove json parser - csv provides better performance --- .../shadowserver/collector_reports_api.py | 7 +- .../bots/parsers/shadowserver/parser_json.py | 171 ------------------ .../test_collector_reports_api.py | 7 +- 3 files changed, 7 insertions(+), 178 deletions(-) delete mode 100644 intelmq/bots/parsers/shadowserver/parser_json.py diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 5e0b045c8..dc8bd6b42 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv' or 'json'). The default is 'json' for compatibility. Using 'csv' is recommended for best performance. + file_format (str): File format to download ('csv'). The 'json' option is not longer supported. """ country = None @@ -67,11 +67,10 @@ def init(self): self._report_list.append(self.country) if self.file_format is not None: - if not (self.file_format == 'csv' or self.file_format == 'json'): + if not (self.file_format == 'csv'): raise ValueError('Invalid file_format') else: - self.file_format = 'json' - self.logger.info("For best performance, set 'file_format' to 'csv' and use intelmq.bots.parsers.shadowserver.parser.") + self.file_format = 'csv' self.preamble = f'{{ "apikey": "{self.api_key}" ' diff --git a/intelmq/bots/parsers/shadowserver/parser_json.py b/intelmq/bots/parsers/shadowserver/parser_json.py deleted file mode 100644 index 893ad877b..000000000 --- a/intelmq/bots/parsers/shadowserver/parser_json.py +++ /dev/null @@ -1,171 +0,0 @@ -""" -Shadowserver JSON Parser - -SPDX-FileCopyrightText: 2020 Intelmq Team -SPDX-License-Identifier: AGPL-3.0-or-later -""" -import re -from typing import Any - -from intelmq.lib.bot import ParserBot -from intelmq.lib.exceptions import InvalidKey, InvalidValue -import intelmq.lib.message as libmessage -import intelmq.bots.parsers.shadowserver._config as config - - -class ShadowserverJSONParserBot(ParserBot): - """Parse all Shadowserver feeds in JSON format (data coming from the reports API) - Shadowserver JSON Parser - - Parameters: - feedname (str): The name of the feed - """ - __is_filename_regex = re.compile(r'^(?:\d{4}-\d{2}-\d{2}-)?(\w+)(-\w+)*\.json$') - feedname = None - _sparser_config = None - recover_line = ParserBot.recover_line_json - overwrite = True - - def init(self): - if self.feedname is not None: - feedname = self.feedname - self._sparser_config = config.get_feed_by_feedname(feedname) - if self._sparser_config: - self.logger.info('Using fixed feed name %r for parsing reports.', feedname) - else: - self.logger.info('Could not determine the feed by the feed name %r given by parameter. ' - 'Will determine the feed from the file names.', feedname) - - def parse(self, report): - report_name = report.get('extra.file_name') - if not report_name: - raise ValueError("No feedname given as parameter and the " - "processed report has no 'extra.file_name'. " - "Ensure that at least one is given. " - "Also have a look at the documentation of the bot.") - - filename_search = self.__is_filename_regex.search(report_name) - - if not filename_search: - raise ValueError(f"Report's 'extra.file_name' {report_name!r} is not valid.") - report_name = filename_search.group(1) - - self.logger.debug("Detected report's file name: %s.", report_name) - retval = config.get_feed_by_filename(report_name) - - if not retval: - raise ValueError('Could not get a config for {!r}, check the documentation.' - ''.format(report_name)) - self.feedname, self._sparser_config = retval - - return self.parse_json(report) - - def parse_line(self, line: Any, report: libmessage.Report): - conf = self._sparser_config - processedkeys = [] - - event = self.new_event(report) - event.add('feed.name', self.feedname, overwrite=self.overwrite) - - extra = {} - - for entry in conf.get('required_fields'): - intelmqkey, shadowserverkey = entry[0], entry[1] - value = self.get_value_from_config(line, entry) - - if value is not None: - event.add(intelmqkey, value) - processedkeys.append(shadowserverkey) - - # Now add optional fields. - # This action may fail, the value is added to - # extra if an add operation failed - for entry in conf.get('optional_fields'): - intelmqkey, shadowserverkey = entry[0], entry[1] - try: - value = self.get_value_from_config(line, entry) - except ValueError: - self.logger.warning('Optional key %s not found in feed %s. Possible change in data' - ' format or misconfiguration.', shadowserverkey, self.feedname) - continue - - intelmqkey, shadowserverkey = entry[0], entry[1] - if value is not None: - if intelmqkey == 'extra.': - extra[shadowserverkey] = value - processedkeys.append(shadowserverkey) - continue - elif intelmqkey and intelmqkey.startswith('extra.'): - extra[intelmqkey.replace('extra.', '', 1)] = value - processedkeys.append(shadowserverkey) - continue - elif intelmqkey is False: - # ignore it explicitly - processedkeys.append(shadowserverkey) - continue - try: - event.add(intelmqkey, value) - processedkeys.append(shadowserverkey) - except InvalidValue: - self.logger.debug('Could not add key %r in feed %r, adding it to extras.', - shadowserverkey, self.feedname) - except InvalidKey: - extra[intelmqkey] = value - processedkeys.append(shadowserverkey) - else: - processedkeys.append(shadowserverkey) - - # Now add additional constant fields. - event.update(conf.get('constant_fields', {})) - - event.add('raw', self.recover_line_json(line)) - - # Add everything which could not be resolved to extra. - for key in line: - if key not in processedkeys: - val = line[key] - if not val == "": - extra[key] = val - - if extra: - event.add('extra', extra) - - yield event - - def get_value_from_config(self, data, entry): - """ - Given a specific config, get the value for that data based on the entry - """ - conv_fun = None - - shadowserverkey = entry[1] - raw_value = data.get(shadowserverkey, None) - value = raw_value - - if raw_value is None: - raise ValueError('Key {!r} not found in feed {!r}. Possible change in data' - ' format or misconfiguration.'.format(shadowserverkey, self.feedname)) - if len(entry) > 2: - conv_fun = entry[2] - - if conv_fun is not None and raw_value is not None: - if len(entry) == 4 and entry[3]: - try: - value = conv_fun(raw_value, data) - except Exception: - self.logger.error('Could not convert shadowserverkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowserverkey, self.feedname, raw_value, conv_fun.__name__) - raise - else: - try: - value = conv_fun(raw_value) - except Exception: - self.logger.error('Could not convert shadowserverkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowserverkey, self.feedname, raw_value, conv_fun.__name__) - raise - return value - - -BOT = ShadowserverJSONParserBot diff --git a/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py b/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py index a625c9d34..2bf6e61e9 100644 --- a/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py +++ b/intelmq/tests/bots/collectors/shadowserver/test_collector_reports_api.py @@ -14,12 +14,13 @@ RANDSTR = secrets.token_urlsafe(50) ASSET_PATH = pathlib.Path(__file__).parent / 'reports-list.json' PARAMETERS = {'reports': 'anarres', 'api_key': RANDSTR, 'secret': RANDSTR, 'logging_level': 'DEBUG', 'types': ['scan_smb', 'cisco_smart_install', 'nonexistent'], 'name': 'shadowservercollector'} -REPORT = {'__type': 'Report', 'extra.file_name': '2020-08-02-scan_smb-anarres-geo.json', 'feed.accuracy': 100.0, 'feed.name': 'shadowservercollector', 'raw': 'e30='} +REPORT = {'__type': 'Report', 'extra.file_name': '2020-08-02-scan_smb-anarres-geo.csv', 'feed.accuracy': 100.0, 'feed.name': 'shadowservercollector', 'raw': 'e30='} def prepare_mocker(mocker): mocker.post('https://transform.shadowserver.org/api2/reports/list', content=ASSET_PATH.read_bytes()) - mocker.post('https://transform.shadowserver.org/api2/reports/download', text='{}') + mocker.get('https://dl.shadowserver.org/xNDSuwXrKnrLrDopU926rR75CAESMWesVCKsuyI8b8ncTv7GCX', text='{}') + mocker.get('https://dl.shadowserver.org/unnzVtn92tS9459rKIEz2J8qb7oJDv0Fa2feGUOiJLCDLqBXnN', text='{}') # Explicit skip_redis is required (although implicitly called by no_cache), otherwise fails in package build environments @@ -80,7 +81,7 @@ def test_report_sent(self, mocker): self.cache.flushdb() prepare_mocker(mocker) self.run_bot(iterations=1, parameters=PARAMETERS) - self.assertAnyLoglineEqual("Sent report: '2020-08-02-cisco_smart_install-anarres-geo.csv' (fixed: '2020-08-02-cisco_smart_install-anarres-geo.json', size: 0.00195 KiB).", 'DEBUG') + self.assertAnyLoglineEqual("Sent report: '2020-08-02-cisco_smart_install-anarres-geo.csv' (fixed: '2020-08-02-cisco_smart_install-anarres-geo.csv', size: 0.00195 KiB).", 'DEBUG') def test_report_content(self, mocker): self.cache.flushdb() From cd3338a3fc938cb14fa020996a6b71dfd7203697 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 11 Apr 2023 23:59:42 +0000 Subject: [PATCH 44/67] dynamic configuration model --- intelmq/bots/parsers/shadowserver/README.md | 7 + intelmq/bots/parsers/shadowserver/_config.py | 4202 +---------------- intelmq/bots/parsers/shadowserver/parser.py | 46 +- .../parsers/shadowserver/schema.json.test | 180 + .../parsers/shadowserver/update_schema.py | 12 + 5 files changed, 303 insertions(+), 4144 deletions(-) create mode 100644 intelmq/bots/parsers/shadowserver/schema.json.test create mode 100644 intelmq/bots/parsers/shadowserver/update_schema.py diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index eb0ddfb4a..297930861 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,3 +7,10 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/_version_. + +For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. + +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory + +The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index bea3d0c0b..a7b80b7a6 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -77,20 +77,34 @@ feed_idx is not complete. """ +import os import re import base64 import binascii +import json +import urllib.request +import tempfile from typing import Optional, Dict, Tuple, Any import intelmq.lib.harmonization as harmonization +class __Container: + pass + +__config = __Container() +__config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') +__config.schema_mtime = 0.0 +__config.feedname_mapping = {} +__config.filename_mapping = {} def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: - return feedname_mapping.get(given_feedname, None) + reload() + return __config.feedname_mapping.get(given_feedname, None) def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, Any]]]: - return filename_mapping.get(given_filename, None) + reload() + return __config.filename_mapping.get(given_filename, None) def add_UTC_to_timestamp(value: str) -> str: @@ -165,11 +179,6 @@ def invalidate_zero(value: str) -> Optional[int]: return int(value) if value and int(value) != 0 else None -# TODO this function is a wild guess... -def set_tor_node(value: str) -> Optional[bool]: - return True if value else None - - def validate_ip(value: str) -> Optional[str]: """Remove "invalid" IP.""" # FIX: https://github.com/certtools/intelmq/issues/1720 # TODO: Find better fix @@ -240,4126 +249,63 @@ def scan_exchange_identifier(field): return 'exchange-server-webshell' return 'vulnerable-exchange-server' +functions = { + 'add_UTC_to_timestamp': add_UTC_to_timestamp, + 'convert_bool': convert_bool, + 'validate_to_none': validate_to_none, + 'convert_int': convert_int, + 'convert_float': convert_float, + 'convert_http_host_and_url': convert_http_host_and_url, + 'invalidate_zero': invalidate_zero, + 'validate_ip': validate_ip, + 'validate_network': validate_network, + 'validate_fqdn': validate_fqdn, + 'convert_date': convert_date, + 'convert_date_utc': convert_date_utc, + 'force_base64': force_base64, + 'scan_exchange_taxonomy': scan_exchange_taxonomy, + 'scan_exchange_type': scan_exchange_type, + 'scan_exchange_identifier': scan_exchange_identifier, + } + + +def reload (): + """ reload the configuration if it has changed """ + mtime = 0.0 + + if (os.path.isfile(__config.schema_file)): + mtime = os.path.getmtime(__config.schema_file) + if __config.schema_mtime == mtime: + return + schema_file = __config.schema_file + else: + # load a test schema if one has not been downloaded yet + schema_file = __config.schema_file + schema_file += '.test' + + __config.feedname_mapping.clear() + __config.filename_mapping.clear() + with open(schema_file) as fh: + schema = json.load(fh) + for report in schema: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + __config.schema_mtime = mtime + +def update_schema (version): + """ download the latest configuration """ + (th, tmp) = tempfile.mkstemp() + url = 'https://interchange.shadowserver.org/intelmq/'+version + try: + urllib.request.urlretrieve(url, tmp) + except: + raise ValueError("Failed to download %r" % url) -# BEGIN CONFGEN - -# https://www.shadowserver.org/what-we-do/network-reporting/blocklist-report/ -blocklist = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.network', 'ip', validate_network), - ('extra.', 'tag', validate_to_none), - ('source.reverse_dns', 'hostname'), - ('extra.', 'source', validate_to_none), - ('extra.', 'reason', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'blacklisted-ip', - 'classification.taxonomy': 'other', - 'classification.type': 'blacklist', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/compromised-website-report/ -compromised_website = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'application', validate_to_none), - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'http_host', validate_fqdn), - ('source.reverse_dns', 'hostname'), - ('malware.name', 'tag'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('event_description.text', 'category', validate_to_none), - ('extra.', 'system', validate_to_none), - ('extra.', 'detected_since', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'redirect_target', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'cc_url', validate_to_none), - ('extra.', 'family', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'intrusions', - 'classification.type': 'system-compromise', - 'classification.identifier': 'compromised-website', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/device-identification-report/ -device_id = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'undetermined', - 'classification.identifier': 'device-id', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ddos-participant-report/ -event_ddos_participant = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ('extra.', 'http_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'ddos-participant', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-brute-force-events-report/ -event_honeypot_brute_force = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'application'), - ('destination.account', 'username', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'service', validate_to_none), - ('extra.', 'start_time', convert_date_utc), - ('extra.', 'end_time', convert_date_utc), - ('extra.', 'client_version', validate_to_none), - ('extra.', 'password', validate_to_none), - ('extra.', 'payload_url', validate_to_none), - ('extra.', 'payload_md5', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'intrusion-attempts', - 'classification.type': 'brute-force', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-darknet-events-report/ -event_honeypot_darknet = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'tag', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'count', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ddos-events/ -event_honeypot_ddos = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ('extra.', 'http_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'honeypot-ddos', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-amplification-ddos-events-report/ -event_honeypot_ddos_amp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'avg_pps', convert_float), - ('extra.', 'max_pps', convert_float), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'count', convert_int), - ('extra.', 'bytes', convert_int), - ('extra.', 'end_time', convert_date_utc), - ('extra.', 'duration', convert_int), - ], - 'constant_fields': { - 'classification.identifier': 'amplification-ddos-victim', - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ddos-target-events-report/ -event_honeypot_ddos_target = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'attack_src_port', convert_int), - ('extra.', 'http_usessl', convert_bool), - ('extra.', 'ip_header_seqnum', convert_int), - ('extra.', 'ip_header_ttl', convert_int), - ('extra.', 'number_of_connections', convert_int), - ('extra.', 'packet_length', convert_int), - ('extra.', 'packet_randomized', convert_bool), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'domain_source', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'dst_network', validate_to_none), - ('extra.', 'dst_netmask', validate_to_none), - ('extra.', 'attack', validate_to_none), - ('extra.', 'duration', convert_int), - ('extra.', 'attack_src_ip', validate_to_none), - ('extra.', 'domain', validate_to_none), - ('extra.', 'domain_transaction_id', validate_to_none), - ('extra.', 'gcip', validate_to_none), - ('extra.', 'http_method', validate_to_none), - ('extra.', 'http_path', validate_to_none), - ('extra.', 'http_postdata', validate_to_none), - ('extra.', 'ip_header_ack', validate_to_none), - ('extra.', 'ip_header_acknum', validate_to_none), - ('extra.', 'ip_header_dont_fragment', validate_to_none), - ('extra.', 'ip_header_fin', validate_to_none), - ('extra.', 'ip_header_identity', validate_to_none), - ('extra.', 'ip_header_psh', validate_to_none), - ('extra.', 'ip_header_rst', validate_to_none), - ('extra.', 'ip_header_syn', validate_to_none), - ('extra.', 'ip_header_tos', validate_to_none), - ('extra.', 'ip_header_urg', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'availability', - 'classification.type': 'ddos', - 'classification.identifier': 'honeypot-ddos-target', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-http-scanner-events/ -event_honeypot_http_scan = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('user_agent', 'http_agent', validate_to_none), - ('extra.method', 'http_request_method', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'pattern', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('extra.', 'url_scheme', validate_to_none), - ('extra.', 'session_tags', validate_to_none), - ('extra.', 'vulnerability_enum', validate_to_none), - ('extra.', 'vulnerability_id', validate_to_none), - ('extra.', 'vulnerability_class', validate_to_none), - ('extra.', 'vulnerability_score', validate_to_none), - ('extra.', 'vulnerability_severity', validate_to_none), - ('extra.', 'vulnerability_version', validate_to_none), - ('extra.', 'threat_framework', validate_to_none), - ('extra.', 'threat_tactic_id', validate_to_none), - ('extra.', 'threat_technique_id', validate_to_none), - ('extra.', 'target_vendor', validate_to_none), - ('extra.', 'target_product', validate_to_none), - ('extra.', 'target_class', validate_to_none), - ('extra.', 'file_md5', validate_to_none), - ('extra.', 'file_sha256', validate_to_none), - ('extra.', 'request_raw', force_base64), - ('extra.', 'body_raw', force_base64), - ], - 'constant_fields': { - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - 'protocol.application': 'http', - 'classification.identifier': 'honeypot-http-scan', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/honeypot-ics-scanner-events-report/ -event_honeypot_ics_scan = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('malware.name', 'infection'), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'state', validate_to_none), - ('extra.', 'sensor_id', validate_to_none), - ('extra.', 'slave_id', validate_to_none), - ('extra.', 'function_code', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'response', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'ics', - 'classification.taxonomy': 'information-gathering', - 'classification.type': 'scanner', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ip-spoofer-events-report/ -event_ip_spoofer = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'infection', validate_to_none), - ('source.network', 'network', validate_network), - ('extra.', 'tag', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'family', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('extra.', 'routedspoof', validate_to_none), - ('extra.', 'session', validate_to_none), - ('extra.', 'nat', convert_bool), - ], - 'constant_fields': { - 'classification.taxonomy': 'fraud', - 'classification.type': 'masquerade', - 'classification.identifier': 'ip-spoofer', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-events-report/ -event_sinkhole = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'infection', validate_to_none), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-dns-events-report/ -event_sinkhole_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('extra.naics', 'src_naics', invalidate_zero), - ('extra.sector', 'src_sector', validate_to_none), - ('extra.dns_query_type', 'query_type'), - ('extra.dns_query', 'query'), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'count', convert_int), - ], - 'constant_fields': { - 'classification.identifier': 'sinkholedns', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-events-report/ -event_sinkhole_http = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'src_ip', validate_ip), - ('source.port', 'src_port', convert_int), - ], - 'optional_fields': [ - ('classification.identifier', 'tag'), - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('source.asn', 'src_asn', invalidate_zero), - ('source.geolocation.cc', 'src_geo'), - ('source.geolocation.region', 'src_region'), - ('source.geolocation.city', 'src_city'), - ('source.reverse_dns', 'src_hostname'), - ('extra.source.naics', 'src_naics', invalidate_zero), - ('extra.source.sector', 'src_sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_agent', validate_to_none), - ('extra.', 'forwarded_by', validate_to_none), - ('extra.', 'ssl_cipher', validate_to_none), - ('extra.', 'http_referer', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'infected-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sinkhole-http-referer-events-report/ -event_sinkhole_http_referer = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ], - 'optional_fields': [ - ('malware.name', 'family', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'infection', validate_to_none), - ('protocol.transport', 'protocol'), - ('extra.', 'http_referer_ip', validate_ip), - ('extra.', 'http_referer_port', convert_int), - ('extra.', 'http_referer_asn', invalidate_zero), - ('extra.', 'http_referer_geo', validate_to_none), - ('extra.', 'http_referer_region', validate_to_none), - ('extra.', 'http_referer_city', validate_to_none), - ('extra.', 'http_referer_hostname', validate_to_none), - ('extra.', 'http_referer_naics', invalidate_zero), - ('extra.', 'http_referer_sector', validate_to_none), - ('destination.ip', 'dst_ip', validate_ip), - ('destination.port', 'dst_port', convert_int), - ('destination.asn', 'dst_asn', invalidate_zero), - ('destination.geolocation.cc', 'dst_geo'), - ('destination.geolocation.region', 'dst_region'), - ('destination.geolocation.city', 'dst_city'), - ('destination.reverse_dns', 'dst_hostname', validate_to_none), - ('extra.destination.naics', 'dst_naics', invalidate_zero), - ('extra.destination.sector', 'dst_sector', validate_to_none), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'application', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'event_id', validate_to_none), - ('destination.url', 'http_url', convert_http_host_and_url, True), - ('destination.fqdn', 'http_host', validate_fqdn), - ('extra.', 'http_referer', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'sinkhole-http-referer', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/malware-url-report/ -malware_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'source', validate_to_none), - ('malware.hash.sha256', 'sha256', validate_to_none), - ('extra.', 'application', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'malware-url', - }, -} - -phish_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'source', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'fraud', - 'classification.type': 'phishing', - 'classification.identifier': 'phish-url', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-http-proxy-report/ -population_http_proxy = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('malware.name', 'tag'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'proxy_authenticate', validate_to_none), - ('extra.', 'via', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http-proxy', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'http', - }, -} - -# http://www.shadowserver.org/wiki/pmwiki.php/Services/Sandbox-Connection -sandbox_conn = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('destination.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('malware.hash.md5', 'md5', validate_to_none), - ('protocol.transport', 'protocol'), - ('extra.', 'bytes_in', validate_to_none), - ('extra.', 'bytes_out', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'sandbox-conn', - }, -} - -sandbox_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ], - 'optional_fields': [ - ('extra.dns_query_type', 'type', validate_to_none), - ('malware.hash.md5', 'md5hash', validate_to_none), - ('extra.', 'request', validate_to_none), - ('extra.', 'response', validate_to_none), - ('extra.', 'family', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'source', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'dns', - 'classification.identifier': 'sandbox-dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/sandbox-url-report/ -sandbox_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('destination.fqdn', 'host', validate_fqdn), - ('extra.http_request_method', 'method', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('malware.hash.md5', 'md5', validate_to_none), - ('destination.url', 'url', convert_http_host_and_url, True), - ('user_agent', 'user_agent', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'malicious-code', - 'classification.type': 'malware-distribution', - 'classification.identifier': 'sandbox-url', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-adb-report/ -scan_adb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'name', validate_to_none), - ('extra.', 'model', validate_to_none), - ('extra.', 'device', validate_to_none), - ('extra.', 'features', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-adb', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'adb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-afp-report/ -scan_afp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'machine_type', validate_to_none), - ('extra.', 'afp_versions', validate_to_none), - ('extra.', 'uams', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'signature', validate_to_none), - ('extra.', 'directory_service', validate_to_none), - ('extra.', 'utf8_servername', validate_to_none), - ('extra.', 'network_address', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-afp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'afp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-amqp-report/ -scan_amqp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'channel', validate_to_none), - ('extra.', 'message_length', convert_int), - ('extra.', 'class', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'version_major', validate_to_none), - ('extra.', 'version_minor', validate_to_none), - ('extra.', 'capabilities', validate_to_none), - ('extra.', 'cluster_name', validate_to_none), - ('extra.', 'platform', validate_to_none), - ('extra.', 'product', validate_to_none), - ('extra.', 'product_version', validate_to_none), - ('extra.', 'mechanisms', validate_to_none), - ('extra.', 'locales', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-amqp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'amqp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-apple-remote-desktop-ard-report/ -scan_ard = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ard', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-chargen-report/ -scan_chargen = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.response_size', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'chargen', - 'classification.identifier': 'open-chargen', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-cisco-smart-install-report/ -scan_cisco_smart_install = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-cisco-smart-install', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'cisco-smart-install', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-coap-report/ -scan_coap = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'response', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-coap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'coap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-couchdb-report/ -scan_couchdb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'server_version', validate_to_none), - ('extra.', 'couchdb_message', validate_to_none), - ('extra.', 'couchdb_version', validate_to_none), - ('extra.', 'git_sha', validate_to_none), - ('extra.', 'features', validate_to_none), - ('extra.', 'vendor', validate_to_none), - ('extra.', 'visible_databases', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'error_reason', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'CouchDB', - 'classification.identifier': 'open-couchdb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-cwmp-report/ -scan_cwmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'cwmp', - 'classification.identifier': 'open-cwmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-db2-discovery-service-report/ -scan_db2 = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'db2_hostname', validate_to_none), - ('extra.', 'servername', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-db2-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'db2', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-ddos-middlebox-report/ -scan_ddos_middlebox = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'source_port', validate_to_none), - ('extra.', 'bytes', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'method', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ddos-middlebox', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/dns-open-resolvers-report/ -scan_dns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'min_amplification', convert_float), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'dns_version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'dns-open-resolver', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'dns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-docker-service-report/ -scan_docker = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'experimental', validate_to_none), - ('extra.', 'api_version', validate_to_none), - ('extra.', 'arch', validate_to_none), - ('extra.', 'go_version', validate_to_none), - ('extra.os.name', 'os', validate_to_none), - ('extra.', 'kernel_version', validate_to_none), - ('extra.', 'git_commit', validate_to_none), - ('extra.', 'min_api_version', validate_to_none), - ('extra.', 'build_time', validate_to_none), - ('extra.', 'pkg_version', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'docker', - 'classification.identifier': 'open-docker', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-dvr-dhcpdiscover-report/ -scan_dvr_dhcpdiscover = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('extra.', 'video_input_channels', convert_int), - ('extra.', 'alarm_input_channels', convert_int), - ('extra.', 'video_output_channels', convert_int), - ('extra.', 'alarm_output_channels', convert_int), - ('extra.', 'remote_video_input_channels', convert_int), - ('extra.', 'ipv4_dhcp_enable', convert_bool), - ('extra.', 'ipv6_dhcp_enable', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_id', validate_to_none), - ('extra.', 'device_serial', validate_to_none), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'manufacturer', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'http_port', convert_int), - ('extra.', 'internal_port', convert_int), - ('extra.', 'mac_address', validate_to_none), - ('extra.', 'ipv4_address', validate_to_none), - ('extra.', 'ipv4_gateway', validate_to_none), - ('extra.', 'ipv4_subnet_mask', validate_to_none), - ('extra.', 'ipv6_address', validate_to_none), - ('extra.', 'ipv6_link_local', validate_to_none), - ('extra.', 'ipv6_gateway', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-dvr-dhcpdiscover', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-elasticsearch-report/ -scan_elasticsearch = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'build_snapshot', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'ok', convert_bool), - ('extra.', 'name', validate_to_none), - ('extra.', 'cluster_name', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'build_hash', validate_to_none), - ('extra.', 'build_timestamp', validate_to_none), - ('extra.', 'lucene_version', validate_to_none), - ('extra.', 'tagline', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'elasticsearch', - 'classification.identifier': 'open-elasticsearch', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-erlang-port-mapper-report-daemon/ -scan_epmd = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'nodes', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'Erlang Port Mapper Daemon', - 'classification.identifier': 'open-epmd', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-exchange-server-report/ -scan_exchange = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('classification.taxonomy', 'tag', scan_exchange_taxonomy), - ('classification.type', 'tag', scan_exchange_type), - ('classification.identifier', 'tag', scan_exchange_identifier), - ('extra.', 'tag', validate_to_none), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'servername', validate_to_none), - ('destination.url', 'url', convert_http_host_and_url, True), - ], - 'constant_fields': { - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ftp-report/ -scan_ftp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'auth_tls_response', validate_to_none), - ('extra.', 'auth_ssl_response', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ftp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ftp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-hadoop-report/ -scan_hadoop = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'total_disk', convert_int), - ('extra.', 'used_disk', convert_int), - ('extra.', 'free_disk', convert_int), - ('source.reverse_dns', 'hostname'), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'clusterid', validate_to_none), - ('extra.', 'livenodes', validate_to_none), - ('extra.', 'namenodeaddress', validate_to_none), - ('extra.', 'volumeinfo', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-hadoop', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'hadoop', - 'protocol.transport': 'tcp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-http-report/ -scan_http = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-http-proxy-report/ -scan_http_proxy = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'proxy_authenticate', validate_to_none), - ('extra.', 'via', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ], - 'constant_fields': { - 'classification.identifier': 'open-http-proxy', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-http-report/ -scan_http_vulnerable = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'version', validate_to_none), - ('extra.', 'build_date', validate_to_none), - ('extra.', 'detail', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-http', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'http', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ics-report/ -scan_ics = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_id', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-ics', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ipmi-report/ -scan_ipmi = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'none_auth', convert_bool), - ('extra.', 'md2_auth', convert_bool), - ('extra.', 'md5_auth', convert_bool), - ('extra.', 'passkey_auth', convert_bool), - ('extra.', 'oem_auth', convert_bool), - ('extra.', 'permessage_auth', convert_bool), - ('extra.', 'userlevel_auth', convert_bool), - ('extra.', 'usernames', convert_bool), - ('extra.', 'nulluser', convert_bool), - ('extra.', 'anon_login', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'ipmi_version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'defaultkg', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'deviceid', validate_to_none), - ('extra.', 'devicerev', validate_to_none), - ('extra.', 'firmwarerev', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'manufacturerid', validate_to_none), - ('extra.', 'manufacturername', validate_to_none), - ('extra.', 'productid', validate_to_none), - ('extra.', 'productname', validate_to_none), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipmi', - 'protocol.transport': 'udp', - 'classification.identifier': 'open-ipmi', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ipp-report/ -scan_ipp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'ipp_version', validate_to_none), - ('extra.', 'cups_version', validate_to_none), - ('extra.', 'printer_uris', validate_to_none), - ('extra.', 'printer_name', validate_to_none), - ('extra.', 'printer_info', validate_to_none), - ('extra.', 'printer_more_info', validate_to_none), - ('extra.', 'printer_make_and_model', validate_to_none), - ('extra.', 'printer_firmware_name', validate_to_none), - ('extra.', 'printer_firmware_string_version', validate_to_none), - ('extra.', 'printer_firmware_version', validate_to_none), - ('extra.', 'printer_organization', validate_to_none), - ('extra.', 'printer_organization_unit', validate_to_none), - ('extra.', 'printer_uuid', validate_to_none), - ('extra.', 'printer_wifi_ssid', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipp', - 'classification.identifier': 'open-ipp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-isakmp-report/ -scan_isakmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'spi_size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'initiator_spi', validate_to_none), - ('extra.', 'responder_spi', validate_to_none), - ('extra.', 'next_payload', validate_to_none), - ('extra.', 'exchange_type', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'message_id', validate_to_none), - ('extra.', 'next_payload2', validate_to_none), - ('extra.', 'domain_of_interpretation', validate_to_none), - ('extra.', 'protocol_id', validate_to_none), - ('extra.', 'notify_message_type', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'open-ike', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ipsec', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-kubernetes-api-server-report/ -scan_kubernetes = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'http', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'date', validate_to_none), - ('extra.', 'major', validate_to_none), - ('extra.', 'minor', validate_to_none), - ('extra.', 'git_version', validate_to_none), - ('extra.', 'git_commit', validate_to_none), - ('extra.', 'git_tree_state', validate_to_none), - ('extra.', 'build_date', validate_to_none), - ('extra.', 'go_version', validate_to_none), - ('extra.', 'compiler', validate_to_none), - ('extra.', 'platform', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'kubernetes', - 'classification.identifier': 'open-kubernetes', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ldap-tcp-report/ -scan_ldap_tcp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'dns_host_name', validate_to_none), - ('extra.', 'domain_controller_functionality', convert_int), - ('extra.', 'domain_functionality', convert_int), - ('extra.', 'forest_functionality', convert_int), - ('extra.', 'highest_committed_usn', convert_int), - ('extra.', 'is_global_catalog_ready', convert_bool), - ('extra.', 'is_synchronized', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'size', convert_int), - ('extra.', 'configuration_naming_context', validate_to_none), - ('extra.', 'current_time', validate_to_none), - ('extra.', 'default_naming_context', validate_to_none), - ('extra.', 'ds_service_name', validate_to_none), - ('extra.', 'ldap_service_name', validate_to_none), - ('extra.', 'naming_contexts', validate_to_none), - ('extra.', 'root_domain_naming_context', validate_to_none), - ('extra.', 'schema_naming_context', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'subschema_subentry', validate_to_none), - ('extra.', 'supported_capabilities', validate_to_none), - ('extra.', 'supported_control', validate_to_none), - ('extra.', 'supported_ldap_policies', validate_to_none), - ('extra.', 'supported_ldap_version', validate_to_none), - ('extra.', 'supported_sasl_mechanisms', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ldap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ldap-report/ -scan_ldap_udp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'dns_host_name', validate_to_none), - ('extra.', 'domain_controller_functionality', convert_int), - ('extra.', 'domain_functionality', convert_int), - ('extra.', 'forest_functionality', convert_int), - ('extra.', 'highest_committed_usn', convert_int), - ('extra.', 'is_global_catalog_ready', convert_bool), - ('extra.', 'is_synchronized', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'size', convert_int), - ('extra.', 'configuration_naming_context', validate_to_none), - ('extra.', 'current_time', validate_to_none), - ('extra.', 'default_naming_context', validate_to_none), - ('extra.', 'ds_service_name', validate_to_none), - ('extra.', 'ldap_service_name', validate_to_none), - ('extra.', 'naming_contexts', validate_to_none), - ('extra.', 'root_domain_naming_context', validate_to_none), - ('extra.', 'schema_naming_context', validate_to_none), - ('extra.', 'server_name', validate_to_none), - ('extra.', 'subschema_subentry', validate_to_none), - ('extra.', 'supported_capabilities', validate_to_none), - ('extra.', 'supported_control', validate_to_none), - ('extra.', 'supported_ldap_policies', validate_to_none), - ('extra.', 'supported_ldap_version', validate_to_none), - ('extra.', 'supported_sasl_mechanisms', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-ldap', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ldap', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mdns-report/ -scan_mdns = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'mdns_name', validate_to_none), - ('extra.', 'mdns_ipv4', validate_to_none), - ('extra.', 'mdns_ipv6', validate_to_none), - ('extra.', 'services', validate_to_none), - ('extra.', 'workstation_name', validate_to_none), - ('extra.', 'workstation_ipv4', validate_to_none), - ('extra.', 'workstation_ipv6', validate_to_none), - ('extra.', 'workstation_info', validate_to_none), - ('extra.', 'http_name', validate_to_none), - ('extra.', 'http_ipv4', validate_to_none), - ('extra.', 'http_ipv6', validate_to_none), - ('extra.', 'http_ptr', validate_to_none), - ('extra.', 'http_info', validate_to_none), - ('extra.', 'http_target', validate_to_none), - ('extra.', 'http_port', convert_int), - ('extra.', 'spotify_name', validate_to_none), - ('extra.', 'spotify_ipv4', validate_to_none), - ('extra.', 'spotify_ipv6', validate_to_none), - ('extra.', 'opc_ua_discovery', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mdns', - 'classification.identifier': 'open-mdns', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-memcached-report/ -scan_memcached = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'pid', convert_int), - ('extra.', 'pointer_size', convert_int), - ('extra.', 'uptime', convert_int), - ('extra.', 'curr_connections', convert_int), - ('extra.', 'total_connections', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'time', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'memcached', - 'classification.identifier': 'open-memcached', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mongodb-report/ -scan_mongodb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'gitversion', validate_to_none), - ('extra.', 'sysinfo', validate_to_none), - ('extra.', 'opensslversion', validate_to_none), - ('extra.', 'allocator', validate_to_none), - ('extra.', 'javascriptengine', validate_to_none), - ('extra.', 'bits', validate_to_none), - ('extra.', 'maxbsonobjectsize', validate_to_none), - ('extra.', 'ok', convert_bool), - ('extra.', 'visible_databases', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mongodb', - 'classification.identifier': 'open-mongodb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mqtt-report/ -scan_mqtt = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'anonymous_access', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'raw_response', validate_to_none), - ('extra.', 'hex_code', validate_to_none), - ('extra.', 'code', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serialNumber', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mqtt', - 'classification.identifier': 'open-mqtt', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-mqtt-report/ -scan_mqtt_anon = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'raw_response', validate_to_none), - ('extra.', 'hex_code', validate_to_none), - ('extra.', 'code', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serialNumber', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mqtt', - 'classification.identifier': 'open-mqtt-anon', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-MSSQL -scan_mssql = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.local_hostname', 'server_name', validate_to_none), - ('extra.', 'tcp_port', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'instance_name', validate_to_none), - ('extra.', 'named_pipe', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'mssql', - 'classification.identifier': 'open-mssql', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-mysql-server-report/ -scan_mysql = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'client_can_handle_expired_passwords', convert_bool), - ('extra.', 'client_compress', convert_bool), - ('extra.', 'client_connect_attrs', convert_bool), - ('extra.', 'client_connect_with_db', convert_bool), - ('extra.', 'client_deprecated_eof', convert_bool), - ('extra.', 'client_found_rows', convert_bool), - ('extra.', 'client_ignore_sigpipe', convert_bool), - ('extra.', 'client_ignore_space', convert_bool), - ('extra.', 'client_interactive', convert_bool), - ('extra.', 'client_local_files', convert_bool), - ('extra.', 'client_long_flag', convert_bool), - ('extra.', 'client_long_password', convert_bool), - ('extra.', 'client_multi_results', convert_bool), - ('extra.', 'client_multi_statements', convert_bool), - ('extra.', 'client_no_schema', convert_bool), - ('extra.', 'client_odbc', convert_bool), - ('extra.', 'client_plugin_auth', convert_bool), - ('extra.', 'client_plugin_auth_len_enc_client_data', convert_bool), - ('extra.', 'client_protocol_41', convert_bool), - ('extra.', 'client_ps_multi_results', convert_bool), - ('extra.', 'client_reserved', convert_bool), - ('extra.', 'client_secure_connection', convert_bool), - ('extra.', 'client_session_track', convert_bool), - ('extra.', 'client_ssl', convert_bool), - ('extra.', 'client_transactions', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'mysql_protocol_version', validate_to_none), - ('extra.', 'server_version', validate_to_none), - ('extra.', 'error_code', validate_to_none), - ('extra.', 'error_id', validate_to_none), - ('extra.', 'error_message', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'mysql', - 'classification.identifier': 'open-mysql', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-NATPMP -scan_nat_pmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'uptime', convert_int), - ('extra.', 'external_ip', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-natpmp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'natpmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-netbios-report/ -scan_netbios = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.account', 'username'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'mac_address', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'workgroup', validate_to_none), - ('extra.', 'machine_name', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-netbios-nameservice', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'netbios-nameservice', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/netcore-netis-router-vulnerability-scan-report/ -scan_netis_router = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'response', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'open-netis', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.transport': 'udp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ntp-version-report/ -scan_ntp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'clk_wander', convert_float), - ('extra.', 'frequency', convert_float), - ('extra.', 'jitter', convert_float), - ('extra.', 'leap', convert_float), - ('extra.', 'offset', convert_float), - ('extra.', 'peer', convert_int), - ('extra.', 'poll', convert_int), - ('extra.', 'precision', convert_int), - ('extra.', 'rootdelay', convert_float), - ('extra.', 'rootdispersion', convert_float), - ('extra.', 'stratum', convert_int), - ('extra.', 'tc', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'version', validate_to_none), - ('extra.', 'clock', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'mintc', validate_to_none), - ('extra.', 'noise', validate_to_none), - ('extra.', 'phase', validate_to_none), - ('extra.', 'processor', validate_to_none), - ('extra.', 'refid', validate_to_none), - ('extra.', 'reftime', validate_to_none), - ('extra.', 'stability', validate_to_none), - ('extra.', 'state', validate_to_none), - ('extra.', 'system', validate_to_none), - ('extra.', 'tai', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'ntp-version', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ntp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/ntp-monitor-report/ -scan_ntpmonitor = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'packets', convert_int), - ('extra.', 'size', convert_int), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'ntp-monitor', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ntp', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Portmapper -scan_portmapper = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'programs', validate_to_none), - ('extra.', 'mountd_port', validate_to_none), - ('extra.', 'exports', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'portmapper', - 'classification.identifier': 'open-portmapper', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-postgresql-server-report/ -scan_postgres = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'startup_error_line', convert_int), - ('extra.', 'client_ssl', convert_bool), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'supported_protocols', validate_to_none), - ('extra.', 'protocol_error_code', validate_to_none), - ('extra.', 'protocol_error_file', validate_to_none), - ('extra.', 'protocol_error_line', validate_to_none), - ('extra.', 'protocol_error_message', validate_to_none), - ('extra.', 'protocol_error_routine', validate_to_none), - ('extra.', 'protocol_error_severity', validate_to_none), - ('extra.', 'protocol_error_severity_v', validate_to_none), - ('extra.', 'startup_error_code', validate_to_none), - ('extra.', 'startup_error_file', validate_to_none), - ('extra.', 'startup_error_message', validate_to_none), - ('extra.', 'startup_error_routine', validate_to_none), - ('extra.', 'startup_error_severity', validate_to_none), - ('extra.', 'startup_error_severity_v', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'postgres', - 'classification.identifier': 'open-postgres', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-QOTD -scan_qotd = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'quote', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'qotd', - 'classification.identifier': 'open-qotd', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-quic-report/ -scan_quic = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'version_field_1', validate_to_none), - ('extra.', 'version_field_2', validate_to_none), - ('extra.', 'version_field_3', validate_to_none), - ('extra.', 'version_field_4', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'open-quic', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-radmin-report/ -scan_radmin = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-radmin', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-rdp-report/ -scan_rdp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'cve20190708_vulnerable', convert_bool), - ('extra.', 'bluekeep_vulnerable', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'rdp_protocol', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'rdp', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-rdp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ms-rdpeudp/ -scan_rdpeudp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sessionid', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-msrdpeudp', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-Redis -scan_redis = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'version', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'git_sha1', validate_to_none), - ('extra.', 'git_dirty_flag', validate_to_none), - ('extra.', 'build_id', validate_to_none), - ('extra.', 'mode', validate_to_none), - ('extra.os.name', 'os', validate_to_none), - ('extra.', 'architecture', validate_to_none), - ('extra.', 'multiplexing_api', validate_to_none), - ('extra.', 'gcc_version', validate_to_none), - ('extra.', 'process_id', validate_to_none), - ('extra.', 'run_id', validate_to_none), - ('extra.', 'uptime', convert_int), - ('extra.', 'connected_clients', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'redis', - 'classification.identifier': 'open-redis', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-rsync-report/ -scan_rsync = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'module', validate_to_none), - ('extra.', 'motd', validate_to_none), - ('extra.', 'has_password', validate_to_none), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-rsync', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'rsync', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-sip-report/ -scan_sip = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'sip', validate_to_none), - ('extra.', 'sip_code', validate_to_none), - ('extra.', 'sip_reason', validate_to_none), - ('user_agent', 'user_agent', validate_to_none), - ('extra.', 'sip_via', validate_to_none), - ('extra.', 'sip_to', validate_to_none), - ('extra.', 'sip_from', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'content_type', validate_to_none), - ('extra.sip_server', 'server', validate_to_none), - ('extra.sip_contact', 'contact', validate_to_none), - ('extra.sip_cseq', 'cseq', validate_to_none), - ('extra.sip_call_id', 'call_id', validate_to_none), - ('extra.sip_allow', 'allow', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'sip', - 'classification.identifier': 'open-sip', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-slp-service-report/ -scan_slp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'function', validate_to_none), - ('extra.', 'function_text', validate_to_none), - ('extra.', 'flags', validate_to_none), - ('extra.', 'next_extension_offset', validate_to_none), - ('extra.', 'xid', validate_to_none), - ('extra.', 'language_tag_length', validate_to_none), - ('extra.', 'language_tag', validate_to_none), - ('extra.', 'error_code', validate_to_none), - ('extra.', 'error_code_text', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'slp', - 'classification.identifier': 'open-slp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-smb-report/ -scan_smb = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'smb_implant', convert_bool), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'arch', validate_to_none), - ('extra.', 'key', validate_to_none), - ('extra.', 'smbv1_support', validate_to_none), - ('extra.', 'smb_major_number', validate_to_none), - ('extra.', 'smb_minor_number', validate_to_none), - ('extra.', 'smb_revision', validate_to_none), - ('extra.', 'smb_version_string', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'smb', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-smb', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-smtp-report/ -scan_smtp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'smtp', - 'classification.identifier': 'open-smtp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/vulnerable-smtp-report/ -scan_smtp_vulnerable = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'smtp', - 'classification.identifier': 'vulnerable-smtp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-snmp-report/ -scan_snmp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'sysdesc', validate_to_none), - ('extra.', 'sysname', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'version', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'tag', validate_to_none), - ('extra.', 'community', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'snmp', - 'classification.identifier': 'open-snmp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-socks4-5-proxy-report/ -scan_socks = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.application', 'tag'), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-socks', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-SSDP -scan_ssdp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'header', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'systime', validate_to_none), - ('extra.', 'cache_control', validate_to_none), - ('extra.', 'location', validate_to_none), - ('extra.', 'server', validate_to_none), - ('extra.', 'search_target', validate_to_none), - ('extra.', 'unique_service_name', validate_to_none), - ('extra.', 'host', validate_to_none), - ('extra.', 'nts', validate_to_none), - ('extra.', 'nt', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'server_port', validate_to_none), - ('extra.', 'instance', validate_to_none), - ('extra.', 'version', validate_to_none), - ('extra.', 'updated_at', validate_to_none), - ('extra.', 'resource_identifier', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ssdp', - 'classification.identifier': 'open-ssdp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ssh-report/ -scan_ssh = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'serverid_raw', validate_to_none), - ('extra.', 'serverid_version', validate_to_none), - ('extra.', 'serverid_software', validate_to_none), - ('extra.', 'serverid_comment', validate_to_none), - ('extra.', 'server_cookie', validate_to_none), - ('extra.', 'available_kex', validate_to_none), - ('extra.', 'available_ciphers', validate_to_none), - ('extra.', 'available_mac', validate_to_none), - ('extra.', 'available_compression', validate_to_none), - ('extra.', 'selected_kex', validate_to_none), - ('extra.', 'algorithm', validate_to_none), - ('extra.', 'selected_cipher', validate_to_none), - ('extra.', 'selected_mac', validate_to_none), - ('extra.', 'selected_compression', validate_to_none), - ('extra.', 'server_signature_value', validate_to_none), - ('extra.', 'server_signature_raw', validate_to_none), - ('extra.', 'server_host_key', validate_to_none), - ('extra.', 'server_host_key_sha256', validate_to_none), - ('extra.', 'rsa_prime', validate_to_none), - ('extra.', 'rsa_prime_length', validate_to_none), - ('extra.', 'rsa_generator', validate_to_none), - ('extra.', 'rsa_generator_length', validate_to_none), - ('extra.', 'rsa_public_key', validate_to_none), - ('extra.', 'rsa_public_key_length', validate_to_none), - ('extra.', 'rsa_exponent', validate_to_none), - ('extra.', 'rsa_modulus', validate_to_none), - ('extra.', 'rsa_length', validate_to_none), - ('extra.', 'dss_prime', validate_to_none), - ('extra.', 'dss_prime_length', validate_to_none), - ('extra.', 'dss_generator', validate_to_none), - ('extra.', 'dss_generator_length', validate_to_none), - ('extra.', 'dss_public_key', validate_to_none), - ('extra.', 'dss_public_key_length', validate_to_none), - ('extra.', 'dss_dsa_public_g', validate_to_none), - ('extra.', 'dss_dsa_public_p', validate_to_none), - ('extra.', 'dss_dsa_public_q', validate_to_none), - ('extra.', 'dss_dsa_public_y', validate_to_none), - ('extra.', 'ecdsa_curve25519', validate_to_none), - ('extra.', 'ecdsa_curve', validate_to_none), - ('extra.', 'ecdsa_public_key_length', validate_to_none), - ('extra.', 'ecdsa_public_key_b', validate_to_none), - ('extra.', 'ecdsa_public_key_gx', validate_to_none), - ('extra.', 'ecdsa_public_key_gy', validate_to_none), - ('extra.', 'ecdsa_public_key_n', validate_to_none), - ('extra.', 'ecdsa_public_key_p', validate_to_none), - ('extra.', 'ecdsa_public_key_x', validate_to_none), - ('extra.', 'ecdsa_public_key_y', validate_to_none), - ('extra.', 'ed25519_curve25519', validate_to_none), - ('extra.', 'ed25519_cert_public_key_nonce', validate_to_none), - ('extra.', 'ed25519_cert_public_key_bytes', validate_to_none), - ('extra.', 'ed25519_cert_public_key_raw', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sha256', validate_to_none), - ('extra.', 'ed25519_cert_public_key_serial', validate_to_none), - ('extra.', 'ed25519_cert_public_key_type_id', validate_to_none), - ('extra.', 'ed25519_cert_public_key_type_name', validate_to_none), - ('extra.', 'ed25519_cert_public_key_keyid', validate_to_none), - ('extra.', 'ed25519_cert_public_key_principles', validate_to_none), - ('extra.', 'ed25519_cert_public_key_valid_after', validate_to_none), - ('extra.', 'ed25519_cert_public_key_valid_before', validate_to_none), - ('extra.', 'ed25519_cert_public_key_duration', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_bytes', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_raw', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_sha256', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sigkey_value', validate_to_none), - ('extra.', 'ed25519_cert_public_key_sig_raw', validate_to_none), - ('extra.', 'banner', validate_to_none), - ('extra.', 'userauth_methods', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'classification.identifier': 'open-ssh', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ssl-report/ -scan_ssl = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'ssl_poodle', convert_bool), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'freak_vulnerable', convert_bool), - ('extra.', 'freak_cipher_suite', validate_to_none), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'https', - 'classification.identifier': 'open-ssl', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Freak-Scan -scan_ssl_freak = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'freak_vulnerable', convert_bool), - ('extra.', 'freak_cipher_suite', validate_to_none), - ('extra.', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'page_sha256fp', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'https', - 'classification.identifier': 'ssl-freak', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Ssl-Scan -scan_ssl_poodle = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('extra.', 'handshake', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'cipher_suite', validate_to_none), - ('extra.', 'ssl_poodle', convert_bool), - ('extra.', 'cert_length', convert_int), - ('extra.', 'subject_common_name', validate_to_none), - ('extra.', 'issuer_common_name', validate_to_none), - ('extra.', 'cert_issue_date', validate_to_none), - ('extra.', 'cert_expiration_date', validate_to_none), - ('extra.', 'sha1_fingerprint', validate_to_none), - ('extra.', 'cert_serial_number', validate_to_none), - ('extra.', 'ssl_version', convert_int), - ('extra.', 'signature_algorithm', validate_to_none), - ('extra.', 'key_algorithm', validate_to_none), - ('extra.', 'subject_organization_name', validate_to_none), - ('extra.', 'subject_organization_unit_name', validate_to_none), - ('extra.', 'subject_country', validate_to_none), - ('extra.', 'subject_state_or_province_name', validate_to_none), - ('extra.', 'subject_locality_name', validate_to_none), - ('extra.', 'subject_street_address', validate_to_none), - ('extra.', 'subject_postal_code', validate_to_none), - ('extra.', 'subject_surname', validate_to_none), - ('extra.', 'subject_given_name', validate_to_none), - ('extra.', 'subject_email_address', validate_to_none), - ('extra.', 'subject_business_category', validate_to_none), - ('extra.', 'subject_serial_number', validate_to_none), - ('extra.', 'issuer_organization_name', validate_to_none), - ('extra.', 'issuer_organization_unit_name', validate_to_none), - ('extra.', 'issuer_country', validate_to_none), - ('extra.', 'issuer_state_or_province_name', validate_to_none), - ('extra.', 'issuer_locality_name', validate_to_none), - ('extra.', 'issuer_street_address', validate_to_none), - ('extra.', 'issuer_postal_code', validate_to_none), - ('extra.', 'issuer_surname', validate_to_none), - ('extra.', 'issuer_given_name', validate_to_none), - ('extra.', 'issuer_email_address', validate_to_none), - ('extra.', 'issuer_business_category', validate_to_none), - ('extra.', 'issuer_serial_number', validate_to_none), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'sha256_fingerprint', validate_to_none), - ('extra.', 'sha512_fingerprint', validate_to_none), - ('extra.', 'md5_fingerprint', validate_to_none), - ('extra.', 'http_response_type', validate_to_none), - ('extra.', 'http_code', convert_int), - ('extra.', 'http_reason', validate_to_none), - ('extra.', 'content_type', validate_to_none), - ('extra.', 'http_connection', validate_to_none), - ('extra.', 'www_authenticate', validate_to_none), - ('extra.', 'set_cookie', validate_to_none), - ('extra.', 'server_type', validate_to_none), - ('extra.', 'content_length', convert_int), - ('extra.', 'transfer_encoding', validate_to_none), - ('extra.', 'http_date', convert_date), - ('extra.', 'cert_valid', convert_bool), - ('extra.', 'self_signed', convert_bool), - ('extra.', 'cert_expired', convert_bool), - ('extra.', 'browser_trusted', convert_bool), - ('extra.', 'validation_level', validate_to_none), - ('extra.', 'browser_error', validate_to_none), - ('extra.', 'tlsv13_support', validate_to_none), - ('extra.', 'tlsv13_cipher', validate_to_none), - ('extra.', 'raw_cert', validate_to_none), - ('extra.', 'raw_cert_chain', validate_to_none), - ('extra.', 'jarm', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ('extra.', 'device_type', validate_to_none), - ('extra.', 'device_model', validate_to_none), - ('extra.', 'device_version', validate_to_none), - ('extra.', 'device_sector', validate_to_none), - ('extra.', 'page_sha256fp', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'https', - 'classification.identifier': 'ssl-poodle', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-stun-service-report/ -scan_stun = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'mapped_port', convert_int), - ('extra.', 'xor_mapped_port', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'transaction_id', validate_to_none), - ('extra.', 'magic_cookie', validate_to_none), - ('extra.', 'message_length', convert_int), - ('extra.', 'message_type', validate_to_none), - ('extra.', 'mapped_family', validate_to_none), - ('extra.', 'mapped_address', validate_to_none), - ('extra.', 'xor_mapped_family', validate_to_none), - ('extra.', 'xor_mapped_address', validate_to_none), - ('extra.', 'software', validate_to_none), - ('extra.', 'fingerprint', validate_to_none), - ('extra.', 'amplification', convert_float), - ('extra.', 'response_size', convert_int), - ], - 'constant_fields': { - 'classification.taxonomy': 'other', - 'classification.type': 'other', - 'protocol.application': 'Session Traversal Utilities for NAT', - 'classification.identifier': 'open-stun', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/synful-scan-report/ -scan_synfulknock = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.', 'ack_number', convert_int), - ('extra.', 'window_size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.', 'sequence_number', validate_to_none), - ('extra.', 'urgent_pointer', validate_to_none), - ('extra.', 'tcp_flags', validate_to_none), - ('extra.', 'raw_packet', validate_to_none), - ('extra.source.sector', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'open-synfulknock', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-telnet-report/ -scan_telnet = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'banner', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'telnet', - 'classification.identifier': 'open-telnet', - }, -} - -# https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-TFTP -scan_tftp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'errorcode', validate_to_none), - ('extra.', 'error', validate_to_none), - ('extra.', 'errormessage', validate_to_none), - ('extra.', 'size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'tftp', - 'classification.identifier': 'open-tftp', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/open-ubiquiti-report/ -scan_ubiquiti = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('extra.mac_address', 'mac', validate_to_none), - ('extra.radio_name', 'radioname', validate_to_none), - ('extra.model', 'modelshort', validate_to_none), - ('extra.model_full', 'modelfull', validate_to_none), - ('extra.firmwarerev', 'firmware', validate_to_none), - ('extra.response_size', 'size', convert_int), - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'essid', validate_to_none), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.identifier': 'accessible-ubiquiti-discovery-service', - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-vnc-report/ -scan_vnc = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('source.reverse_dns', 'hostname'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'product', validate_to_none), - ('extra.', 'banner', validate_to_none), - ('extra.', 'sector', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'vnc', - 'protocol.transport': 'tcp', - 'classification.identifier': 'open-vnc', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-ws-discovery-service-report/ -scan_ws_discovery = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sic', 'sic', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('extra.', 'response_size', convert_int), - ('extra.', 'amplification', convert_float), - ('extra.', 'error', validate_to_none), - ('extra.', 'raw_response', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'ws-discovery', - 'classification.identifier': 'open-ws-discovery', - }, -} - -# https://www.shadowserver.org/what-we-do/network-reporting/accessible-xdmcp-service-report/ -scan_xdmcp = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.reverse_dns', 'hostname'), - ('extra.', 'tag', validate_to_none), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sic', invalidate_zero), - ('extra.', 'opcode', validate_to_none), - ('extra.', 'reported_hostname', validate_to_none), - ('extra.', 'status', validate_to_none), - ('extra.', 'size', convert_int), - ('extra.', 'amplification', convert_float), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'protocol.application': 'xdmcp', - 'classification.identifier': 'open-xdmcp', - }, -} - -# http://www.shadowserver.org/wiki/pmwiki.php/Services/Spam-URL -spam_url = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ], - 'optional_fields': [ - ('source.url', 'url', convert_http_host_and_url, True), - ('source.fqdn', 'host', validate_fqdn), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('extra.', 'naics', invalidate_zero), - ('extra.', 'sector', validate_to_none), - ('extra.', 'source', validate_to_none), - ('extra.', 'sender', validate_to_none), - ('extra.', 'subject', validate_to_none), - ('malware.hash.md5', 'md5', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'abusive-content', - 'classification.type': 'spam', - 'classification.identifier': 'spam-url', - }, -} - -special = { - 'required_fields': [ - ('time.source', 'timestamp', add_UTC_to_timestamp), - ('source.ip', 'ip', validate_ip), - ('source.port', 'port', convert_int), - ], - 'optional_fields': [ - ('protocol.transport', 'protocol'), - ('source.asn', 'asn', invalidate_zero), - ('source.geolocation.cc', 'geo'), - ('source.geolocation.region', 'region'), - ('source.geolocation.city', 'city'), - ('source.reverse_dns', 'hostname'), - ('extra.source.naics', 'naics', invalidate_zero), - ('extra.source.sector', 'sector', validate_to_none), - ('malware.name', 'tag'), - ('extra.', 'public_source', validate_to_none), - ('extra.', 'status', validate_to_none), - ('extra.', 'detail', validate_to_none), - ('extra.', 'method', validate_to_none), - ('extra.', 'device_vendor', validate_to_none), - ], - 'constant_fields': { - 'classification.taxonomy': 'vulnerable', - 'classification.type': 'vulnerable-system', - 'classification.identifier': 'special', - }, -} - -mapping = ( - # feed name, file name, function - ('Blocklist', 'blocklist', blocklist), - ('Compromised-Website', 'compromised_website', compromised_website), - ('Device-Identification IPv4', 'device_id', device_id), - ('Device-Identification IPv6', 'device_id6', device_id), - ('DDoS-Participant', 'event4_ddos_participant', event_ddos_participant), - ('Honeypot-Brute-Force-Events', 'event4_honeypot_brute_force', event_honeypot_brute_force), - ('Honeypot-Darknet', 'event4_honeypot_darknet', event_honeypot_darknet), - ('Honeypot-DDoS', 'event4_honeypot_ddos', event_honeypot_ddos), - ('Honeypot-Amplification-DDoS-Events', 'event4_honeypot_ddos_amp', event_honeypot_ddos_amp), - ('Honeypot-DDoS-Target', 'event4_honeypot_ddos_target', event_honeypot_ddos_target), - ('Honeypot-HTTP-Scan', 'event4_honeypot_http_scan', event_honeypot_http_scan), - ('Honeypot-ICS-Scanner', 'event4_honeypot_ics_scan', event_honeypot_ics_scan), - ('IP-Spoofer-Events', 'event4_ip_spoofer', event_ip_spoofer), - ('Microsoft-Sinkhole-Events IPv4', 'event4_microsoft_sinkhole', event_sinkhole), - ('Microsoft-Sinkhole-Events-HTTP IPv4', 'event4_microsoft_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events IPv4', 'event4_sinkhole', event_sinkhole), - ('Sinkhole-DNS', 'event4_sinkhole_dns', event_sinkhole_dns), - ('Sinkhole-Events-HTTP IPv4', 'event4_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer IPv4', 'event4_sinkhole_http_referer', event_sinkhole_http_referer), - ('Sinkhole-Events IPv6', 'event6_sinkhole', event_sinkhole), - ('Sinkhole-Events-HTTP IPv6', 'event6_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer IPv6', 'event6_sinkhole_http_referer', event_sinkhole_http_referer), - ('Malware-URL', 'malware_url', malware_url), - ('Phish-URL', 'phish_url', phish_url), - ('IPv6-Accessible-HTTP-Proxy', 'population6_http_proxy', population_http_proxy), - ('Accessible-HTTP-Proxy', 'population_http_proxy', population_http_proxy), - ('Sandbox-Connections', 'sandbox_conn', sandbox_conn), - ('Sandbox-DNS', 'sandbox_dns', sandbox_dns), - ('Sandbox-URL', 'sandbox_url', sandbox_url), - ('IPv6-Accessible-CWMP', 'scan6_cwmp', scan_cwmp), - ('IPv6-DNS-Open-Resolvers', 'scan6_dns', scan_dns), - ('IPv6-Vulnerable-Exchange', 'scan6_exchange', scan_exchange), - ('IPv6-Accessible-FTP', 'scan6_ftp', scan_ftp), - ('IPv6-Accessible-HTTP', 'scan6_http', scan_http), - ('IPv6-Open-HTTP-Proxy', 'scan6_http_proxy', scan_http_proxy), - ('IPv6-Vulnerable-HTTP', 'scan6_http_vulnerable', scan_http_vulnerable), - ('IPv6-Open-IPP', 'scan6_ipp', scan_ipp), - ('IPv6-Open-LDAP-TCP', 'scan6_ldap_tcp', scan_ldap_tcp), - ('IPv6-Open-MQTT', 'scan6_mqtt', scan_mqtt), - ('IPv6-Open-Anonymous-MQTT', 'scan6_mqtt_anon', scan_mqtt_anon), - ('IPv6-Accessible-MySQL', 'scan6_mysql', scan_mysql), - ('IPv6-NTP-Version', 'scan6_ntp', scan_ntp), - ('IPv6-NTP-Monitor', 'scan6_ntpmonitor', scan_ntpmonitor), - ('IPv6-Accessible-PostgreSQL', 'scan6_postgres', scan_postgres), - ('IPv6-Accessible-RDP', 'scan6_rdp', scan_rdp), - ('IPv6-Accessible-SLP', 'scan6_slp', scan_slp), - ('IPv6-Accessible-SMB', 'scan6_smb', scan_smb), - ('IPv6-Accessible-SMTP', 'scan6_smtp', scan_smtp), - ('IPv6-Vulnerable-SMTP', 'scan6_smtp_vulnerable', scan_smtp_vulnerable), - ('IPv6-Open-SNMP', 'scan6_snmp', scan_snmp), - ('IPv6-Accessible-SSH', 'scan6_ssh', scan_ssh), - ('IPv6-Accessible-SSL', 'scan6_ssl', scan_ssl), - ('SSL-FREAK-Vulnerable-Servers IPv6', 'scan6_ssl_freak', scan_ssl_freak), - ('SSL-POODLE-Vulnerable-Servers IPv6', 'scan6_ssl_poodle', scan_ssl_poodle), - ('IPv6-Accessible-Session-Traversal-Utilities-for-NAT', 'scan6_stun', scan_stun), - ('IPv6-Accessible-Telnet', 'scan6_telnet', scan_telnet), - ('IPv6-Accessible-VNC', 'scan6_vnc', scan_vnc), - ('Accessible-ADB', 'scan_adb', scan_adb), - ('Accessible-AFP', 'scan_afp', scan_afp), - ('Accessible-AMQP', 'scan_amqp', scan_amqp), - ('Accessible-ARD', 'scan_ard', scan_ard), - ('Open-Chargen', 'scan_chargen', scan_chargen), - ('Accessible-Cisco-Smart-Install', 'scan_cisco_smart_install', scan_cisco_smart_install), - ('Accessible-CoAP', 'scan_coap', scan_coap), - ('Accessible-CouchDB', 'scan_couchdb', scan_couchdb), - ('Accessible-CWMP', 'scan_cwmp', scan_cwmp), - ('Open-DB2-Discovery-Service', 'scan_db2', scan_db2), - ('Vulnerable-DDoS-Middlebox', 'scan_ddos_middlebox', scan_ddos_middlebox), - ('DNS-Open-Resolvers', 'scan_dns', scan_dns), - ('Accessible-Docker', 'scan_docker', scan_docker), - ('Accessible-DVR-DHCPDiscover', 'scan_dvr_dhcpdiscover', scan_dvr_dhcpdiscover), - ('Open-Elasticsearch', 'scan_elasticsearch', scan_elasticsearch), - ('Accessible-Erlang-Port-Mapper-Daemon', 'scan_epmd', scan_epmd), - ('Vulnerable-Exchange-Server', 'scan_exchange', scan_exchange), - ('Accessible-FTP', 'scan_ftp', scan_ftp), - ('Accessible-Hadoop', 'scan_hadoop', scan_hadoop), - ('Accessible-HTTP', 'scan_http', scan_http), - ('Open-HTTP-Proxy', 'scan_http_proxy', scan_http_proxy), - ('Vulnerable-HTTP', 'scan_http_vulnerable', scan_http_vulnerable), - ('Accessible-ICS', 'scan_ics', scan_ics), - ('Open-IPMI', 'scan_ipmi', scan_ipmi), - ('Open-IPP', 'scan_ipp', scan_ipp), - ('Vulnerable-ISAKMP', 'scan_isakmp', scan_isakmp), - ('Accessible-Kubernetes-API', 'scan_kubernetes', scan_kubernetes), - ('Open-LDAP-TCP', 'scan_ldap_tcp', scan_ldap_tcp), - ('Open-LDAP', 'scan_ldap_udp', scan_ldap_udp), - ('Open-mDNS', 'scan_mdns', scan_mdns), - ('Open-Memcached', 'scan_memcached', scan_memcached), - ('Open-MongoDB', 'scan_mongodb', scan_mongodb), - ('Open-MQTT', 'scan_mqtt', scan_mqtt), - ('Open-Anonymous-MQTT', 'scan_mqtt_anon', scan_mqtt_anon), - ('Open-MSSQL', 'scan_mssql', scan_mssql), - ('Accessible-MySQL', 'scan_mysql', scan_mysql), - ('Open-NATPMP', 'scan_nat_pmp', scan_nat_pmp), - ('Open-NetBIOS-Nameservice', 'scan_netbios', scan_netbios), - ('Open-Netis', 'scan_netis_router', scan_netis_router), - ('NTP-Version', 'scan_ntp', scan_ntp), - ('NTP-Monitor', 'scan_ntpmonitor', scan_ntpmonitor), - ('Open-Portmapper', 'scan_portmapper', scan_portmapper), - ('Accessible-PostgreSQL', 'scan_postgres', scan_postgres), - ('Open-QOTD', 'scan_qotd', scan_qotd), - ('Accessible-QUIC', 'scan_quic', scan_quic), - ('Accessible-Radmin', 'scan_radmin', scan_radmin), - ('Accessible-RDP', 'scan_rdp', scan_rdp), - ('Accessible-MS-RDPEUDP', 'scan_rdpeudp', scan_rdpeudp), - ('Open-Redis', 'scan_redis', scan_redis), - ('Accessible-Rsync', 'scan_rsync', scan_rsync), - ('Accessible-SIP', 'scan_sip', scan_sip), - ('Accessible-SLP', 'scan_slp', scan_slp), - ('Accessible-SMB', 'scan_smb', scan_smb), - ('Accessible-SMTP', 'scan_smtp', scan_smtp), - ('Vulnerable-SMTP', 'scan_smtp_vulnerable', scan_smtp_vulnerable), - ('Open-SNMP', 'scan_snmp', scan_snmp), - ('Accessible-SOCKS4/5-Proxy', 'scan_socks', scan_socks), - ('Open-SSDP', 'scan_ssdp', scan_ssdp), - ('Accessible-SSH', 'scan_ssh', scan_ssh), - ('Accessible-SSL', 'scan_ssl', scan_ssl), - ('SSL-FREAK-Vulnerable-Servers', 'scan_ssl_freak', scan_ssl_freak), - ('SSL-POODLE-Vulnerable-Servers IPv4', 'scan_ssl_poodle', scan_ssl_poodle), - ('Accessible-Session-Traversal-Utilities-for-NAT', 'scan_stun', scan_stun), - ('SYNful-Knock', 'scan_synfulknock', scan_synfulknock), - ('Accessible-Telnet', 'scan_telnet', scan_telnet), - ('Open-TFTP', 'scan_tftp', scan_tftp), - ('Accessible-Ubiquiti-Discovery-Service', 'scan_ubiquiti', scan_ubiquiti), - ('Accessible-VNC', 'scan_vnc', scan_vnc), - ('Accessible-WS-Discovery-Service', 'scan_ws_discovery', scan_ws_discovery), - ('Open-XDMCP', 'scan_xdmcp', scan_xdmcp), - ('Spam-URL', 'spam_url', spam_url), - ('Special', 'special', special), - ('Accessible-RDPEUDP', 'scan_rdpeudp', scan_rdpeudp), - ('Sinkhole-Events', 'event4_sinkhole', event_sinkhole), - ('Sinkhole-Events-HTTP', 'event4_sinkhole_http', event_sinkhole_http), - ('Sinkhole-Events-HTTP-Referer', 'event4_sinkhole_http_referer', event_sinkhole_http_referer), -) -# END CONFGEN + try: + with open(tmp) as fh: + schema = json.load(fh) + except: + # leave tempfile behind for diagnosis + raise ValueError("Failed to validate %r" % tmp) -feedname_mapping = {feedname: function for feedname, filename, function in mapping} -filename_mapping = {filename: (feedname, function) for feedname, filename, function in mapping} + os.replace(tmp, __config.schema_file) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 70ba3b4bb..f14549141 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -22,6 +22,7 @@ """ import copy import re +import os from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue @@ -29,7 +30,13 @@ class ShadowserverParserBot(ParserBot): - """Parse all ShadowServer feeds""" + """ + Parse all ShadowServer feeds + + Parameters: + schema_file (str): Path to the report schema file + + """ recover_line = ParserBot.recover_line_csv_dict _csv_params = {'dialect': 'unix'} @@ -124,10 +131,17 @@ def parse_line(self, row, report): value = raw_value if conv_func is not None and raw_value is not None: - if len(item) == 4 and item[3]: - value = conv_func(raw_value, row) - else: - value = conv_func(raw_value) + try: + if len(item) == 4 and item[3]: + value = config.functions[conv_func](raw_value, row) + else: + value = config.functions[conv_func](raw_value) + except Exception: + """ fail early and often in this case. We want to be able to convert everything """ + self.logger.error('Could not convert shadowkey: %r in feed %r, ' + 'value: %r via conversion function %r.', + shadowkey, self.feedname, raw_value, conv_func) + raise if value is not None: event.add(intelmqkey, value) @@ -153,17 +167,17 @@ def parse_line(self, row, report): value = raw_value if conv_func is not None and raw_value is not None: - if len(item) == 4 and item[3]: - value = conv_func(raw_value, row) - else: - try: - value = conv_func(raw_value) - except Exception: - """ fail early and often in this case. We want to be able to convert everything """ - self.logger.error('Could not convert shadowkey: %r in feed %r, ' - 'value: %r via conversion function %r.', - shadowkey, self.feedname, raw_value, conv_func.__name__) - raise + try: + if len(item) == 4 and item[3]: + value = config.functions[conv_func](raw_value, row) + else: + value = config.functions[conv_func](raw_value) + except Exception: + """ fail early and often in this case. We want to be able to convert everything """ + self.logger.error('Could not convert shadowkey: %r in feed %r, ' + 'value: %r via conversion function %r.', + shadowkey, self.feedname, raw_value, conv_func) + raise if value is not None: if intelmqkey == 'extra.': diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test b/intelmq/bots/parsers/shadowserver/schema.json.test new file mode 100644 index 000000000..2cfb8bb1d --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/schema.json.test @@ -0,0 +1,180 @@ +{ + "test_smb" : { + "constant_fields" : { + "classification.identifier" : "test-smb", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "smb", + "protocol.transport" : "tcp" + }, + "feed_name" : "Test-Accessible-SMB", + "file_name" : "test_smb", + "optional_fields" : [ + [ + "extra.", + "smb_implant", + "convert_bool" + ], + [ + "source.reverse_dns", + "hostname" + ], + [ + "extra.", + "tag" + ], + [ + "source.asn", + "asn", + "invalidate_zero" + ], + [ + "source.geolocation.cc", + "geo" + ], + [ + "source.geolocation.region", + "region" + ], + [ + "source.geolocation.city", + "city" + ], + [ + "extra.source.naics", + "naics", + "invalidate_zero" + ], + [ + "extra.source.sic", + "sic", + "invalidate_zero" + ], + [ + "extra.", + "arch", + "validate_to_none" + ], + [ + "extra.", + "key", + "validate_to_none" + ], + [ + "extra.", + "smbv1_support", + "validate_to_none" + ], + [ + "extra.", + "smb_major_number", + "validate_to_none" + ], + [ + "extra.", + "smb_minor_number", + "validate_to_none" + ], + [ + "extra.", + "smb_revision", + "validate_to_none" + ], + [ + "extra.", + "smb_version_string", + "validate_to_none" + ] + ], + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ] + ] + }, + "test_telnet" : { + "constant_fields" : { + "classification.identifier" : "test-telnet", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "telnet" + }, + "feed_name" : "Test-Accessible-Telnet", + "file_name" : "test_telnet", + "optional_fields" : [ + [ + "protocol.transport", + "protocol" + ], + [ + "source.reverse_dns", + "hostname" + ], + [ + "extra.", + "tag", + "validate_to_none" + ], + [ + "source.asn", + "asn", + "invalidate_zero" + ], + [ + "source.geolocation.cc", + "geo" + ], + [ + "source.geolocation.region", + "region" + ], + [ + "source.geolocation.city", + "city" + ], + [ + "extra.", + "naics", + "invalidate_zero" + ], + [ "extra.", + "sic", + "invalidate_zero" + ], + [ + "extra.", + "banner", + "validate_to_none" + ] + ], + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ] + ] + } +} diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py new file mode 100644 index 000000000..040f67259 --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/update_schema.py @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import intelmq.bots.parsers.shadowserver._config as config + +if __name__ == '__main__': # pragma: no cover + exec(open(os.path.join(os.path.dirname(__file__), '../../../version.py')).read()) # defines __version__ + config.update_schema(__version__) From b081509850f40f8626379ca2100f495dbfd52b96 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Wed, 12 Apr 2023 00:01:32 +0000 Subject: [PATCH 45/67] revised tests --- .../bots/parsers/shadowserver/test_broken.py | 12 +- .../bots/parsers/shadowserver/test_mapping.py | 8 +- .../parsers/shadowserver/test_parameters.py | 37 +++--- .../parsers/shadowserver/test_report_smb.py | 124 ++++++++++++++++++ .../shadowserver/test_report_switch.py | 16 +-- .../shadowserver/test_report_telnet.py | 87 ++++++++++++ .../shadowserver/testdata/test_smb.csv | 4 + .../testdata/test_smb.csv.license | 2 + .../shadowserver/testdata/test_telnet.csv | 3 + .../testdata/test_telnet.csv.license | 2 + 10 files changed, 260 insertions(+), 35 deletions(-) create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_report_smb.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv create mode 100644 intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 472dd0b90..2b803142e 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -13,12 +13,12 @@ REPORT1 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_http-test-test.csv", + "extra.file_name": "2019-01-01-test_smb-test-test.csv", } REPORT2 = {"raw": utils.base64_encode('timestamp,ip,port\n2018-08-01T00:00:00+00,127.0.0.1,80'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_ftp-test-test.csv", + "extra.file_name": "2019-01-01-test_telnet-test-test.csv", } REPORT3 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", @@ -48,10 +48,10 @@ def test_broken(self): """ self.input_message = REPORT1 self.run_bot(allowed_error_count=1) - self.assertLogMatches(pattern="Detected report's file name: 'scan_http'.", + self.assertLogMatches(pattern="Detected report's file name: 'test_smb'.", levelname="DEBUG") self.assertLogMatches(pattern="Failed to parse line.") - self.assertLogMatches(pattern="ValueError: Required column 'timestamp' not found in feed 'Accessible-HTTP'. Possible change in data format or misconfiguration.") + self.assertLogMatches(pattern="ValueError: Required column 'timestamp' not found in feed 'Test-Accessible-SMB'. Possible change in data format or misconfiguration.") self.assertLogMatches(pattern=r"Sent 0 events and found 1 problem\(s\)\.", levelname="INFO") @@ -61,9 +61,9 @@ def test_half_broken(self): """ self.input_message = REPORT2 self.run_bot(allowed_warning_count=63) - self.assertLogMatches(pattern="Detected report's file name: 'scan_ftp'.", + self.assertLogMatches(pattern="Detected report's file name: 'test_telnet'.", levelname="DEBUG") - self.assertLogMatches(pattern="Optional key 'jarm' not found in feed 'Accessible-FTP'.", + self.assertLogMatches(pattern="Optional key 'banner' not found in feed 'Test-Accessible-Telnet'.", levelname="WARNING") self.assertLogMatches(pattern=r"Sent 1 events and found 0 problem\(s\)\.", levelname="INFO") diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index f58aed66e..6a2af9447 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -11,22 +11,22 @@ with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_telnet.csv')) as handle: + 'testdata/test_telnet.csv')) as handle: TELNET_FILE = handle.read() EXAMPLE_TELNET = { "raw": utils.base64_encode(TELNET_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_telnet.csv", + "extra.file_name": "2019-01-01-test_telnet.csv", } with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_vnc.csv')) as handle: + 'testdata/test_smb.csv')) as handle: TELNET_FILE = handle.read() EXAMPLE_VNC = { "raw": utils.base64_encode(TELNET_FILE), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_vnc.csv", + "extra.file_name": "2019-01-01-test_smb.csv", } diff --git a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py index a5ea81f19..677cd0319 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py @@ -12,38 +12,41 @@ from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_dns.csv')) as handle: + 'testdata/test_smb.csv')) as handle: EXAMPLE_FILE = handle.read() EXAMPLE_LINES = EXAMPLE_FILE.splitlines() EXAMPLE_REPORT = {"raw": utils.base64_encode(EXAMPLE_FILE), "__type": "Report", "time.observation": "2018-07-30T00:00:00+00:00", - "extra.file_name": "2019-01-01-scan_dns-test-test.csv", + "extra.file_name": "2019-01-01-test_smb-test-test.csv", 'feed.name': 'report feedname', } EVENTS = [{ '__type': 'Event', 'feed.name': 'report feedname', - "classification.identifier": "dns-open-resolver", + "classification.identifier": 'test-smb', "classification.taxonomy": "vulnerable", "classification.type": "vulnerable-system", - "extra.dns_version": "dnsmasq-2.66", - "extra.min_amplification": 4.619, - "extra.tag": "openresolver", - "protocol.application": "dns", - "protocol.transport": "udp", + "extra.smb_implant": False, + "extra.smb_major_number": '2', + "extra.smb_minor_number": '1', + "extra.smb_version_string": 'SMB 2.1', + "extra.smbv1_support": 'N', + "extra.tag": "smb", + "protocol.application": "smb", + "protocol.transport": "tcp", 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - "source.asn": 25255, - "source.geolocation.cc": "AT", - "source.geolocation.city": "VIENNA", - "source.geolocation.region": "WIEN", - "source.ip": "198.51.100.179", - "source.port": 53, - "source.reverse_dns": "198-51-100-189.example.net", + "source.asn": 64512, + "source.geolocation.cc": "ZZ", + "source.geolocation.city": "City", + "source.geolocation.region": "Region", + "source.ip": "192.168.0.1", + "source.port": 445, + "source.reverse_dns": "node01.example.com", "time.observation": "2018-07-30T00:00:00+00:00", - "time.source": "2018-04-14T00:14:34+00:00" + "time.source": "2010-02-10T00:00:00+00:00" }, ] @@ -70,7 +73,7 @@ def test_overwrite_feed_name(self): self.run_bot(prepare=False) for i, EVENT in enumerate(EVENTS): event = EVENT.copy() - event['feed.name'] = 'DNS-Open-Resolvers' + event['feed.name'] = 'Test-Accessible-SMB' self.assertMessageEqual(i, event) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py new file mode 100644 index 000000000..c7eefdf0a --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import unittest + +import intelmq.lib.test as test +import intelmq.lib.utils as utils +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot + +with open(os.path.join(os.path.dirname(__file__), + 'testdata/test_smb.csv')) as handle: + EXAMPLE_FILE = handle.read() +EXAMPLE_LINES = EXAMPLE_FILE.splitlines() + +EXAMPLE_REPORT = {'feed.name': 'Test-Accessible-SMB', + "raw": utils.base64_encode(EXAMPLE_FILE), + "__type": "Report", + "time.observation": "2015-01-01T00:00:00+00:00", + "extra.file_name": "2019-01-01-test_smb-test-geo.csv", + } +EVENTS = [ +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.1', + 'source.port' : 445, + 'source.reverse_dns' : 'node01.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:00+00:00' +}, + +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.2', + 'source.port' : 445, + 'source.reverse_dns' : 'node02.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:01+00:00' +}, + +{ + '__type' : 'Event', + 'classification.identifier' : 'test-smb', + 'classification.taxonomy' : 'vulnerable', + 'classification.type' : 'vulnerable-system', + 'extra.smb_implant' : False, + 'extra.smb_major_number' : '2', + 'extra.smb_minor_number' : '1', + 'extra.smb_version_string' : 'SMB 2.1', + 'extra.smbv1_support' : 'N', + 'extra.tag' : 'smb', + 'feed.name' : 'Test-Accessible-SMB', + 'protocol.application' : 'smb', + 'protocol.transport' : 'tcp', + 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), + 'source.asn' : 64512, + 'source.geolocation.cc' : 'ZZ', + 'source.geolocation.city' : 'City', + 'source.geolocation.region' : 'Region', + 'source.ip' : '192.168.0.3', + 'source.port' : 445, + 'source.reverse_dns' : 'node03.example.com', + 'time.observation' : '2015-01-01T00:00:00+00:00', + 'time.source' : '2010-02-10T00:00:02+00:00' +} + ] + + +class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for a ShadowserverParserBot. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.default_input_message = EXAMPLE_REPORT + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() + for i, EVENT in enumerate(EVENTS): + self.assertMessageEqual(i, EVENT) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index 0a34a69f0..570d612fb 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -12,24 +12,24 @@ from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot with open(os.path.join(os.path.dirname(__file__), - 'testdata/scan_ftp.csv')) as handle: + 'testdata/test_smb.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] -FIRST_REPORT = {'feed.name': 'Accessible FTP', +FIRST_REPORT = {'feed.name': 'Test-Accessible-SMB', "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), "__type": "Report", "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-scan_ftp-test-test.csv", + "extra.file_name": "2019-03-25-test_smb-test-test.csv", } -with open(os.path.join(os.path.dirname(__file__), 'testdata/blocklist.csv')) as handle: +with open(os.path.join(os.path.dirname(__file__), 'testdata/test_telnet.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] SECOND_REPORT = { - 'feed.name': 'Blocklist', + 'feed.name': 'Test-Accessible-Telnet', "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", - "extra.file_name": "2019-01-01-blocklist-test-geo.csv", + "extra.file_name": "2019-01-01-test_telnet-test-geo.csv", } @@ -48,9 +48,9 @@ def test_event(self): """ Test if the parser correctly detects and handles different report types. """ self.input_message = [FIRST_REPORT, SECOND_REPORT] self.run_bot(iterations=2) - self.assertLogMatches("Detected report's file name: 'scan_ftp'", + self.assertLogMatches("Detected report's file name: 'test_smb'", levelname='DEBUG') - self.assertLogMatches("Detected report's file name: 'blocklist'", + self.assertLogMatches("Detected report's file name: 'test_telnet'", levelname='DEBUG') diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py new file mode 100644 index 000000000..6d539ac4a --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- + +import os +import unittest + +import intelmq.lib.test as test +import intelmq.lib.utils as utils +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot + +with open(os.path.join(os.path.dirname(__file__), + 'testdata/test_telnet.csv')) as handle: + EXAMPLE_FILE = handle.read() +EXAMPLE_LINES = EXAMPLE_FILE.splitlines() + +EXAMPLE_REPORT = {'feed.name': 'Test-Accessible-Telnet', + "raw": utils.base64_encode(EXAMPLE_FILE), + "__type": "Report", + "time.observation": "2015-01-01T00:00:00+00:00", + "extra.file_name": "2019-01-01-test_telnet-test-geo.csv", + } +EVENTS = [{'__type': 'Event', + 'feed.name': 'Test-Accessible-Telnet', + "classification.identifier": "test-telnet", + "classification.taxonomy": "vulnerable", + "classification.type": "vulnerable-system", + "extra.banner": "|MikroTik v6.5|Login:", + "extra.tag": "telnet-alt", + "protocol.application": "telnet", + "protocol.transport": "tcp", + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], + EXAMPLE_LINES[1]])), + "source.asn": 20255, + "source.geolocation.cc": "AA", + "source.geolocation.city": "LOCATION", + "source.geolocation.region": "LOCATION", + "source.ip": "198.123.245.145", + "source.port": 5678, + "source.reverse_dns": "example.local", + "time.observation": "2015-01-01T00:00:00+00:00", + "time.source": "2019-09-04T12:27:34+00:00" + }, + {'__type': 'Event', + 'feed.name': 'Test-Accessible-Telnet', + "classification.identifier": "test-telnet", + "classification.taxonomy": "vulnerable", + "classification.type": "vulnerable-system", + "extra.banner": "|MikroTik v6.45.3 (stable)|Login:", + "extra.tag": "telnet-alt", + "protocol.application": "telnet", + "protocol.transport": "tcp", + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], + EXAMPLE_LINES[2]])), + "source.asn": 20255, + "source.geolocation.cc": "AA", + "source.geolocation.city": "LOCATION", + "source.geolocation.region": "LOCATION", + "source.ip": "198.123.245.145", + "source.port": 5678, + "source.reverse_dns": "example.local", + "time.observation": "2015-01-01T00:00:00+00:00", + "time.source": "2019-09-04T12:27:40+00:00" + }] + + +class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): + """ + A TestCase for a ShadowserverParserBot. + """ + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.default_input_message = EXAMPLE_REPORT + + def test_event(self): + """ Test if correct Event has been produced. """ + self.run_bot() + for i, EVENT in enumerate(EVENTS): + self.assertMessageEqual(i, EVENT) + + +if __name__ == '__main__': # pragma: no cover + unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv new file mode 100644 index 000000000..fc7fe2fff --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv @@ -0,0 +1,4 @@ +"timestamp","ip","port","hostname","tag","asn","geo","region","city","naics","sic","smb_implant","arch","key","smbv1_support","smb_major_number","smb_minor_number","smb_revision","smb_version_string" +"2010-02-10 00:00:00",192.168.0.1,445,node01.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" +"2010-02-10 00:00:01",192.168.0.2,445,node02.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" +"2010-02-10 00:00:02",192.168.0.3,445,node03.example.com,smb,64512,ZZ,Region,City,0,0,N,,,N,2,1,0,"SMB 2.1" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license new file mode 100644 index 000000000..f512a890e --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_smb.csv.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2022 The Shadowserver Foundation +SPDX-License-Identifier: AGPL-3.0-or-later diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv new file mode 100644 index 000000000..3309e9a3d --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv @@ -0,0 +1,3 @@ +"timestamp","ip","protocol","port","hostname","tag","asn","geo","region","city","naics","sic","banner" +"2019-09-04 12:27:34","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.5|Login:" +"2019-09-04 12:27:40","198.123.245.145","tcp",5678,"example.local","telnet-alt",20255,"AA","LOCATION","LOCATION",0,0,"|MikroTik v6.45.3 (stable)|Login:" diff --git a/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license new file mode 100644 index 000000000..942a94035 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/testdata/test_telnet.csv.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2019 Guillermo Rodriguez +SPDX-License-Identifier: AGPL-3.0-or-later From c6108d6b219a1588cd45ba6bf7ec89dd6a5c5a42 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 8 May 2023 15:05:12 +0000 Subject: [PATCH 46/67] Updated to reset report type on reload #2361 --- intelmq/bots/parsers/shadowserver/README.md | 2 +- intelmq/bots/parsers/shadowserver/_config.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index 297930861..bb6216b9a 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -11,6 +11,6 @@ The report configuration is now stored in a _schema.json_ file downloaded from h For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index a7b80b7a6..29382d278 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -272,15 +272,14 @@ def scan_exchange_identifier(field): def reload (): """ reload the configuration if it has changed """ mtime = 0.0 + schema_file = __config.schema_file - if (os.path.isfile(__config.schema_file)): + if os.path.isfile(__config.schema_file): mtime = os.path.getmtime(__config.schema_file) if __config.schema_mtime == mtime: return - schema_file = __config.schema_file else: # load a test schema if one has not been downloaded yet - schema_file = __config.schema_file schema_file += '.test' __config.feedname_mapping.clear() From 308ec67e4227634cece6276ac47e53adff7aed63 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 23 May 2023 01:12:47 +0000 Subject: [PATCH 47/67] Added schema download on startup and additional logging --- intelmq/bots/parsers/shadowserver/_config.py | 33 +++++++++++++------ intelmq/bots/parsers/shadowserver/parser.py | 1 + .../parsers/shadowserver/update_schema.py | 3 +- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 29382d278..f766be322 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -106,6 +106,8 @@ def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, A reload() return __config.filename_mapping.get(given_filename, None) +def set_logger(logger): + __config.logger = logger def add_UTC_to_timestamp(value: str) -> str: return value + ' UTC' @@ -272,29 +274,38 @@ def scan_exchange_identifier(field): def reload (): """ reload the configuration if it has changed """ mtime = 0.0 - schema_file = __config.schema_file if os.path.isfile(__config.schema_file): mtime = os.path.getmtime(__config.schema_file) if __config.schema_mtime == mtime: return else: - # load a test schema if one has not been downloaded yet - schema_file += '.test' + __config.logger.info("The schema file does not exist.") + + if __config.schema_mtime == 0.0 and mtime == 0.0 and not os.environ.get('INTELMQ_SKIP_INTERNET'): + __config.logger.info("Attempting to download schema.") + update_schema() __config.feedname_mapping.clear() __config.filename_mapping.clear() - with open(schema_file) as fh: - schema = json.load(fh) - for report in schema: - __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) - __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + for schema_file in [ __config.schema_file, ".".join([__config.schema_file, 'test']) ]: + if os.path.isfile(schema_file): + with open(schema_file) as fh: + schema = json.load(fh) + for report in schema: + if report == "_meta": + __config.logger.info("Loading schema %s." % schema[report]['date_created']) + for msg in schema[report]['change_log']: + __config.logger.info(msg) + else: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime -def update_schema (version): +def update_schema (): """ download the latest configuration """ (th, tmp) = tempfile.mkstemp() - url = 'https://interchange.shadowserver.org/intelmq/'+version + url = 'https://interchange.shadowserver.org/intelmq/v1' try: urllib.request.urlretrieve(url, tmp) except: @@ -307,4 +318,6 @@ def update_schema (version): # leave tempfile behind for diagnosis raise ValueError("Failed to validate %r" % tmp) + if os.path.exists(__config.schema_file): + os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) os.replace(tmp, __config.schema_file) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index f14549141..2f20262bf 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -47,6 +47,7 @@ class ShadowserverParserBot(ParserBot): overwrite = False def init(self): + config.set_logger(self.logger) if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py index 040f67259..a7975147e 100644 --- a/intelmq/bots/parsers/shadowserver/update_schema.py +++ b/intelmq/bots/parsers/shadowserver/update_schema.py @@ -8,5 +8,4 @@ import intelmq.bots.parsers.shadowserver._config as config if __name__ == '__main__': # pragma: no cover - exec(open(os.path.join(os.path.dirname(__file__), '../../../version.py')).read()) # defines __version__ - config.update_schema(__version__) + config.update_schema() From 9ecf36616a2cec50de0eb5a562403ea2e212de8c Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 23 May 2023 23:32:53 +0000 Subject: [PATCH 48/67] Added version support to the schema update function. --- intelmq/bots/parsers/shadowserver/README.md | 6 ++-- intelmq/bots/parsers/shadowserver/_config.py | 32 +++++++++++++++++--- intelmq/bots/parsers/shadowserver/parser.py | 4 +++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index bb6216b9a..c757020e9 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,10 +7,10 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/_version_. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1. -For environments that have internet connectivity the `update_schema.py` script should be setup as a cron job to obtain the latest revision. +For environments that have internet connectivity the `update_schema.py` script can be called from a cron job to obtain the latest revision. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. +For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index f766be322..bb67db525 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -97,6 +97,11 @@ class __Container: __config.feedname_mapping = {} __config.filename_mapping = {} +def set_logger(logger): + """ Sets the logger instance. """ + __config.logger = logger + + def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: reload() return __config.feedname_mapping.get(given_feedname, None) @@ -106,8 +111,6 @@ def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, A reload() return __config.filename_mapping.get(given_filename, None) -def set_logger(logger): - __config.logger = logger def add_UTC_to_timestamp(value: str) -> str: return value + ' UTC' @@ -304,20 +307,39 @@ def reload (): def update_schema (): """ download the latest configuration """ - (th, tmp) = tempfile.mkstemp() + if os.environ.get('INTELMQ_SKIP_INTERNET'): + return None + + (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) url = 'https://interchange.shadowserver.org/intelmq/v1' try: urllib.request.urlretrieve(url, tmp) except: raise ValueError("Failed to download %r" % url) + new_version = '' + old_version = '' + try: with open(tmp) as fh: schema = json.load(fh) + new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis raise ValueError("Failed to validate %r" % tmp) if os.path.exists(__config.schema_file): - os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) - os.replace(tmp, __config.schema_file) + old_version = '' + try: + with open(__config.schema_file) as fh: + schema = json.load(fh) + old_version = schema['_meta']['date_created'] + if new_version != old_version: + os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) + except: + pass + + if new_version != old_version: + os.replace(tmp, __config.schema_file) + else: + os.unlink(tmp) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 2f20262bf..71489e2ec 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -48,6 +48,10 @@ class ShadowserverParserBot(ParserBot): def init(self): config.set_logger(self.logger) + try: + config.update_schema() + except Exception as e: + logger.warning(f"Schema update failed: {e}.") if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: From 9c4a1a4dfd47a3be3bc5dd1cc77228464a426450 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Sun, 28 May 2023 23:13:54 +0000 Subject: [PATCH 49/67] Documentation and style updates. --- CHANGELOG.md | 6 + .../shadowserver/collector_reports_api.py | 2 +- intelmq/bots/parsers/shadowserver/README.md | 39 ++++- intelmq/bots/parsers/shadowserver/_config.py | 52 +++--- intelmq/bots/parsers/shadowserver/parser.py | 2 +- .../bots/parsers/shadowserver/test_broken.py | 4 +- .../bots/parsers/shadowserver/test_mapping.py | 1 - .../parsers/shadowserver/test_report_smb.py | 151 +++++++++--------- .../shadowserver/test_report_switch.py | 10 +- .../shadowserver/test_report_telnet.py | 4 +- 10 files changed, 154 insertions(+), 117 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72d950193..ea36275bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,15 +118,21 @@ CHANGELOG ### Bots #### Collectors +<<<<<<< HEAD - `intelmq.bots.collector.rt`: - restrict `python-rt` to be below version 3.0 due to introduced breaking changes, - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). +======= +- `intelmq.bots.collectors.shadowserver.collector_reports_api`: + - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) +>>>>>>> Documentation and style updates. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: - Reset detected `feedname` at shutdown to re-detect the feedname on reloads (PR#2361 by @elsif2, fixes #2360). + - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. Please see intelmq/bots/parsers/shadowserver/README.md for details. (PR#2372) - `intelmq.bots.parsers.shadowserver._config`: - Added 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Service' report. (PR#2338) - Removed unused `p0f_genre` and `p0f_detail` from the 'DNS-Open-Resolvers' report. (PR#2338) diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index dc8bd6b42..5e7117bd2 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,7 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv'). The 'json' option is not longer supported. + file_format (str): File format to download ('csv'). The 'json' option is no longer supported. """ country = None diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index c757020e9..ae38dcb8c 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,10 +7,45 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1. +The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. -For environments that have internet connectivity the `update_schema.py` script can be called from a cron job to obtain the latest revision. +For environments that have internet connectivity the `update_schema.py` script should be called from a cron job to obtain the latest revision. +The parser will attempt to download a schema update on startup unless INTELMQ_SKIP_INTERNET is set. For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. The parser will automatically reload the configuration when the file changes. + + +## Sample configuration: + +``` +shadowserver-collector: + description: Our bot responsible for getting reports from Shadowserver + enabled: true + group: Collector + module: intelmq.bots.collectors.shadowserver.collector_reports_api + name: Shadowserver_Collector + parameters: + destination_queues: + _default: [shadowserver-parser-queue] + file_format: csv + api_key: "$API_KEY_received_from_the_shadowserver_foundation" + secret: "$SECRET_received_from_the_shadowserver_foundation" + run_mode: continuous +``` + +``` +shadowserver-parser: + bot_id: shadowserver-parser + name: Shadowserver Parser + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.shadowserver.parser + parameters: + destination_queues: + _default: [file-output-queue] + run_mode: continuous +``` + diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index bb67db525..5219fdb34 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -88,15 +88,18 @@ import intelmq.lib.harmonization as harmonization + class __Container: pass + __config = __Container() __config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') __config.schema_mtime = 0.0 __config.feedname_mapping = {} __config.filename_mapping = {} + def set_logger(logger): """ Sets the logger instance. """ __config.logger = logger @@ -254,27 +257,28 @@ def scan_exchange_identifier(field): return 'exchange-server-webshell' return 'vulnerable-exchange-server' + functions = { - 'add_UTC_to_timestamp': add_UTC_to_timestamp, - 'convert_bool': convert_bool, - 'validate_to_none': validate_to_none, - 'convert_int': convert_int, - 'convert_float': convert_float, - 'convert_http_host_and_url': convert_http_host_and_url, - 'invalidate_zero': invalidate_zero, - 'validate_ip': validate_ip, - 'validate_network': validate_network, - 'validate_fqdn': validate_fqdn, - 'convert_date': convert_date, - 'convert_date_utc': convert_date_utc, - 'force_base64': force_base64, - 'scan_exchange_taxonomy': scan_exchange_taxonomy, - 'scan_exchange_type': scan_exchange_type, - 'scan_exchange_identifier': scan_exchange_identifier, - } - - -def reload (): + 'add_UTC_to_timestamp': add_UTC_to_timestamp, + 'convert_bool': convert_bool, + 'validate_to_none': validate_to_none, + 'convert_int': convert_int, + 'convert_float': convert_float, + 'convert_http_host_and_url': convert_http_host_and_url, + 'invalidate_zero': invalidate_zero, + 'validate_ip': validate_ip, + 'validate_network': validate_network, + 'validate_fqdn': validate_fqdn, + 'convert_date': convert_date, + 'convert_date_utc': convert_date_utc, + 'force_base64': force_base64, + 'scan_exchange_taxonomy': scan_exchange_taxonomy, + 'scan_exchange_type': scan_exchange_type, + 'scan_exchange_identifier': scan_exchange_identifier, +} + + +def reload(): """ reload the configuration if it has changed """ mtime = 0.0 @@ -291,7 +295,7 @@ def reload (): __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [ __config.schema_file, ".".join([__config.schema_file, 'test']) ]: + for schema_file in [__config.schema_file, ".".join([__config.schema_file, 'test'])]: if os.path.isfile(schema_file): with open(schema_file) as fh: schema = json.load(fh) @@ -305,13 +309,14 @@ def reload (): __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime -def update_schema (): + +def update_schema(): """ download the latest configuration """ if os.environ.get('INTELMQ_SKIP_INTERNET'): return None (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) - url = 'https://interchange.shadowserver.org/intelmq/v1' + url = 'https://interchange.shadowserver.org/intelmq/v1/schema' try: urllib.request.urlretrieve(url, tmp) except: @@ -329,7 +334,6 @@ def update_schema (): raise ValueError("Failed to validate %r" % tmp) if os.path.exists(__config.schema_file): - old_version = '' try: with open(__config.schema_file) as fh: schema = json.load(fh) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 71489e2ec..668a81534 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -51,7 +51,7 @@ def init(self): try: config.update_schema() except Exception as e: - logger.warning(f"Schema update failed: {e}.") + self.logger.warning("Schema update failed: %s." % e) if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 2b803142e..3797f03cd 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -24,12 +24,12 @@ "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2019-01-01-some_string-test-test.csv", -} + } REPORT4 = {"raw": utils.base64_encode('adasdasdasdasd\nadasdasdafgf'), "__type": "Report", "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2020.wrong-filename.csv", -} + } class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index 6a2af9447..d296dfdc2 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -52,6 +52,5 @@ def test_changed_feed(self): self.run_bot(iterations=2) - if __name__ == '__main__': # pragma: no cover unittest.main() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py index c7eefdf0a..93d592d15 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -22,85 +22,78 @@ "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2019-01-01-test_smb-test-geo.csv", } -EVENTS = [ -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.1', - 'source.port' : 445, - 'source.reverse_dns' : 'node01.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:00+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.2', - 'source.port' : 445, - 'source.reverse_dns' : 'node02.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:01+00:00' -}, - -{ - '__type' : 'Event', - 'classification.identifier' : 'test-smb', - 'classification.taxonomy' : 'vulnerable', - 'classification.type' : 'vulnerable-system', - 'extra.smb_implant' : False, - 'extra.smb_major_number' : '2', - 'extra.smb_minor_number' : '1', - 'extra.smb_version_string' : 'SMB 2.1', - 'extra.smbv1_support' : 'N', - 'extra.tag' : 'smb', - 'feed.name' : 'Test-Accessible-SMB', - 'protocol.application' : 'smb', - 'protocol.transport' : 'tcp', - 'raw' : utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), - 'source.asn' : 64512, - 'source.geolocation.cc' : 'ZZ', - 'source.geolocation.city' : 'City', - 'source.geolocation.region' : 'Region', - 'source.ip' : '192.168.0.3', - 'source.port' : 445, - 'source.reverse_dns' : 'node03.example.com', - 'time.observation' : '2015-01-01T00:00:00+00:00', - 'time.source' : '2010-02-10T00:00:02+00:00' -} - ] +EVENTS = [{'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[1]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.1', + 'source.port': 445, + 'source.reverse_dns': 'node01.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:00+00:00' + }, + {'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[2]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.2', + 'source.port': 445, + 'source.reverse_dns': 'node02.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:01+00:00' + }, + {'__type': 'Event', + 'classification.identifier': 'test-smb', + 'classification.taxonomy': 'vulnerable', + 'classification.type': 'vulnerable-system', + 'extra.smb_implant': False, + 'extra.smb_major_number': '2', + 'extra.smb_minor_number': '1', + 'extra.smb_version_string': 'SMB 2.1', + 'extra.smbv1_support': 'N', + 'extra.tag': 'smb', + 'feed.name': 'Test-Accessible-SMB', + 'protocol.application': 'smb', + 'protocol.transport': 'tcp', + 'raw': utils.base64_encode('\n'.join([EXAMPLE_LINES[0], EXAMPLE_LINES[3]])), + 'source.asn': 64512, + 'source.geolocation.cc': 'ZZ', + 'source.geolocation.city': 'City', + 'source.geolocation.region': 'Region', + 'source.ip': '192.168.0.3', + 'source.port': 445, + 'source.reverse_dns': 'node03.example.com', + 'time.observation': '2015-01-01T00:00:00+00:00', + 'time.source': '2010-02-10T00:00:02+00:00' + }] class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index 570d612fb..a9be8a0a1 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -16,11 +16,11 @@ EXAMPLE_LINES = handle.read().splitlines()[:2] FIRST_REPORT = {'feed.name': 'Test-Accessible-SMB', - "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), - "__type": "Report", - "time.observation": "2019-03-25T00:00:00+00:00", - "extra.file_name": "2019-03-25-test_smb-test-test.csv", - } + "raw": utils.base64_encode('\n'.join(EXAMPLE_LINES)), + "__type": "Report", + "time.observation": "2019-03-25T00:00:00+00:00", + "extra.file_name": "2019-03-25-test_smb-test-test.csv", + } with open(os.path.join(os.path.dirname(__file__), 'testdata/test_telnet.csv')) as handle: EXAMPLE_LINES = handle.read().splitlines()[:2] diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py index 6d539ac4a..df9cf25dc 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -42,7 +42,7 @@ "source.reverse_dns": "example.local", "time.observation": "2015-01-01T00:00:00+00:00", "time.source": "2019-09-04T12:27:34+00:00" - }, + }, {'__type': 'Event', 'feed.name': 'Test-Accessible-Telnet', "classification.identifier": "test-telnet", @@ -63,7 +63,7 @@ "source.reverse_dns": "example.local", "time.observation": "2015-01-01T00:00:00+00:00", "time.source": "2019-09-04T12:27:40+00:00" - }] + }] class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): From e4f9ac4670a21a1bdc582d441e243d38f8f91331 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 30 May 2023 16:05:26 +0000 Subject: [PATCH 50/67] Added schema.json.test.license. --- intelmq/bots/parsers/shadowserver/schema.json.test.license | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 intelmq/bots/parsers/shadowserver/schema.json.test.license diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test.license b/intelmq/bots/parsers/shadowserver/schema.json.test.license new file mode 100644 index 000000000..9f58c89ef --- /dev/null +++ b/intelmq/bots/parsers/shadowserver/schema.json.test.license @@ -0,0 +1,2 @@ +SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +SPDX-License-Identifier: AGPL-3.0-or-later From 460344fa4b26b7b69f7930a2e014183ae3da63e1 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 27 Jul 2023 20:19:25 +0000 Subject: [PATCH 51/67] Updates in response to feedback. --- .../shadowserver/collector_reports_api.py | 9 +++- intelmq/bots/parsers/shadowserver/README.md | 21 ++++++-- intelmq/bots/parsers/shadowserver/_config.py | 53 +++++++++++++------ intelmq/bots/parsers/shadowserver/parser.py | 45 +++++++++++++--- .../parsers/shadowserver/update_schema.py | 11 ---- .../shadowserver/test_download_schema.py | 28 ++++++++++ 6 files changed, 130 insertions(+), 37 deletions(-) delete mode 100644 intelmq/bots/parsers/shadowserver/update_schema.py create mode 100644 intelmq/tests/bots/parsers/shadowserver/test_download_schema.py diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 5e7117bd2..05bffa898 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -68,12 +68,19 @@ def init(self): if self.file_format is not None: if not (self.file_format == 'csv'): - raise ValueError('Invalid file_format') + raise ValueError("Invalid file_format '%s'. Must be 'csv'." % self.file_format) else: self.file_format = 'csv' self.preamble = f'{{ "apikey": "{self.api_key}" ' + def check(parameters: dict): + for key in parameters: + if key == 'file_format' and parameters[key] != 'csv': + return [["error", "Invalid file_format '%s'. Must be 'csv'." % parameters[key]]] + elif key == 'country': + return [["warning", "Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0."]] + def _headers(self, data): return {'HMAC2': hmac.new(self.secret.encode(), data.encode('utf-8'), digestmod=hashlib.sha256).hexdigest()} diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index ae38dcb8c..cd750d00b 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,16 +7,28 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. +The report configuration is now stored in a _shadowserver-schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. -For environments that have internet connectivity the `update_schema.py` script should be called from a cron job to obtain the latest revision. -The parser will attempt to download a schema update on startup unless INTELMQ_SKIP_INTERNET is set. +The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. -For air-gapped systems automation will be required to download and copy the _schema.json_ file into this directory. +Schema downloads can also be scheduled as a cron job: + +``` +02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema +``` + +For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. The parser will automatically reload the configuration when the file changes. +## Schema contract + +Once set the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static. + +Once set report fields will not be deleted. + + ## Sample configuration: ``` @@ -46,6 +58,7 @@ shadowserver-parser: parameters: destination_queues: _default: [file-output-queue] + auto_update: true run_mode: continuous ``` diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 5219fdb34..afe3a6b11 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -82,11 +82,12 @@ import base64 import binascii import json -import urllib.request import tempfile from typing import Optional, Dict, Tuple, Any import intelmq.lib.harmonization as harmonization +from intelmq.lib.utils import create_request_session +from intelmq import VAR_STATE_PATH class __Container: @@ -94,8 +95,10 @@ class __Container: __config = __Container() -__config.schema_file = os.path.join(os.path.dirname(__file__), 'schema.json') +__config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') +__config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') __config.schema_mtime = 0.0 +__config.auto_update = False __config.feedname_mapping = {} __config.filename_mapping = {} @@ -105,13 +108,16 @@ def set_logger(logger): __config.logger = logger +def enable_auto_update(enable): + """ Enable automatic schema update. """ + __config.auto_update = enable + + def get_feed_by_feedname(given_feedname: str) -> Optional[Dict[str, Any]]: - reload() return __config.feedname_mapping.get(given_feedname, None) def get_feed_by_filename(given_filename: str) -> Optional[Tuple[str, Dict[str, Any]]]: - reload() return __config.filename_mapping.get(given_filename, None) @@ -289,19 +295,18 @@ def reload(): else: __config.logger.info("The schema file does not exist.") - if __config.schema_mtime == 0.0 and mtime == 0.0 and not os.environ.get('INTELMQ_SKIP_INTERNET'): - __config.logger.info("Attempting to download schema.") + if __config.schema_mtime == 0.0 and mtime == 0.0 and __config.auto_update: update_schema() __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [__config.schema_file, ".".join([__config.schema_file, 'test'])]: + for schema_file in [__config.schema_file, __config.schema_base]: if os.path.isfile(schema_file): with open(schema_file) as fh: schema = json.load(fh) for report in schema: if report == "_meta": - __config.logger.info("Loading schema %s." % schema[report]['date_created']) + __config.logger.info("Loading schema %r." % schema[report]['date_created']) for msg in schema[report]['change_log']: __config.logger.info(msg) else: @@ -313,37 +318,55 @@ def reload(): def update_schema(): """ download the latest configuration """ if os.environ.get('INTELMQ_SKIP_INTERNET'): - return None + return False - (th, tmp) = tempfile.mkstemp(dir=os.path.dirname(__file__)) + # download the schema to a temp file + (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) url = 'https://interchange.shadowserver.org/intelmq/v1/schema' + __config.logger.info("Attempting to download schema from %r" % url) + __config.logger.debug("Using temp file %r for the download." % tmp) try: - urllib.request.urlretrieve(url, tmp) + with create_request_session() as session: + with session.get(url, stream=True) as r: + r.raise_for_status() + with open(tmp, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) except: - raise ValueError("Failed to download %r" % url) + __config.logger.error("Failed to download %r" % url) + return False + __config.logger.info("Download successful.") new_version = '' old_version = '' try: + # validate the downloaded file with open(tmp) as fh: schema = json.load(fh) new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis - raise ValueError("Failed to validate %r" % tmp) + __config.logger.error("Failed to validate %r" % tmp) + return False if os.path.exists(__config.schema_file): + # compare the new version against the old; rename the existing file try: with open(__config.schema_file) as fh: schema = json.load(fh) old_version = schema['_meta']['date_created'] if new_version != old_version: os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) - except: - pass + except Exception as e: + __config.logger.error("Unable to replace schema file: %s" % str(e)) + return False if new_version != old_version: os.replace(tmp, __config.schema_file) + __config.logger.info("New schema version is %r." % new_version) + return True else: os.unlink(tmp) + + return False diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 668a81534..2e383a004 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -26,6 +26,8 @@ from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue +from intelmq.bin.intelmqctl import IntelMQController +import intelmq.lib.utils as utils import intelmq.bots.parsers.shadowserver._config as config @@ -34,8 +36,7 @@ class ShadowserverParserBot(ParserBot): Parse all ShadowServer feeds Parameters: - schema_file (str): Path to the report schema file - + auto_update (boolean): Enable automatic schema download """ recover_line = ParserBot.recover_line_csv_dict @@ -45,13 +46,15 @@ class ShadowserverParserBot(ParserBot): feedname = None _mode = None overwrite = False + auto_update = False def init(self): config.set_logger(self.logger) - try: - config.update_schema() - except Exception as e: - self.logger.warning("Schema update failed: %s." % e) + if self.auto_update: + config.enable_auto_update(True) + self.logger.debug("Feature 'auto_update' is enabled.") + config.reload() + if self.feedname is not None: self._sparser_config = config.get_feed_by_feedname(self.feedname) if self._sparser_config: @@ -228,5 +231,35 @@ def parse_line(self, row, report): def shutdown(self): self.feedname = None + @classmethod + def _create_argparser(cls): + argparser = super()._create_argparser() + argparser.add_argument("--update-schema", action='store_true', help='downloads latest report schema') + argparser.add_argument("--verbose", action='store_true', help='be verbose') + return argparser + + @classmethod + def run(cls, parsed_args=None): + if not parsed_args: + parsed_args = cls._create_argparser().parse_args() + if parsed_args.update_schema: + logger = utils.log(__name__, log_path=None) + if parsed_args.verbose: + logger.setLevel('INFO') + else: + logger.setLevel('ERROR') + config.set_logger(logger) + if config.update_schema(): + runtime_conf = utils.get_bots_settings() + try: + ctl = IntelMQController() + for bot in runtime_conf: + if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('auto_update', True): + ctl.bot_reload(bot) + except Exception as e: + logger.error("Failed to signal bot: %r" % str(e)) + else: + super().run(parsed_args=parsed_args) + BOT = ShadowserverParserBot diff --git a/intelmq/bots/parsers/shadowserver/update_schema.py b/intelmq/bots/parsers/shadowserver/update_schema.py deleted file mode 100644 index a7975147e..000000000 --- a/intelmq/bots/parsers/shadowserver/update_schema.py +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation -# -# SPDX-License-Identifier: AGPL-3.0-or-later - -# -*- coding: utf-8 -*- - -import os -import intelmq.bots.parsers.shadowserver._config as config - -if __name__ == '__main__': # pragma: no cover - config.update_schema() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py new file mode 100644 index 000000000..e68587682 --- /dev/null +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2023 The Shadowserver Foundation +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +# -*- coding: utf-8 -*- +""" +Created on Thu Jul 27 19:44:44 2023 + +""" + +import unittest +import os +import logging +from intelmq import VAR_STATE_PATH +import intelmq.bots.parsers.shadowserver._config as config +import intelmq.lib.utils as utils +import intelmq.lib.test as test + +@test.skip_internet() +class TestShadowserverSchemaDownload(unittest.TestCase): + + def test_download(self): + schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + config.set_logger(utils.log('test-bot', log_path=None)) + if os.path.exists(schema_file): + os.unlink(schema_file) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From fec1fd2a22f1d26578ec5d9aeed752fe760c14ee Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 28 Jul 2023 14:17:41 +0000 Subject: [PATCH 52/67] Removed file_format parameter --- .../shadowserver/collector_reports_api.py | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/intelmq/bots/collectors/shadowserver/collector_reports_api.py b/intelmq/bots/collectors/shadowserver/collector_reports_api.py index 05bffa898..66169d96f 100644 --- a/intelmq/bots/collectors/shadowserver/collector_reports_api.py +++ b/intelmq/bots/collectors/shadowserver/collector_reports_api.py @@ -34,7 +34,6 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): A list of strings or a comma-separated list of the mailing lists you want to process. types (list): A list of strings or a string of comma-separated values with the names of reporttypes you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). - file_format (str): File format to download ('csv'). The 'json' option is no longer supported. """ country = None @@ -42,7 +41,6 @@ class ShadowServerAPICollectorBot(CollectorBot, HttpMixin, CacheMixin): secret = None types = None reports = None - file_format = None rate_limit: int = 86400 redis_cache_db: int = 12 redis_cache_host: str = "127.0.0.1" # TODO: type could be ipadress @@ -66,18 +64,12 @@ def init(self): self.logger.warn("Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0.") self._report_list.append(self.country) - if self.file_format is not None: - if not (self.file_format == 'csv'): - raise ValueError("Invalid file_format '%s'. Must be 'csv'." % self.file_format) - else: - self.file_format = 'csv' - self.preamble = f'{{ "apikey": "{self.api_key}" ' def check(parameters: dict): for key in parameters: - if key == 'file_format' and parameters[key] != 'csv': - return [["error", "Invalid file_format '%s'. Must be 'csv'." % parameters[key]]] + if key == 'file_format': + return [["error", "The file_format parameter is no longer supported. All reports are CSV."]] elif key == 'country': return [["warning", "Deprecated parameter 'country' found. Please use 'reports' instead. The backwards-compatibility will be removed in IntelMQ version 4.0.0."]] @@ -129,11 +121,7 @@ def _report_download(self, reportid: str): data = self.preamble data += f',"id": "{reportid}"}}' self.logger.debug('Downloading report with data: %s.', data) - - if (self.file_format == 'json'): - response = self.http_session().post(APIROOT + 'reports/download', data=data, headers=self._headers(data)) - else: - response = self.http_session().get(DLROOT + reportid) + response = self.http_session().get(DLROOT + reportid) response.raise_for_status() return response.text @@ -150,7 +138,7 @@ def process(self): for item in reportslist: filename = item['file'] - filename_fixed = FILENAME_PATTERN.sub('.' + self.file_format, filename, count=1) + filename_fixed = FILENAME_PATTERN.sub('.csv', filename, count=1) if self.cache_get(filename): self.logger.debug('Processed file %r (fixed: %r) already.', filename, filename_fixed) continue From fe2a37c6c6526950e3602647303ec4a4efa79c86 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 24 Aug 2023 16:04:21 +0000 Subject: [PATCH 53/67] Minor changes based on feedback 2023-08-24 --- CHANGELOG.md | 2 - intelmq/bots/parsers/shadowserver/README.md | 2 + intelmq/bots/parsers/shadowserver/_config.py | 49 ++++++++++--------- intelmq/bots/parsers/shadowserver/parser.py | 6 ++- .../bots/parsers/shadowserver/test_broken.py | 5 ++ .../bots/parsers/shadowserver/test_mapping.py | 1 + .../parsers/shadowserver/test_parameters.py | 3 +- .../parsers/shadowserver/test_report_smb.py | 1 + .../shadowserver/test_report_switch.py | 1 + .../shadowserver/test_report_telnet.py | 1 + 10 files changed, 45 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea36275bc..8cee9e520 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,10 +124,8 @@ CHANGELOG - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). -======= - `intelmq.bots.collectors.shadowserver.collector_reports_api`: - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) ->>>>>>> Documentation and style updates. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index cd750d00b..4969acb6d 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -28,6 +28,8 @@ Once set the `classification.identifier`, `classification.taxonomy`, and `classi Once set report fields will not be deleted. +The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. + ## Sample configuration: diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index afe3a6b11..4bfadb9d9 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -95,8 +95,10 @@ class __Container: __config = __Container() +__config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema' __config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') __config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') +__config.schema_active = __config.schema_file __config.schema_mtime = 0.0 __config.auto_update = False __config.feedname_mapping = {} @@ -108,6 +110,13 @@ def set_logger(logger): __config.logger = logger +def enable_test_mode(enable): + """ Set which schema to load. """ + if enable: + __config.schema_active = __config.schema_base + else: + __config.schema_active = __config.schema_file + def enable_auto_update(enable): """ Enable automatic schema update. """ __config.auto_update = enable @@ -300,40 +309,36 @@ def reload(): __config.feedname_mapping.clear() __config.filename_mapping.clear() - for schema_file in [__config.schema_file, __config.schema_base]: - if os.path.isfile(schema_file): - with open(schema_file) as fh: - schema = json.load(fh) - for report in schema: - if report == "_meta": - __config.logger.info("Loading schema %r." % schema[report]['date_created']) - for msg in schema[report]['change_log']: - __config.logger.info(msg) - else: - __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) - __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) + if os.path.isfile(__config.schema_active): + with open(__config.schema_active) as fh: + schema = json.load(fh) + for report in schema: + if report == "_meta": + __config.logger.info("Loading schema %r.", schema[report]['date_created']) + for msg in schema[report]['change_log']: + __config.logger.info(msg) + else: + __config.feedname_mapping[schema[report]['feed_name']] = (schema[report]['feed_name'], schema[report]) + __config.filename_mapping[schema[report]['file_name']] = (schema[report]['feed_name'], schema[report]) __config.schema_mtime = mtime def update_schema(): """ download the latest configuration """ - if os.environ.get('INTELMQ_SKIP_INTERNET'): - return False # download the schema to a temp file (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) - url = 'https://interchange.shadowserver.org/intelmq/v1/schema' - __config.logger.info("Attempting to download schema from %r" % url) - __config.logger.debug("Using temp file %r for the download." % tmp) + __config.logger.info("Attempting to download schema from %r", __config.schema_url) + __config.logger.debug("Using temp file %r for the download.", tmp) try: with create_request_session() as session: - with session.get(url, stream=True) as r: + with session.get(__config.schema_url, stream=True) as r: r.raise_for_status() with open(tmp, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) except: - __config.logger.error("Failed to download %r" % url) + __config.logger.error("Failed to download %r", __config.schema_url) return False __config.logger.info("Download successful.") @@ -347,7 +352,7 @@ def update_schema(): new_version = schema['_meta']['date_created'] except: # leave tempfile behind for diagnosis - __config.logger.error("Failed to validate %r" % tmp) + __config.logger.error("Failed to validate %r", tmp) return False if os.path.exists(__config.schema_file): @@ -359,12 +364,12 @@ def update_schema(): if new_version != old_version: os.replace(__config.schema_file, ".".join([__config.schema_file, 'bak'])) except Exception as e: - __config.logger.error("Unable to replace schema file: %s" % str(e)) + __config.logger.error("Unable to replace schema file: %s", str(e)) return False if new_version != old_version: os.replace(tmp, __config.schema_file) - __config.logger.info("New schema version is %r." % new_version) + __config.logger.info("New schema version is %r.", new_version) return True else: os.unlink(tmp) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 2e383a004..fd9fa6b2c 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -37,6 +37,7 @@ class ShadowserverParserBot(ParserBot): Parameters: auto_update (boolean): Enable automatic schema download + test_mode (boolean): Use test schema """ recover_line = ParserBot.recover_line_csv_dict @@ -47,9 +48,12 @@ class ShadowserverParserBot(ParserBot): _mode = None overwrite = False auto_update = False + test_mode = False def init(self): config.set_logger(self.logger) + if self.test_mode: + config.enable_test_mode(True) if self.auto_update: config.enable_auto_update(True) self.logger.debug("Feature 'auto_update' is enabled.") @@ -254,7 +258,7 @@ def run(cls, parsed_args=None): try: ctl = IntelMQController() for bot in runtime_conf: - if runtime_conf[bot]["module"] == __name__ and runtime_conf[bot]['parameters'].get('auto_update', True): + if runtime_conf[bot]["module"] == __name__: ctl.bot_reload(bot) except Exception as e: logger.error("Failed to signal bot: %r" % str(e)) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 3797f03cd..54a85e780 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -46,6 +46,7 @@ def test_broken(self): """ Test a report which does not have valid fields """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT1 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="Detected report's file name: 'test_smb'.", @@ -59,6 +60,7 @@ def test_half_broken(self): """ Test a report which does not have an optional field. """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT2 self.run_bot(allowed_warning_count=63) self.assertLogMatches(pattern="Detected report's file name: 'test_telnet'.", @@ -72,6 +74,7 @@ def test_no_config(self): """ Test a report which does not have a valid extra.file_name """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT3 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: Could not get a config for 'some_string', check the documentation.") @@ -80,6 +83,7 @@ def test_invalid_filename(self): """ Test a report which does not have a valid extra.file_name """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = REPORT4 self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: Report's 'extra.file_name' '2020.wrong-filename.csv' is not valid.") @@ -89,6 +93,7 @@ def test_no_report_name(self): Test a report without file_name and no given feedname as parameter. Error message should be verbose. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot(allowed_error_count=1) self.assertLogMatches(pattern="ValueError: No feedname given as parameter and the " "processed report has no 'extra.file_name'. " diff --git a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py index d296dfdc2..b764de827 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_mapping.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_mapping.py @@ -48,6 +48,7 @@ def test_changed_feed(self): Tests if the parser correctly re-detects the feed for the second received report #1493 """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = (EXAMPLE_TELNET, EXAMPLE_VNC) self.run_bot(iterations=2) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py index 677cd0319..45a4a8735 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_parameters.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_parameters.py @@ -63,13 +63,14 @@ def set_bot(cls): def test_default(self): """ Test if feed name is not overwritten has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) def test_overwrite_feed_name(self): """ Test if feed name is overwritten if asked to do so. """ - self.prepare_bot(parameters={'overwrite': True}) + self.prepare_bot(parameters={'test_mode': True, 'overwrite': True}) self.run_bot(prepare=False) for i, EVENT in enumerate(EVENTS): event = EVENT.copy() diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py index 93d592d15..aa6940061 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_smb.py @@ -108,6 +108,7 @@ def set_bot(cls): def test_event(self): """ Test if correct Event has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py index a9be8a0a1..488f5a51a 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_switch.py @@ -46,6 +46,7 @@ def set_bot(cls): def test_event(self): """ Test if the parser correctly detects and handles different report types. """ + self.prepare_bot(parameters={'test_mode': True}) self.input_message = [FIRST_REPORT, SECOND_REPORT] self.run_bot(iterations=2) self.assertLogMatches("Detected report's file name: 'test_smb'", diff --git a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py index df9cf25dc..b2499c589 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_report_telnet.py @@ -78,6 +78,7 @@ def set_bot(cls): def test_event(self): """ Test if correct Event has been produced. """ + self.prepare_bot(parameters={'test_mode': True}) self.run_bot() for i, EVENT in enumerate(EVENTS): self.assertMessageEqual(i, EVENT) From ec066ce06a06dd87912ad3b4337c84fe12821eba Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 24 Aug 2023 16:26:59 +0000 Subject: [PATCH 54/67] Added VAR_STATE_PATH check. --- intelmq/bots/parsers/shadowserver/_config.py | 1 + .../parsers/shadowserver/test_download_schema.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 4bfadb9d9..6ffffdae8 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -117,6 +117,7 @@ def enable_test_mode(enable): else: __config.schema_active = __config.schema_file + def enable_auto_update(enable): """ Enable automatic schema update. """ __config.auto_update = enable diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index e68587682..f9512ca98 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -20,9 +20,10 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') - config.set_logger(utils.log('test-bot', log_path=None)) - if os.path.exists(schema_file): - os.unlink(schema_file) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + if os.path.isdir(VAR_STATE_PATH): + schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + config.set_logger(utils.log('test-bot', log_path=None)) + if os.path.exists(schema_file): + os.unlink(schema_file) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From d1427f3365aa03c6df3c8befd0f270db3e94d96f Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 15:37:51 +0000 Subject: [PATCH 55/67] Changes based on feedback 2023-08-25. --- CHANGELOG.md | 6 +- docs/user/bots.rst | 171 ++++++------------ intelmq/bots/parsers/shadowserver/README.md | 57 ------ intelmq/bots/parsers/shadowserver/_config.py | 10 +- .../shadowserver/test_download_schema.py | 8 +- 5 files changed, 72 insertions(+), 180 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cee9e520..9fdc10225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,20 +118,18 @@ CHANGELOG ### Bots #### Collectors -<<<<<<< HEAD - `intelmq.bots.collector.rt`: - restrict `python-rt` to be below version 3.0 due to introduced breaking changes, - added support for `Subject NOT LIKE` queries, - added support for multiple values in ticket subject queries. - `intelmq.bots.collectors.rsync`: Support for optional private key, relative time parsing for the source path, extra rsync parameters and strict host key checking (PR#2241 by Mateo Durante). - `intelmq.bots.collectors.shadowserver.collector_reports_api`: - - The 'json' option is no longer supported as the 'csv' option provides better performance. Please see intelmq/bots/parsers/shadowserver/README.md for a sample configuration. (PR#2372) + - The 'json' option is no longer supported as the 'csv' option provides better performance. #### Parsers - `intelmq.bots.parsers.shadowserver._config`: - Reset detected `feedname` at shutdown to re-detect the feedname on reloads (PR#2361 by @elsif2, fixes #2360). - - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. Please see intelmq/bots/parsers/shadowserver/README.md for details. (PR#2372) -- `intelmq.bots.parsers.shadowserver._config`: + - Switch to dynamic configuration to decouple report schema changes from IntelMQ releases. - Added 'IPv6-Vulnerable-Exchange' alias and 'Accessible-WS-Discovery-Service' report. (PR#2338) - Removed unused `p0f_genre` and `p0f_detail` from the 'DNS-Open-Resolvers' report. (PR#2338) - Added 'Accessible-SIP' report. (PR#2348) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index 2fbe27df8..a758ff8ad 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -673,6 +673,23 @@ The resulting reports contain the following special field: * `extra.file_name`: The name of the downloaded file, with fixed filename extension. The API returns file names with the extension `.csv`, although the files are JSON, not CSV. Therefore, for clarity and better error detection in the parser, the file name in `extra.file_name` uses `.json` as extension. +**Sample configuration** + +.. code-block:: yaml + + shadowserver-collector: + description: Our bot responsible for getting reports from Shadowserver + enabled: true + group: Collector + module: intelmq.bots.collectors.shadowserver.collector_reports_api + name: Shadowserver_Collector + parameters: + destination_queues: + _default: [shadowserver-parser-queue] + file_format: csv + api_key: "$API_KEY_received_from_the_shadowserver_foundation" + secret: "$SECRET_received_from_the_shadowserver_foundation" + run_mode: continuous .. _intelmq.bots.collectors.shodan.collector_stream: @@ -1557,17 +1574,15 @@ This does not affect URLs which already include the scheme. .. _intelmq.bots.parsers.shadowserver.parser: -.. _intelmq.bots.parsers.shadowserver.parser_json: Shadowserver ^^^^^^^^^^^^ -There are two Shadowserver parsers, one for data in ``CSV`` format (``intelmq.bots.parsers.shadowserver.parser``) and one for data in ``JSON`` format (``intelmq.bots.parsers.shadowserver.parser_json``). -The latter was added in IntelMQ 2.3 and is meant to be used together with the Shadowserver API collector. +The Shadowserver parser operates on ``CSV`` formatted data. **Information** -* `name:` `intelmq.bots.parsers.shadowserver.parser` (for CSV data) or `intelmq.bots.parsers.shadowserver.parser_json` (for JSON data) +* `name:` `intelmq.bots.parsers.shadowserver.parser` * `public:` yes * `description:` Parses different reports from Shadowserver. @@ -1603,107 +1618,45 @@ A list of possible feeds can be found in the table below in the column "feed nam **Supported reports** -These are the supported feed name and their corresponding file name for automatic detection: - - ======================================= ========================= - feed name file name - ======================================= ========================= - Accessible-ADB `scan_adb` - Accessible-AFP `scan_afp` - Accessible-AMQP `scan_amqp` - Accessible-ARD `scan_ard` - Accessible-Cisco-Smart-Install `cisco_smart_install` - Accessible-CoAP `scan_coap` - Accessible-CWMP `scan_cwmp` - Accessible-MS-RDPEUDP `scan_msrdpeudp` - Accessible-FTP `scan_ftp` - Accessible-Hadoop `scan_hadoop` - Accessible-HTTP `scan_http` - Accessible-Radmin `scan_radmin` - Accessible-RDP `scan_rdp` - Accessible-Rsync `scan_rsync` - Accessible-SMB `scan_smb` - Accessible-Telnet `scan_telnet` - Accessible-Ubiquiti-Discovery-Service `scan_ubiquiti` - Accessible-VNC `scan_vnc` - Blacklisted-IP (deprecated) `blacklist` - Blocklist `blocklist` - Compromised-Website `compromised_website` - Device-Identification IPv4 / IPv6 `device_id`/`device_id6` - DNS-Open-Resolvers `scan_dns` - Honeypot-Amplification-DDoS-Events `event4_honeypot_ddos_amp` - Honeypot-Brute-Force-Events `event4_honeypot_brute_force` - Honeypot-Darknet `event4_honeypot_darknet` - Honeypot-HTTP-Scan `event4_honeypot_http_scan` - HTTP-Scanners `hp_http_scan` - ICS-Scanners `hp_ics_scan` - IP-Spoofer-Events `event4_ip_spoofer` - Microsoft-Sinkhole-Events IPv4 `event4_microsoft_sinkhole` - Microsoft-Sinkhole-Events-HTTP IPv4 `event4_microsoft_sinkhole_http` - NTP-Monitor `scan_ntpmonitor` - NTP-Version `scan_ntp` - Open-Chargen `scan_chargen` - Open-DB2-Discovery-Service `scan_db2` - Open-Elasticsearch `scan_elasticsearch` - Open-IPMI `scan_ipmi` - Open-IPP `scan_ipp` - Open-LDAP `scan_ldap` - Open-LDAP-TCP `scan_ldap_tcp` - Open-mDNS `scan_mdns` - Open-Memcached `scan_memcached` - Open-MongoDB `scan_mongodb` - Open-MQTT `scan_mqtt` - Open-MSSQL `scan_mssql` - Open-NATPMP `scan_nat_pmp` - Open-NetBIOS-Nameservice `scan_netbios` - Open-Netis `netis_router` - Open-Portmapper `scan_portmapper` - Open-QOTD `scan_qotd` - Open-Redis `scan_redis` - Open-SNMP `scan_snmp` - Open-SSDP `scan_ssdp` - Open-TFTP `scan_tftp` - Open-XDMCP `scan_xdmcp` - Outdated-DNSSEC-Key `outdated_dnssec_key` - Outdated-DNSSEC-Key-IPv6 `outdated_dnssec_key_v6` - Sandbox-URL `cwsandbox_url` - Sinkhole-DNS `sinkhole_dns` - Sinkhole-Events `event4_sinkhole`/`event6_sinkhole` - Sinkhole-Events IPv4 `event4_sinkhole` - Sinkhole-Events IPv6 `event6_sinkhole` - Sinkhole-HTTP-Events `event4_sinkhole_http`/`event6_sinkhole_http` - Sinkhole-HTTP-Events IPv4 `event4_sinkhole_http` - Sinkhole-HTTP-Events IPv6 `event6_sinkhole_http` - Sinkhole-Events-HTTP-Referer `event4_sinkhole_http_referer`/`event6_sinkhole_http_referer` - Sinkhole-Events-HTTP-Referer IPv4 `event4_sinkhole_http_referer` - Sinkhole-Events-HTTP-Referer IPv6 `event6_sinkhole_http_referer` - Spam-URL `spam_url` - SSL-FREAK-Vulnerable-Servers `scan_ssl_freak` - SSL-POODLE-Vulnerable-Servers `scan_ssl_poodle`/`scan6_ssl_poodle` - Vulnerable-Exchange-Server `*` `scan_exchange` - Vulnerable-ISAKMP `scan_isakmp` - Vulnerable-HTTP `scan_http` - Vulnerable-SMTP `scan_smtp_vulnerable` - ======================================= ========================= - -`*` This report can also contain data on active webshells (column `tag` is `exchange;webshell`), and are therefore not only vulnerable but also actively infected. - -In addition, the following legacy reports are supported: - - =========================== =================================================== ======================== - feed name successor feed name file name - =========================== =================================================== ======================== - Amplification-DDoS-Victim Honeypot-Amplification-DDoS-Events ``ddos_amplification`` - CAIDA-IP-Spoofer IP-Spoofer-Events ``caida_ip_spoofer`` - Darknet Honeypot-Darknet ``darknet`` - Drone Sinkhole-Events ``botnet_drone`` - Drone-Brute-Force Honeypot-Brute-Force-Events, Sinkhole-HTTP-Events ``drone_brute_force`` - Microsoft-Sinkhole Sinkhole-HTTP-Events ``microsoft_sinkhole`` - Sinkhole-HTTP-Drone Sinkhole-HTTP-Events ``sinkhole_http_drone`` - IPv6-Sinkhole-HTTP-Drone Sinkhole-HTTP-Events ``sinkhole6_http`` - =========================== =================================================== ======================== - -More information on these legacy reports can be found in `Changes in Sinkhole and Honeypot Report Types and Formats `_. +The report configuration is stored in a `shadowserver-schema.json` file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. + +The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. + +Schema downloads can also be scheduled as a cron job: + +.. code-block:: bash + + 02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema + + +For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. + +The parser will automatically reload the configuration when the file changes. + +**Schema contract** + +Once set in the schema, the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static for a specific report. + +Report fields will not be removed from a report. + +The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. + +**Sample configuration** + +.. code-block:: yaml + + shadowserver-parser: + bot_id: shadowserver-parser + name: Shadowserver Parser + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.shadowserver.parser + parameters: + destination_queues: + _default: [file-output-queue] + auto_update: true + run_mode: continuous **Development** @@ -1715,14 +1668,6 @@ The parser consists of two files: Both files are required for the parser to work properly. -**Add new Feedformats** - -Add a new feed format and conversions if required to the file -``_config.py``. Don't forget to update the ``mapping`` dict. -It is required to look up the correct configuration. - -Look at the documentation in the bot's ``_config.py`` file for more information. - .. _intelmq.bots.parsers.shodan.parser: diff --git a/intelmq/bots/parsers/shadowserver/README.md b/intelmq/bots/parsers/shadowserver/README.md index 4969acb6d..eb0ddfb4a 100644 --- a/intelmq/bots/parsers/shadowserver/README.md +++ b/intelmq/bots/parsers/shadowserver/README.md @@ -7,60 +7,3 @@ This module is maintained by [The Shadowserver Foundation](https://www.shadowser Please contact intelmq@shadowserver.org with any issues or concerns. -The report configuration is now stored in a _shadowserver-schema.json_ file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. - -The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. - -Schema downloads can also be scheduled as a cron job: - -``` -02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema -``` - -For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. - -The parser will automatically reload the configuration when the file changes. - - -## Schema contract - -Once set the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static. - -Once set report fields will not be deleted. - -The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. - - -## Sample configuration: - -``` -shadowserver-collector: - description: Our bot responsible for getting reports from Shadowserver - enabled: true - group: Collector - module: intelmq.bots.collectors.shadowserver.collector_reports_api - name: Shadowserver_Collector - parameters: - destination_queues: - _default: [shadowserver-parser-queue] - file_format: csv - api_key: "$API_KEY_received_from_the_shadowserver_foundation" - secret: "$SECRET_received_from_the_shadowserver_foundation" - run_mode: continuous -``` - -``` -shadowserver-parser: - bot_id: shadowserver-parser - name: Shadowserver Parser - enabled: true - group: Parser - groupname: parsers - module: intelmq.bots.parsers.shadowserver.parser - parameters: - destination_queues: - _default: [file-output-queue] - auto_update: true - run_mode: continuous -``` - diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 6ffffdae8..279093dfe 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -95,6 +95,7 @@ class __Container: __config = __Container() +__config.var_state_path = VAR_STATE_PATH __config.schema_url = 'https://interchange.shadowserver.org/intelmq/v1/schema' __config.schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') __config.schema_base = os.path.join(os.path.dirname(__file__), 'schema.json.test') @@ -328,7 +329,7 @@ def update_schema(): """ download the latest configuration """ # download the schema to a temp file - (th, tmp) = tempfile.mkstemp(dir=VAR_STATE_PATH) + (th, tmp) = tempfile.mkstemp(dir=__config.var_state_path) __config.logger.info("Attempting to download schema from %r", __config.schema_url) __config.logger.debug("Using temp file %r for the download.", tmp) try: @@ -376,3 +377,10 @@ def update_schema(): os.unlink(tmp) return False + + +def prepare_update_schema_test(path): + """ Reconfigure internal settings to perform a schema update test. """ + __config.var_state_path = path + __config.schema_file = os.path.join(path, 'shadowserver-schema.json') + return __config.schema_file diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index f9512ca98..5246e6bb6 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -10,8 +10,8 @@ import unittest import os +import tempfile import logging -from intelmq import VAR_STATE_PATH import intelmq.bots.parsers.shadowserver._config as config import intelmq.lib.utils as utils import intelmq.lib.test as test @@ -20,10 +20,8 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - if os.path.isdir(VAR_STATE_PATH): - schema_file = os.path.join(VAR_STATE_PATH, 'shadowserver-schema.json') + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) config.set_logger(utils.log('test-bot', log_path=None)) - if os.path.exists(schema_file): - os.unlink(schema_file) self.assertEqual(True, config.update_schema()) self.assertEqual(True, os.path.exists(schema_file)) From ae54e7cf783f770f0f7b25dd919f21d890964c3d Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 15:51:38 +0000 Subject: [PATCH 56/67] Added INTELMQ_SKIP_INTERNET check --- .../bots/parsers/shadowserver/test_download_schema.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index 5246e6bb6..203a3c0b1 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -20,8 +20,9 @@ class TestShadowserverSchemaDownload(unittest.TestCase): def test_download(self): - with tempfile.TemporaryDirectory() as tmp_dir: - schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None)) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + if not os.environ.get('INTELMQ_SKIP_INTERNET'): + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) + config.set_logger(utils.log('test-bot', log_path=None)) + self.assertEqual(True, config.update_schema()) + self.assertEqual(True, os.path.exists(schema_file)) From e4e50637c0da38f32f6b8bbb95aa71875d0c4ad9 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 16:11:21 +0000 Subject: [PATCH 57/67] Added debug logging for CI test. --- intelmq/bots/parsers/shadowserver/_config.py | 3 ++- .../tests/bots/parsers/shadowserver/test_download_schema.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 279093dfe..d573d12c6 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -339,8 +339,9 @@ def update_schema(): with open(tmp, 'wb') as f: for chunk in r.iter_content(chunk_size=8192): f.write(chunk) - except: + except Exception as e: __config.logger.error("Failed to download %r", __config.schema_url) + __config.logger.debug(str(e)) return False __config.logger.info("Download successful.") diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index 203a3c0b1..abcd0ca2a 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -23,6 +23,6 @@ def test_download(self): if not os.environ.get('INTELMQ_SKIP_INTERNET'): with tempfile.TemporaryDirectory() as tmp_dir: schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None)) + config.set_logger(utils.log('test-bot', log_path=None, log_level=logging.DEBUG)) self.assertEqual(True, config.update_schema()) self.assertEqual(True, os.path.exists(schema_file)) From 128048272e04ab012ff80f67f588e326d10859c3 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Fri, 25 Aug 2023 18:47:54 +0000 Subject: [PATCH 58/67] Refactored test_download_schema to utilize mocking. --- intelmq/bots/parsers/shadowserver/parser.py | 6 ++++ .../shadowserver/test_download_schema.py | 30 ++++++++++++------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index fd9fa6b2c..48cbba901 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -23,6 +23,7 @@ import copy import re import os +import tempfile from intelmq.lib.bot import ParserBot from intelmq.lib.exceptions import InvalidKey, InvalidValue @@ -265,5 +266,10 @@ def run(cls, parsed_args=None): else: super().run(parsed_args=parsed_args) + def test_update_schema(cls): + with tempfile.TemporaryDirectory() as tmp_dir: + schema_file = config.prepare_update_schema_test(tmp_dir) + return config.update_schema() + BOT = ShadowserverParserBot diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index abcd0ca2a..abf27a5bd 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -8,21 +8,29 @@ """ -import unittest -import os -import tempfile import logging -import intelmq.bots.parsers.shadowserver._config as config +import unittest +import unittest.mock as mock +from intelmq.bots.parsers.shadowserver.parser import ShadowserverParserBot import intelmq.lib.utils as utils import intelmq.lib.test as test + @test.skip_internet() -class TestShadowserverSchemaDownload(unittest.TestCase): +class TestShadowserverSchemaDownload(test.BotTestCase, unittest.TestCase): + + @classmethod + def set_bot(cls): + cls.bot_reference = ShadowserverParserBot + cls.sysconfig = {"logging_level": "DEBUG"} def test_download(self): - if not os.environ.get('INTELMQ_SKIP_INTERNET'): - with tempfile.TemporaryDirectory() as tmp_dir: - schema_file = config.prepare_update_schema_test(tmp_dir) - config.set_logger(utils.log('test-bot', log_path=None, log_level=logging.DEBUG)) - self.assertEqual(True, config.update_schema()) - self.assertEqual(True, os.path.exists(schema_file)) + self.prepare_bot(prepare_source_queue=False, parameters={'test_mode': True}) + result = False + with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): + with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)): + self.log_stream.truncate(0) + result = self.bot.test_update_schema() + self.bot.stop(exitcode=0) + print(self.log_stream.getvalue()) + self.assertEqual(True, result) From 2a60d2e10a581c9332151da909f3e716d5a825c3 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 28 Aug 2023 14:18:22 +0000 Subject: [PATCH 59/67] Added docstring for test_update_schema(). --- intelmq/bots/parsers/shadowserver/parser.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 48cbba901..4485a2602 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -267,6 +267,13 @@ def run(cls, parsed_args=None): super().run(parsed_args=parsed_args) def test_update_schema(cls): + """ + Test schema download to a temporary directory. + + This is necessary as the request session requires mocking in order to function. + + Returns True on success. + """ with tempfile.TemporaryDirectory() as tmp_dir: schema_file = config.prepare_update_schema_test(tmp_dir) return config.update_schema() From e401e2c1950851092c6febc37d8739eef402a3b4 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 29 Aug 2023 14:09:33 +0000 Subject: [PATCH 60/67] Removed logging output. --- intelmq/tests/bots/parsers/shadowserver/test_download_schema.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py index abf27a5bd..84922bf17 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_download_schema.py @@ -29,8 +29,6 @@ def test_download(self): result = False with mock.patch('intelmq.lib.utils.load_configuration', new=self.mocked_config): with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)): - self.log_stream.truncate(0) result = self.bot.test_update_schema() self.bot.stop(exitcode=0) - print(self.log_stream.getvalue()) self.assertEqual(True, result) From 66ae9f5a10898dda15f3008656b18d44551b5b91 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 31 Aug 2023 20:52:17 +0000 Subject: [PATCH 61/67] Removed the assertion regarding report fields. --- docs/user/bots.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index a758ff8ad..ae17cbf55 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -1637,8 +1637,6 @@ The parser will automatically reload the configuration when the file changes. Once set in the schema, the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static for a specific report. -Report fields will not be removed from a report. - The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. **Sample configuration** From e04dfeee04cfa9308602f870a48af0b933616527 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 16 Oct 2023 17:57:46 +0000 Subject: [PATCH 62/67] Skip and log a warning message for fields not in the IDF. --- intelmq/bots/parsers/shadowserver/parser.py | 5 ++- .../parsers/shadowserver/schema.json.test | 37 +++++++++++++++++++ .../bots/parsers/shadowserver/test_broken.py | 15 ++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index 4485a2602..cfa343138 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -157,7 +157,10 @@ def parse_line(self, row, report): raise if value is not None: - event.add(intelmqkey, value) + try: + event.add(intelmqkey, value) + except InvalidKey: + self.logger.warning('Key not found in IDF %r.', intelmqkey) fields.remove(shadowkey) # Now add optional fields. diff --git a/intelmq/bots/parsers/shadowserver/schema.json.test b/intelmq/bots/parsers/shadowserver/schema.json.test index 2cfb8bb1d..932b8df03 100644 --- a/intelmq/bots/parsers/shadowserver/schema.json.test +++ b/intelmq/bots/parsers/shadowserver/schema.json.test @@ -176,5 +176,42 @@ "convert_int" ] ] + }, + "test_afs" : { + "constant_fields" : { + "classification.identifier" : "test-afs", + "classification.taxonomy" : "vulnerable", + "classification.type" : "vulnerable-system", + "protocol.application" : "afs" + }, + "feed_name" : "Test-Accessible-AFS", + "file_name" : "test_afs", + "required_fields" : [ + [ + "time.source", + "timestamp", + "add_UTC_to_timestamp" + ], + [ + "source.ip", + "ip", + "validate_ip" + ], + [ + "source.port", + "port", + "convert_int" + ], + [ + "not_in_idf", + "severity" + ] + ], + "optional_fields" : [ + [ + "protocol.transport", + "protocol" + ] + ] } } diff --git a/intelmq/tests/bots/parsers/shadowserver/test_broken.py b/intelmq/tests/bots/parsers/shadowserver/test_broken.py index 54a85e780..f1af08e58 100644 --- a/intelmq/tests/bots/parsers/shadowserver/test_broken.py +++ b/intelmq/tests/bots/parsers/shadowserver/test_broken.py @@ -30,6 +30,11 @@ "time.observation": "2015-01-01T00:00:00+00:00", "extra.file_name": "2020.wrong-filename.csv", } +REPORT5 = {"raw": utils.base64_encode('timestamp,ip,protocol,port,severity\n2018-08-01T00:00:00+00,127.0.0.1,tcp,7000,critical'), + "__type": "Report", + "time.observation": "2023-10-16T00:00:00+00:00", + "extra.file_name": "2023-10-16-test_afs-test-test.csv", + } class TestShadowserverParserBot(test.BotTestCase, unittest.TestCase): @@ -100,6 +105,16 @@ def test_no_report_name(self): "Ensure that at least one is given. " "Also have a look at the documentation of the bot.") + def test_field_not_in_idf(self): + """ + Test a report that contains a field mapping not in the IDF. + Error message should be verbose. + """ + self.prepare_bot(parameters={'test_mode': True}) + self.input_message = REPORT5 + self.run_bot(allowed_error_count=0, allowed_warning_count=1) + self.assertLogMatches(pattern="Key not found in IDF", levelname="WARNING") + if __name__ == '__main__': # pragma: no cover unittest.main() From 6f2388349c7f217e6ee35ecf1d7266e726db783b Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 31 Oct 2023 16:19:09 +0000 Subject: [PATCH 63/67] Updated convert_http_host_and_url and added category_or_detail. --- intelmq/bots/parsers/shadowserver/_config.py | 22 ++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index d573d12c6..178bd0869 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -170,12 +170,15 @@ def convert_http_host_and_url(value: str, row: Dict[str, str]) -> str: Sinkhole-HTTP-Drone: http_host, url With some reports, url/http_url holds only the path, with others the full HTTP request. """ + hostname = '' if "cc_dns" in row: hostname = row.get('cc_dns', '') - elif "http_host" in row: - hostname = row.get('http_host', '') - else: - hostname = '' + if not hostname and "http_host" in row: + hostname = row.get("http_host") + if not hostname and "hostname" in row: + hostname = row.get("hostname") + if not hostname and "ip" in row: + hostname = row.get("ip") if "url" in row: path = row.get('url', '') @@ -275,6 +278,16 @@ def scan_exchange_identifier(field): return 'vulnerable-exchange-server' +def category_or_detail(value: str, row: Dict[str, str]) -> str: + """ + Returns the category or detail field from the row. + """ + category = row.get('category', '') + if category != "": + return category + return row.get('detail', '') + + functions = { 'add_UTC_to_timestamp': add_UTC_to_timestamp, 'convert_bool': convert_bool, @@ -292,6 +305,7 @@ def scan_exchange_identifier(field): 'scan_exchange_taxonomy': scan_exchange_taxonomy, 'scan_exchange_type': scan_exchange_type, 'scan_exchange_identifier': scan_exchange_identifier, + 'category_or_detail': category_or_detail, } From a0b34cbc4c7121dd25e0ec5cbec495224e1dc690 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Tue, 31 Oct 2023 18:19:01 +0000 Subject: [PATCH 64/67] Avoid exception when a conversion function is not available in the current version. --- intelmq/bots/parsers/shadowserver/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intelmq/bots/parsers/shadowserver/parser.py b/intelmq/bots/parsers/shadowserver/parser.py index cfa343138..ec1908269 100644 --- a/intelmq/bots/parsers/shadowserver/parser.py +++ b/intelmq/bots/parsers/shadowserver/parser.py @@ -182,7 +182,7 @@ def parse_line(self, row, report): raw_value = row.get(shadowkey) value = raw_value - if conv_func is not None and raw_value is not None: + if conv_func is not None and raw_value is not None and conv_func in config.functions: try: if len(item) == 4 and item[3]: value = config.functions[conv_func](raw_value, row) From 61c756d77322cdeb18beeae5b78fcc1c5f4cb64d Mon Sep 17 00:00:00 2001 From: elsif2 Date: Sat, 4 Nov 2023 00:43:26 +0000 Subject: [PATCH 65/67] Added exception for missing schema and added intelmq user to the crontab suggestion. --- docs/user/bots.rst | 2 +- intelmq/bots/parsers/shadowserver/_config.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/user/bots.rst b/docs/user/bots.rst index ae17cbf55..6c1e65759 100644 --- a/docs/user/bots.rst +++ b/docs/user/bots.rst @@ -1622,7 +1622,7 @@ The report configuration is stored in a `shadowserver-schema.json` file download The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. -Schema downloads can also be scheduled as a cron job: +Schema downloads can also be scheduled as a cron job for the `intelmq` user: .. code-block:: bash diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index 178bd0869..c84bc7e85 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -102,6 +102,7 @@ class __Container: __config.schema_active = __config.schema_file __config.schema_mtime = 0.0 __config.auto_update = False +__config.test_mode = False __config.feedname_mapping = {} __config.filename_mapping = {} @@ -113,6 +114,7 @@ def set_logger(logger): def enable_test_mode(enable): """ Set which schema to load. """ + __config.test_mode = enable if enable: __config.schema_active = __config.schema_base else: @@ -318,7 +320,8 @@ def reload(): if __config.schema_mtime == mtime: return else: - __config.logger.info("The schema file does not exist.") + if not __config.test_mode: + raise ValueError("The schema file does not exist.") if __config.schema_mtime == 0.0 and mtime == 0.0 and __config.auto_update: update_schema() From 307386d7d80dad00199c6ee3f89b6530bdbd6867 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Mon, 13 Nov 2023 17:49:05 +0000 Subject: [PATCH 66/67] Documentation update. --- docs/user/bots.md | 174 ++++++++++++++++------------------------------ 1 file changed, 61 insertions(+), 113 deletions(-) diff --git a/docs/user/bots.md b/docs/user/bots.md index 29977f56e..e32d874c1 100644 --- a/docs/user/bots.md +++ b/docs/user/bots.md @@ -929,11 +929,6 @@ The resulting reports contain the following special field: **Parameters (also expects [feed parameters](#feed-parameters) and [cache parameters](#cache-parameters)):** -**`country`** - -(required, string) **Deprecated:** The country you want to download the reports for. Will be removed in IntelMQ version -4.0.0, use *reports* instead. - **`apikey`** (required, string) Your Shadowserver API key. @@ -948,7 +943,27 @@ The resulting reports contain the following special field: **`types`** -(optional, string/array of strings) An array of strings (or a list of comma-separated values) with the names of report types you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). The possible report types are equivalent to the file names given in the section Supported Reports of the [Shadowserver parser](#intelmq.bots.parsers.shadowserver.parser_json). +(optional, string/array of strings) An array of strings (or a list of comma-separated values) with the names of report types you want to process. If you leave this empty, all the available reports will be downloaded and processed (i.e. 'scan', 'drones', 'intel', 'sandbox_connection', 'sinkhole_combined'). The possible report types are equivalent to the file names given in the section Supported Reports of the [Shadowserver parser](#intelmq.bots.parsers.shadowserver.parser). + +**Sample configuration** + +```yaml + + shadowserver-collector: + description: Our bot responsible for getting reports from Shadowserver + enabled: true + group: Collector + module: intelmq.bots.collectors.shadowserver.collector_reports_api + name: Shadowserver_Collector + parameters: + destination_queues: + _default: [shadowserver-parser-queue] + file_format: csv + api_key: "$API_KEY_received_from_the_shadowserver_foundation" + secret: "$SECRET_received_from_the_shadowserver_foundation" + run_mode: continuous + +``` --- @@ -2071,12 +2086,10 @@ No additional parameters. --- -### Shadowserver
+### Shadowserver
-Parses various reports from Shadowserver. +The Shadowserver parser operates on CSV formatted data. -There are two Shadowserver parsers, one for data in `CSV` format and one for data in `JSON` format. The latter was added -in IntelMQ 2.3 and is meant to be used together with the Shadowserver API collector. **How this bot works?** @@ -2105,8 +2118,7 @@ correct mapping of the columns: **Module:** -`intelmq.bots.parsers.shadowserver.parser` (for CSV data) -`intelmq.bots.parsers.shadowserver.parser_json` (for JSON data) +`intelmq.bots.parsers.shadowserver.parser` **Parameters:** @@ -2120,108 +2132,44 @@ correct mapping of the columns: **Supported reports:** -These are the supported report types and their corresponding file name for automatic detection: - -| Report Type (`feedname`) | File Name | -|-----------|-----------| -| Accessible-ADB | `scan_adb` | -| Accessible-AFP | `scan_afp` | -| Accessible-AMQP | `scan_amqp` | -| Accessible-ARD | `scan_ard` | -| Accessible-Cisco-Smart-Install | `cisco_smart_install` | -| Accessible-CoAP | `scan_coap` | -| Accessible-CWMP | `scan_cwmp` | -| Accessible-MS-RDPEUDP | `scan_msrdpeudp` | -| Accessible-FTP | `scan_ftp` | -| Accessible-Hadoop | `scan_hadoop` | -| Accessible-HTTP | `scan_http` | -| Accessible-Radmin | `scan_radmin` | -| Accessible-RDP | `scan_rdp` | -| Accessible-Rsync | `scan_rsync` | -| Accessible-SMB | `scan_smb` | -| Accessible-Telnet | `scan_telnet` | -| Accessible-Ubiquiti-Discovery-Service | `scan_ubiquiti` | -| Accessible-VNC | `scan_vnc` | -| Blacklisted-IP (deprecated) | `blacklist` | -| Blocklist | `blocklist` | -| Compromised-Website| `compromised_website` | -| Device-Identification-IPv4 | `device_id` | -| Device-Identification-IPv6 | `device_id6` | -| DNS-Open-Resolvers | `scan_dns` | -| Honeypot-Amplification-DDoS-Events | `event4_honeypot_ddos_amp` | -| Honeypot-Brute-Force-Events | `event4_honeypot_brute_force` | -| Honeypot-Darknet | `event4_honeypot_darknet` | -| Honeypot-HTTP-Scan | `event4_honeypot_http_scan` | -| HTTP-Scanners | `hp_http_scan` | -| ICS-Scanners | `hp_ics_scan` | -| IP-Spoofer-Events | `event4_ip_spoofer` | -| Microsoft-Sinkhole-Events-IPv4 | `event4_microsoft_sinkhole` | -| Microsoft-Sinkhole-Events-HTTP | `event4_microsoft_sinkhole_http` | -| NTP-Monitor | `scan_ntpmonitor` | -| NTP-Version | `scan_ntp` | -| Open-Chargen | `scan_chargen` | -| Open-DB2-Discovery-Service | `scan_db2` | -| Open-Elasticsearch | `scan_elasticsearch` | -| Open-IPMI| `scan_ipmi` | -| Open-IPP | `scan_ipp` | -| Open-LDAP | `scan_ldap` | -| Open-LDAP-TCP | `scan_ldap_tcp` | -| Open-mDNS | `scan_mdns` | -| Open-Memcached | `scan_memcached` | -| Open-MongoDB | `scan_mongodb` | -| Open-MQTT | `scan_mqtt` | -| Open-MSSQL | `scan_mssql` | -| Open-NATPMP | `scan_nat_pmp` | -| Open-NetBIOS-Nameservice | `scan_netbios` | -| Open-Netis | `netis_router` | -| Open-Portmapper | `scan_portmapper` | -| Open-QOTD | `scan_qotd` | -| Open-Redis | `scan_redis` | -| Open-SNMP | `scan_snmp` | -| Open-SSDP | `scan_ssdp` | -| Open-TFTP | `scan_tftp` | -| Open-XDMCP | `scan_xdmcp` | -| Outdated-DNSSEC-Key| `outdated_dnssec_key` | -| Outdated-DNSSEC-Key-IPv6 | `outdated_dnssec_key_v6` | -| Sandbox-URL | `cwsandbox_url` | -| Sinkhole-DNS | `sinkhole_dns` | -| Sinkhole-Events | `event4_sinkhole` | -| Sinkhole-Events IPv4 | `event4_sinkhole` | -| Sinkhole-Events IPv6 | `event6_sinkhole` | -| Sinkhole-HTTP-Events | `event4_sinkhole_http`/`event6_sinkhole_http` | -| Sinkhole-HTTP-Events IPv4 | `event4_sinkhole_http` | -| Sinkhole-HTTP-Events IPv6 | `event6_sinkhole_http` | -| Sinkhole-Events-HTTP-Referer| `event4_sinkhole_http_referer`/`event6_sinkhole_http_referer` | -| Sinkhole-Events-HTTP-Referer IPv4 | `event4_sinkhole_http_referer` | -| Sinkhole-Events-HTTP-Referer IPv6 | `event6_sinkhole_http_referer` | -| Spam-URL | `spam_url` | -| SSL-FREAK-Vulnerable-Servers | `scan_ssl_freak` | -| SSL-POODLE-Vulnerable-Servers | `scan_ssl_poodle`/`scan6_ssl_poodle` | -| Vulnerable-Exchange-Server* | `scan_exchange` | -| Vulnerable-ISAKMP | `scan_isakmp` | -| Vulnerable-HTTP | `scan_http` | -| Vulnerable-SMTP | `scan_smtp_vulnerable` | - -\* This report can also contain data on active webshells (column `tag` is `exchange;webshell`), and are therefore not -only vulnerable but also actively infected. - -In addition, the following legacy reports are supported: - -| Legacy Report Type | Successor Report Type | File Name | -|--------------------|-----------------------|-----------| -| Amplification-DDoS-Victim | Honeypot-Amplification-DDoS-Events | `ddos_amplification` | -| CAIDA-IP-Spoofer | IP-Spoofer-Events | `caida_ip_spoofer` | -| Darknet | Honeypot-Darknet | `darknet` | -| Drone | Sinkhole-Events | `botnet_drone` | -| Drone-Brute-Force | Honeypot-Brute-Force-Events, Sinkhole-HTTP-Events | `drone_brute_force` | -| Microsoft-Sinkhole | Sinkhole-HTTP-Events | `microsoft_sinkhole` | -| Sinkhole-HTTP-Drone | Sinkhole-HTTP-Events | `sinkhole_http_drone` | -| IPv6-Sinkhole-HTTP-Drone | Sinkhole-HTTP-Events | `sinkhole6_http` | - -More information on these legacy reports can be found -in [Changes in Sinkhole and Honeypot Report Types and Formats](https://www.shadowserver.org/news/changes-in-sinkhole-and-honeypot-report-types-and-formats/) -. +The report configuration is stored in a `shadowserver-schema.json` file downloaded from https://interchange.shadowserver.org/intelmq/v1/schema. + +The parser will attempt to download a schema update on startup when the *auto_update* option is enabled. +Schema downloads can also be scheduled as a cron job for the `intelmq` user: + +```bash + 02 01 * * * intelmq.bots.parsers.shadowserver.parser --update-schema +``` + +For air-gapped systems automation will be required to download and copy the file to VAR_STATE_PATH/shadowserver-schema.json. + +The parser will automatically reload the configuration when the file changes. + + +**Schema contract** + +Once set in the schema, the `classification.identifier`, `classification.taxonomy`, and `classification.type` fields will remain static for a specific report. + +The schema revision history is maintained at https://github.com/The-Shadowserver-Foundation/report_schema/. + + +**Sample configuration** + +```yaml + shadowserver-parser: + bot_id: shadowserver-parser + name: Shadowserver Parser + enabled: true + group: Parser + groupname: parsers + module: intelmq.bots.parsers.shadowserver.parser + parameters: + destination_queues: + _default: [file-output-queue] + auto_update: true + run_mode: continuous +``` --- ### Shodan
From ac0447189f2f1afe679c7ec9bdc2a58b922decc2 Mon Sep 17 00:00:00 2001 From: elsif2 Date: Thu, 16 Nov 2023 15:53:55 +0000 Subject: [PATCH 67/67] Removed old unsorted doc and updated the taxonomy functions for the scan_exchange report. --- docs/unsorted/shadowserver.md | 24 -------------------- intelmq/bots/parsers/shadowserver/_config.py | 10 ++++---- 2 files changed, 5 insertions(+), 29 deletions(-) delete mode 100644 docs/unsorted/shadowserver.md diff --git a/docs/unsorted/shadowserver.md b/docs/unsorted/shadowserver.md deleted file mode 100644 index 1c7c2918e..000000000 --- a/docs/unsorted/shadowserver.md +++ /dev/null @@ -1,24 +0,0 @@ - - - -# Shadowserver Parser - -**Structure of this Parser Bot** - -The parser consists of two files: - -: - `_config.py` - -- `parser.py` or `parser_json.py` - -Both files are required for the parser to work properly. - -**Add new Feedformats** - -Add a new feed format and conversions if required to the file -`_config.py`. Don't forget to update the `mapping` dict. It is required to look up the correct configuration. - -Look at the documentation in the bot's `_config.py` file for more information. diff --git a/intelmq/bots/parsers/shadowserver/_config.py b/intelmq/bots/parsers/shadowserver/_config.py index c84bc7e85..6931e5410 100644 --- a/intelmq/bots/parsers/shadowserver/_config.py +++ b/intelmq/bots/parsers/shadowserver/_config.py @@ -263,19 +263,19 @@ def force_base64(value: Optional[str]) -> Optional[str]: def scan_exchange_taxonomy(field): - if field == 'exchange;webshell': + if 'webshell' in field: return 'intrusions' return 'vulnerable' def scan_exchange_type(field): - if field == 'exchange;webshell': + if 'webshell' in field: return 'system-compromise' - return 'infected-system' + return 'vulnerable-system' def scan_exchange_identifier(field): - if field == 'exchange;webshell': + if 'webshell' in field: return 'exchange-server-webshell' return 'vulnerable-exchange-server' @@ -321,7 +321,7 @@ def reload(): return else: if not __config.test_mode: - raise ValueError("The schema file does not exist.") + raise ValueError("The schema file does not exist: %r.", __config.schema_file) if __config.schema_mtime == 0.0 and mtime == 0.0 and __config.auto_update: update_schema()