From c828636b326cf0aa2e86bdcabc9b4268311ab1f5 Mon Sep 17 00:00:00 2001 From: mstopa-splunk <139441697+mstopa-splunk@users.noreply.github.com> Date: Mon, 5 Feb 2024 11:48:56 +0100 Subject: [PATCH] fix: fix app-syslog-pan_panos (#2322) --- .../conflib/syslog/app-syslog-pan_panos.conf | 54 ++++++++++++++++--- .../palo-alto/app-syslog-pan_panos.conf | 54 ++++++++++++++++--- tests/test_palo_alto.py | 47 +++++++++++++--- 3 files changed, 137 insertions(+), 18 deletions(-) diff --git a/package/etc/conf.d/conflib/syslog/app-syslog-pan_panos.conf b/package/etc/conf.d/conflib/syslog/app-syslog-pan_panos.conf index 11860f723e..12034fc193 100644 --- a/package/etc/conf.d/conflib/syslog/app-syslog-pan_panos.conf +++ b/package/etc/conf.d/conflib/syslog/app-syslog-pan_panos.conf @@ -107,11 +107,11 @@ block parser app-syslog-pan_panos() { flags(escape-double-char) ); }; - rewrite{ + rewrite{ r_set_splunk_dest_update_v2( class('correlation') sourcetype('pan:correlation') - ); + ); }; } elif (message(',USERID,' type(string) flags(substring))) { parser { @@ -131,6 +131,15 @@ block parser app-syslog-pan_panos() { ); }; } elif (message(',GLOBALPROTECT,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","log_type","future_use2","version","time_generated","vsys","event_id","stage","auth_method","tunnel_type","src_user","src_region","machine_name","public_ip","public_ipv6","private_ip","private_ipv6","host_id","serial_number","client_ver","client_os","client_os_ver","repeat_count","reason","error","opaque","status","location","login_duration","connect_method","error_code","portal","sequence_number","action_flags","event_time","selection_type","response_time","priority","attempted_gateways","gateway","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name","vsys_id") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; rewrite { r_set_splunk_dest_update_v2( index('netfw') @@ -138,6 +147,39 @@ block parser app-syslog-pan_panos() { sourcetype('pan:globalprotect') ); }; + } elif (message(',DECRYPTION,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","log_type","log_subtype","version","generated_time","src_ip","dest_ip","src_translated_ip","dest_translated_ip","rule","src_user","dest_user","app","vsys","src_zone","dest_zone","src_interface","dest_interface","log_forwarding_profile","start_time","session_id","repeat_count","src_port","dest_port","src_translated_port","dest_translated_port","flags","IP_PROTOCOL","action","tunnel_id","future_use2","future_use3","src_vm_uuid","dest_vm_uuid","uuid_rule","stage_client_firewall","stage_firewall_client","tls_version","key_exchange_algorithm","encryption_algorithm","hash_algorithm","rule","elliptic_curve","error_index","root_status","chain_status","proxy_type","cert_serial_number","fingerprint","cert_start_time","cert_end_time","cert_version","cert_size","cn_length","issuer_cn_length","root_cn_length","sni_length","cert_flags","subject_cn","issuer_subject_cn","root_subject_cn","server_name","error","container_id","pod_namespace","pod_name","src_edl","dest_edl","src_dag","dest_dag","timestamp","src_dvc_category","src_dvc_profile","src_dvc_model","src_dvc_vendor","src_dvc_os","src_dvc_os_version","src_name","src_mac","dest_dvc_category","dest_dvc_profile","dest_dvc_model","dest_dvc_vendor","dest_dvc_os","dest_dvc_os_version","dest_name","dest_mac","sequence_number","action_flags") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('decryption') + sourcetype('pan:decryption') + ); + }; + } elif (message(',AUTH,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","log_type","log_subtype","version","generated_time","vsys","src_ip","user","user_normalized","object","authentication_policy","repeast_count","authentication_id","pan_vendor","log_action","server_profile","description","client_type","event_type","factor_number","sequence_number","action_flags","device_group_hierarchy_1","device_group_hierarchy_2","device_group_hierarchy_3","device_group_hierarchy_4","vsys","dvc_name","vsys_id","authentication_protocol","rule","timestamp","src_host_category","src_host_profile","src_host_model","src_host_vendor","src_host_os_name","src_host_os_version","src_host","src_mac","region","future_use2","user_agent","session_id","cluster_name") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + index('netauth') + class('authentication') + sourcetype('pan:auth') + ); + }; } else { }; # Palo IETF (5424) event is entirely contained in $MESSAGE; for BSD format event needs to be constructed from @@ -162,19 +204,19 @@ block parser app-syslog-pan_panos() { }; }; application app-syslog-pan_panos-pgm[sc4s-syslog-pgm] { - filter { + filter { program('logforwarder' type(string)) ; - }; + }; parser { app-syslog-pan_panos(); }; }; application app-syslog-pan_panos[sc4s-syslog] { - filter { + filter { "${PROGRAM}" eq "" and message('1,' type(string) flags(prefix)) and message('^1,[^,]+,[^,]+,[A-Z]+\,') ; - }; + }; parser { app-syslog-pan_panos(); }; }; diff --git a/package/lite/etc/addons/palo-alto/app-syslog-pan_panos.conf b/package/lite/etc/addons/palo-alto/app-syslog-pan_panos.conf index 11860f723e..12034fc193 100644 --- a/package/lite/etc/addons/palo-alto/app-syslog-pan_panos.conf +++ b/package/lite/etc/addons/palo-alto/app-syslog-pan_panos.conf @@ -107,11 +107,11 @@ block parser app-syslog-pan_panos() { flags(escape-double-char) ); }; - rewrite{ + rewrite{ r_set_splunk_dest_update_v2( class('correlation') sourcetype('pan:correlation') - ); + ); }; } elif (message(',USERID,' type(string) flags(substring))) { parser { @@ -131,6 +131,15 @@ block parser app-syslog-pan_panos() { ); }; } elif (message(',GLOBALPROTECT,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","log_type","future_use2","version","time_generated","vsys","event_id","stage","auth_method","tunnel_type","src_user","src_region","machine_name","public_ip","public_ipv6","private_ip","private_ipv6","host_id","serial_number","client_ver","client_os","client_os_ver","repeat_count","reason","error","opaque","status","location","login_duration","connect_method","error_code","portal","sequence_number","action_flags","event_time","selection_type","response_time","priority","attempted_gateways","gateway","devicegroup_level1","devicegroup_level2","devicegroup_level3","devicegroup_level4","vsys_name","dvc_name","vsys_id") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; rewrite { r_set_splunk_dest_update_v2( index('netfw') @@ -138,6 +147,39 @@ block parser app-syslog-pan_panos() { sourcetype('pan:globalprotect') ); }; + } elif (message(',DECRYPTION,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","log_type","log_subtype","version","generated_time","src_ip","dest_ip","src_translated_ip","dest_translated_ip","rule","src_user","dest_user","app","vsys","src_zone","dest_zone","src_interface","dest_interface","log_forwarding_profile","start_time","session_id","repeat_count","src_port","dest_port","src_translated_port","dest_translated_port","flags","IP_PROTOCOL","action","tunnel_id","future_use2","future_use3","src_vm_uuid","dest_vm_uuid","uuid_rule","stage_client_firewall","stage_firewall_client","tls_version","key_exchange_algorithm","encryption_algorithm","hash_algorithm","rule","elliptic_curve","error_index","root_status","chain_status","proxy_type","cert_serial_number","fingerprint","cert_start_time","cert_end_time","cert_version","cert_size","cn_length","issuer_cn_length","root_cn_length","sni_length","cert_flags","subject_cn","issuer_subject_cn","root_subject_cn","server_name","error","container_id","pod_namespace","pod_name","src_edl","dest_edl","src_dag","dest_dag","timestamp","src_dvc_category","src_dvc_profile","src_dvc_model","src_dvc_vendor","src_dvc_os","src_dvc_os_version","src_name","src_mac","dest_dvc_category","dest_dvc_profile","dest_dvc_model","dest_dvc_vendor","dest_dvc_os","dest_dvc_os_version","dest_name","dest_mac","sequence_number","action_flags") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + class('decryption') + sourcetype('pan:decryption') + ); + }; + } elif (message(',AUTH,' type(string) flags(substring))) { + parser { + csv-parser( + columns("future_use1","receive_time","serial_number","log_type","log_subtype","version","generated_time","vsys","src_ip","user","user_normalized","object","authentication_policy","repeast_count","authentication_id","pan_vendor","log_action","server_profile","description","client_type","event_type","factor_number","sequence_number","action_flags","device_group_hierarchy_1","device_group_hierarchy_2","device_group_hierarchy_3","device_group_hierarchy_4","vsys","dvc_name","vsys_id","authentication_protocol","rule","timestamp","src_host_category","src_host_profile","src_host_model","src_host_vendor","src_host_os_name","src_host_os_version","src_host","src_mac","region","future_use2","user_agent","session_id","cluster_name") + prefix(".values.") + delimiters(',') + quote-pairs('""') + flags(escape-double-char) + ); + }; + rewrite { + r_set_splunk_dest_update_v2( + index('netauth') + class('authentication') + sourcetype('pan:auth') + ); + }; } else { }; # Palo IETF (5424) event is entirely contained in $MESSAGE; for BSD format event needs to be constructed from @@ -162,19 +204,19 @@ block parser app-syslog-pan_panos() { }; }; application app-syslog-pan_panos-pgm[sc4s-syslog-pgm] { - filter { + filter { program('logforwarder' type(string)) ; - }; + }; parser { app-syslog-pan_panos(); }; }; application app-syslog-pan_panos[sc4s-syslog] { - filter { + filter { "${PROGRAM}" eq "" and message('1,' type(string) flags(prefix)) and message('^1,[^,]+,[^,]+,[A-Z]+\,') ; - }; + }; parser { app-syslog-pan_panos(); }; }; diff --git a/tests/test_palo_alto.py b/tests/test_palo_alto.py index e33d4ca4b6..2ee073b6a7 100644 --- a/tests/test_palo_alto.py +++ b/tests/test_palo_alto.py @@ -259,7 +259,9 @@ def test_palo_alto_hipmatch(record_property, setup_splunk, setup_sc4s): def test_palo_alto_globalprotect( record_property, setup_splunk, setup_sc4s ): - host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + get_host_name = lambda: f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + orig_host = get_host_name() + overwritten_host_name = get_host_name() dt = datetime.datetime.now() _, bsd, time, _, tzoffset, _, epoch = time_operations(dt) @@ -270,21 +272,21 @@ def test_palo_alto_globalprotect( epoch = epoch[:-7] mt = env.from_string( - '{{ mark }} {{ bsd }} {{ host }} 1,{{ time }},012001006066,GLOBALPROTECT,0,2305,{{ time }},,gateway-hip-report,host-info,,,user,,SysAdmin,76.1.1.1,0.0.0.0,10.1.15.252,0.0.0.0,f8:ff:c2:47:4c:73,C02ZV00YP4G2,5.0.8,,"",1,,,"",success,,0,,0,opo-mgm-portal,93939,0x8000000000000000' + '{{ mark }} {{ bsd }} {{ orig_host }} 1,{{ time }},XXXXXXXXXXXXXXXXXX,GLOBALPROTECT,0,2561,{{ time }},vsys1,gateway-logout,logout,,,XXXXXXXX,XX,XXXXXXXXXXXXXX,8.8.8.8,0.0.0.0,192.0.0.1,0.0.0.0,XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX,XXXXXXXXXXXX,5.2.12,Windows,"Microsoft Windows 10 Enterprise , 64-bit",1,,,"client logout",success,,1554,,0,XXXXXXXXXXXXXXXXXXXX,XXXXXXXXXXXXXXXX,0x8000000000000000,2023-11-09T16:39:17.223+01:00,,,,,,13,19,52,450,,{{ overwritten_host_name }},1' + "\n" ) - message = mt.render(mark="<111>", bsd=bsd, host=host, time=time) + message = mt.render(mark="<111>", bsd=bsd, orig_host=orig_host, time=time, overwritten_host_name=overwritten_host_name) sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) st = env.from_string( - 'search _time={{ epoch }} index=netfw host="{{ host }}" sourcetype="pan:globalprotect"' + 'search _time={{ epoch }} index=netfw host={{ overwritten_host_name }} sourcetype="pan:globalprotect"' ) - search = st.render(epoch=epoch, host=host) + search = st.render(epoch=epoch, overwritten_host_name=overwritten_host_name) result_count, _ = splunk_single(setup_splunk, search) - record_property("host", host) + record_property("host", overwritten_host_name) record_property("resultCount", result_count) record_property("message", message) @@ -359,3 +361,36 @@ def test_palo_alto_system_futureproof( record_property("message", message) assert result_count == 1 + + +# <14>1 2023-07-06T19:20:22+00:00 DEVICE_NAME 1,{{ time }},007XXXXX341044,DECRYPTION,0,2562,{{ time }},XXX.XXX.XXX.XXX,XXX.XXX.XXX.XXX,XXX.XXX.XXX.XXX,XXX.XXX.XXX.XXX,XXX.XXX.XXX.XXX,AWS Services by URL - Egress,,,incomplete,vsys1,Default Zone,Default Zone,ethernet1/1,ethernet1/1,ANONYMIZED,{{ time }},504326,1,37612,443,0,0,0x1000000,tcp,allow,N/A,,,,,ANONYMIZED,Server_Hello_Done,Client_Hello,TLS1.2,ECDHE,AES_128_GCM,SHA256,ANONYMIZED,secp256r1,Certificate,trusted,Trusted,Forward,ANONYMIZED,XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,[DATE], [DATE],V3,2048,12,45,34,18,:::::RSA,*.badssl.com,ANONYMIZED,ANONYMIZED,expired.badssl.com,Received fatal alert CertificateExpired from client. CA Issuer URL (truncated):ANONYMIZED,[DATE-TIME],,,,,,,,,,ANONYMIZED,0x8000000000000000,29,82,454,0,,ANONYMIZED,1,unknown,unknown,unknown,1,,,incomplete,no,no +@mark.addons("paloalto") +def test_palo_alto_decryption(record_property, setup_splunk, setup_sc4s): + host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" + + dt = datetime.datetime.now() + _, bsd, time, _, _, _, epoch = time_operations(dt) + + # Tune time functions + time = dt.strftime("%Y/%m/%d %H:%M:%S") + epoch = epoch[:-7] + + mt = env.from_string( + '{{ mark }} {{ bsd }} {{ host }} 1,{{ time }},007XXXXX341044,DECRYPTION,0,2562,{{ time }},XXX.XXX.XXX.XXX,XXX.XXX.XXX.XXX,XXX.XXX.XXX.XXX,XXX.XXX.XXX.XXX,XXX.XXX.XXX.XXX,AWS Services by URL - Egress,,,incomplete,vsys1,Default Zone,Default Zone,ethernet1/1,ethernet1/1,ANONYMIZED,{{ time }},504326,1,37612,443,0,0,0x1000000,tcp,allow,N/A,,,,,ANONYMIZED,Server_Hello_Done,Client_Hello,TLS1.2,ECDHE,AES_128_GCM,SHA256,ANONYMIZED,secp256r1,Certificate,trusted,Trusted,Forward,ANONYMIZED,XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,[DATE], [DATE],V3,2048,12,45,34,18,:::::RSA,*.badssl.com,ANONYMIZED,ANONYMIZED,expired.badssl.com,Received fatal alert CertificateExpired from client. CA Issuer URL (truncated):ANONYMIZED,[DATE-TIME],,,,,,,,,,ANONYMIZED,0x8000000000000000,29,82,454,0,,ANONYMIZED,1,unknown,unknown,unknown,1,,,incomplete,no,no\n' + ) + message = mt.render(mark="<14>", bsd=bsd, host=host, time=time) + + sendsingle(message, setup_sc4s[0], setup_sc4s[1][514]) + + st = env.from_string( + 'search _time={{ epoch }} index=netops host="{{ host }}" sourcetype="pan:decryption"' + ) + search = st.render(epoch=epoch, host=host) + + result_count, _ = splunk_single(setup_splunk, search) + + record_property("host", host) + record_property("resultCount", result_count) + record_property("message", message) + + assert result_count == 1 \ No newline at end of file