Skip to content

Commit

Permalink
Akamai: fix pipeline error when converting empty field (#12275)
Browse files Browse the repository at this point in the history 
Fix pipeline error when converting an empty numerical field.
  • Loading branch information
@chemamartinez
chemamartinez authored Jan 8, 2025
1 parent b3417e8 commit de2fbe9
Show file tree
Hide file tree
Showing 5 changed files with 148 additions and 4 deletions.
5 changes: 5 additions & 0 deletions packages/akamai/changelog.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "2.27.1"
changes:
- description: Fix pipeline error when converting an empty numerical field.
type: bugfix
link: https://github.com/elastic/integrations/pull/12275
- version: "2.27.0"
changes:
- description: Add "preserve_original_event" tag to documents with `event.kind` set to "pipeline_error".
Expand Down
1 change: 1 addition & 0 deletions packages/akamai/data_stream/siem/_dev/test/pipeline/test-http-json.log
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,5 @@
{"attackData":{"clientIP":"81.2.69.144","configId":"92384723","policyId":"prkg_252151","ruleActions":"ZGVueQ%3d%3d","ruleData":"","ruleMessages":"VW5hdXRob3JpemVkIHBlZXIgSVAgMTM2LjI0NC45MC4xNzYgaW4gVmFuZ3VhcmQgLSBDU09DIEJsYWNrbGlzdA%3d%3d","ruleSelectors":"","ruleTags":"SVBCTE9DSw%3d%3d","ruleVersions":"","rules":"SVBCTE9DSw%3d%3d"},"format":"json","geo":{"asn":"20473","city":"FRANKFURT","continent":"EU","country":"DE","regionCode":"HE"},"httpMessage":{"bytes":"717","host":"cow.company.com","method":"POST","path":"/cgi-bin/%%32%65%%32%65/%%32%65%%32%65/%%32%65%%32%65/%%32%65%%32%65/%%32%65%%32%65/%%32%65%%32%65/%%32%65%%32%65/bin/sh","port":"443","protocol":"HTTP/1.1","requestHeaders":"Host%3a%20cow.company.com%0d%0aOrigin%3a%20https%3a%2f%2fcow.company.com%0d%0aContent-Type%3a%20application%2fx-www-form-urlencoded%0d%0a","requestId":"59ea711b","responseHeaders":"Server%3a%20AkamaiGHost%0d%0aMime-Version%3a%201.0%0d%0aContent-Type%3a%20text%2fhtml%0d%0aContent-Length%3a%20717%0d%0aExpires%3a%20Tue,%2008%20Oct%202024%2007%3a49%3a37%20GMT%0d%0aDate%3a%20Tue,%2008%20Oct%202024%2007%3a49%3a37%20GMT%0d%0aConnection%3a%20close%0d%0aAkamai-GRN%3a%200.50fbd217.1728373777.59ea711b%0d%0aStrict-Transport-Security%3a%20max-age%3d15768000%20%3b%20includeSubDomains%0d%0a","start":"1728373777","status":"403","tls":"tls1.3"},"type":"akamai_siem","version":"1.0"}
{"attackData":{"apiId":"API_234508975","apiKey":"","clientIP":"81.2.69.144","configId":"92384723","policyId":"prse_111965","ruleActions":"YWxlcnQ%3d%3bbW9uaXRvcg%3d%3d%3b","ruleData":"%3bZmVlZGZldGNoZXItZ29vZ2xl%3b","ruleMessages":"YXJlIHlvdSBzdGlsbCBsb29raW5nPw%3d%3d%3bUlNTIEZlZWQgUmVhZGVyIEJvdHM%3d%3b","ruleSelectors":"%3b%3b","ruleTags":"YWxlcnQ%3d%3bQUtBTUFJL0JPVC9BS0FNQUlfQ0FURUdPUklaRUQ%3d%3b","ruleVersions":"%3bMQ%3d%3d%3b","rules":"NjAxMDc4Njc%3d%3bMzk5MTAxNQ%3d%3d%3b"},"format":"json","geo":{"asn":"15169","city":"DALLAS","continent":"NA","country":"US","regionCode":"TX"},"httpMessage":{"bytes":"5405","host":"peeps.company.com","method":"GET","path":"/us/FundsRSS","port":"443","protocol":"HTTP/1.1","query":"FundId=%\u0026foo=1","requestHeaders":"Cache-Control%3a%20no-cache,max-age%3d0%0d%0aHost%3a%20peeps.company.com%0d%0aConnection%3a%20keep-alive%0d%0aAccept%3a%20*%2f*%0d%0aFrom%3a%20googlebot(at)googlebot.com%0d%0aUser-Agent%3a%20FeedFetcher-Google%3b%20(+http%3a%2f%2fwww.google.com%2ffeedfetcher.html)%0d%0aAccept-Encoding%3a%20gzip,%20deflate,%20br%0d%0a","requestId":"b7fb413c","responseHeaders":"Server%3a%20Company%20Server%0d%0aX-Frame-Options%3a%20SAMEORIGIN%0d%0aVary%3a%20accept-encoding%0d%0aContent-Encoding%3a%20gzip%0d%0aContent-Type%3a%20text%2fhtml%3bcharset%3dISO-8859-1%0d%0aContent-Length%3a%205405%0d%0aExpires%3a%20Mon,%2007%20Oct%202024%2007%3a00%3a47%20GMT%0d%0aCache-Control%3a%20max-age%3d0,%20no-cache,%20no-store%0d%0aDate%3a%20Mon,%2007%20Oct%202024%2007%3a00%3a47%20GMT%0d%0aConnection%3a%20keep-alive%0d%0a","start":"1728284447","status":"200","tls":"tls1.3"},"type":"akamai_siem","version":"1.0"}
{"attackData":{"clientIP":"81.2.69.144","configId":"92384723","policyId":"prkg_252151","ruleActions":"ZGVueQ%3d%3d","ruleData":"","ruleMessages":"VW5hdXRob3JpemVkIHBlZXIgSVAgNDUuNjEuMTg4LjEzMSBpbiBWYW5ndWFyZCAtIENTT0MgQmxhY2tsaXN0","ruleSelectors":"","ruleTags":"SVBCTE9DSw%3d%3d","ruleVersions":"","rules":"SVBCTE9DSw%3d%3d"},"format":"json","geo":{"asn":"53667","city":"MIAMI","continent":"NA","country":"US","regionCode":"FL"},"httpMessage":{"bytes":"424","host":"prodgateway.company.com","method":"GET","path":"/plugins/weathermap/editor.php","port":"443","protocol":"HTTP/1.1","query":"plug=0\u0026mapname=poc.conf\u0026action=set_map_properties\u0026param\u0026param2\u0026debug=existing\u0026node_name\u0026node_x\u0026node_y\u0026node_new_name\u0026node_label\u0026node_infourl\u0026node_hover\u0026node_iconfilename=--NONE--\u0026link_name\u0026link_bandwidth_in\u0026link_bandwidth_out\u0026link_target\u0026link_width\u0026link_infourl\u0026link_hover\u0026map_title=46ea1712d4b13b55b3f680cc5b8b54e8\u0026map_legend=Traffic+Load\u0026map_stamp=Created:+%b+%d+%Y+%H:%M:%S\u0026map_linkdefaultwidth=7","requestHeaders":"Host%3a%20prodgateway.company.com%0d%0aUser-Agent%3a%20Mozilla%2f5.0%20(Windows%20NT%2010.0%3b%20rv%3a109.0)%20Gecko%2f20100101%20Firefox%2f118.0%0d%0aConnection%3a%20close%0d%0aAccept%3a%20*%2f*%0d%0aAccept-Language%3a%20en%0d%0aAccept-Encoding%3a%20gzip%0d%0a","requestId":"115683da","responseHeaders":"Server%3a%20AkamaiGHost%0d%0aMime-Version%3a%201.0%0d%0aContent-Type%3a%20text%2fhtml%0d%0aContent-Length%3a%20424%0d%0aExpires%3a%20Sat,%2005%20Oct%202024%2017%3a59%3a35%20GMT%0d%0aDate%3a%20Sat,%2005%20Oct%202024%2017%3a59%3a35%20GMT%0d%0aConnection%3a%20close%0d%0aAkamai-GRN%3a%200.9b9a2d17.1728151175.115683da%0d%0aStrict-Transport-Security%3a%20max-age%3d15768000%20%3b%20includeSubDomains%0d%0a","start":"1728151175","status":"403","tls":"tls1.3"},"type":"akamai_siem","version":"1.0"}
{"attackData":{"slowPostRate":"","clientIP":"81.2.69.144","configId":"92384723","policyId":"prkg_252151","ruleActions":"ZGVueQ%3d%3d","ruleData":"","ruleMessages":"VW5hdXRob3JpemVkIHBlZXIgSVAgNDUuNjEuMTg4LjEzMSBpbiBWYW5ndWFyZCAtIENTT0MgQmxhY2tsaXN0","ruleSelectors":"","ruleTags":"SVBCTE9DSw%3d%3d","ruleVersions":"","rules":"SVBCTE9DSw%3d%3d"},"format":"json","geo":{"asn":"","city":"MIAMI","continent":"NA","country":"US","regionCode":"FL"},"httpMessage":{"bytes":"424","host":"prodgateway.company.com","method":"GET","path":"/plugins/weathermap/editor.php","port":"443","protocol":"HTTP/1.1","query":"plug=0\u0026mapname=poc.conf\u0026action=set_map_properties\u0026param\u0026param2\u0026debug=existing\u0026node_name\u0026node_x\u0026node_y\u0026node_new_name\u0026node_label\u0026node_infourl\u0026node_hover\u0026node_iconfilename=--NONE--\u0026link_name\u0026link_bandwidth_in\u0026link_bandwidth_out\u0026link_target\u0026link_width\u0026link_infourl\u0026link_hover\u0026map_title=46ea1712d4b13b55b3f680cc5b8b54e8\u0026map_legend=Traffic+Load\u0026map_stamp=Created:+%b+%d+%Y+%H:%M:%S\u0026map_linkdefaultwidth=7","requestHeaders":"Host%3a%20prodgateway.company.com%0d%0aUser-Agent%3a%20Mozilla%2f5.0%20(Windows%20NT%2010.0%3b%20rv%3a109.0)%20Gecko%2f20100101%20Firefox%2f118.0%0d%0aConnection%3a%20close%0d%0aAccept%3a%20*%2f*%0d%0aAccept-Language%3a%20en%0d%0aAccept-Encoding%3a%20gzip%0d%0a","requestId":"115683da","responseHeaders":"Server%3a%20AkamaiGHost%0d%0aMime-Version%3a%201.0%0d%0aContent-Type%3a%20text%2fhtml%0d%0aContent-Length%3a%20424%0d%0aExpires%3a%20Sat,%2005%20Oct%202024%2017%3a59%3a35%20GMT%0d%0aDate%3a%20Sat,%2005%20Oct%202024%2017%3a59%3a35%20GMT%0d%0aConnection%3a%20close%0d%0aAkamai-GRN%3a%200.9b9a2d17.1728151175.115683da%0d%0aStrict-Transport-Security%3a%20max-age%3d15768000%20%3b%20includeSubDomains%0d%0a","start":"1728151175","status":"403","tls":"tls1.3"},"type":"akamai_siem","version":"1.0"}
{"total":10000,"offset":"71cca;3phZmEdPj6YEqml0rvbdWDZGW3mCiJIwjyhkJfsLFM2gVYPgE8-N_0CiLI9gwH0_4OJ87xDQ3b-gIsx_kEBdf7aaC_AvDpG9fMxypeaCma10FKrY9VKE","limit":10000}
126 changes: 125 additions & 1 deletion packages/akamai/data_stream/siem/_dev/test/pipeline/test-http-json.log-expected.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1907,6 +1907,130 @@
"query": "plug=0&mapname=poc.conf&action=set_map_properties&param&param2&debug=existing&node_name&node_x&node_y&node_new_name&node_label&node_infourl&node_hover&node_iconfilename=--NONE--&link_name&link_bandwidth_in&link_bandwidth_out&link_target&link_width&link_infourl&link_hover&map_title=46ea1712d4b13b55b3f680cc5b8b54e8&map_legend=Traffic+Load&map_stamp=Created:+%b+%d+%Y+%H:%M:%S&map_linkdefaultwidth=7"
}
},
{
"@timestamp": "2024-10-05T17:59:35.000Z",
"akamai": {
"siem": {
"config_id": "92384723",
"policy_id": "prkg_252151",
"request": {
"headers": {
"Accept": "*/*",
"Accept-Encoding": "gzip",
"Accept-Language": "en",
"Connection": "close",
"Host": "prodgateway.company.com",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/118.0"
}
},
"response": {
"headers": {
"Akamai-GRN": "0.9b9a2d17.1728151175.115683da",
"Connection": "close",
"Content-Length": "424",
"Content-Type": "text/html",
"Date": "Sat, 05 Oct 2024 17:59:35 GMT",
"Expires": "Sat, 05 Oct 2024 17:59:35 GMT",
"Mime-Version": "1.0",
"Server": "AkamaiGHost",
"Strict-Transport-Security": "max-age=15768000 ; includeSubDomains"
}
},
"rule_actions": [
"deny"
],
"rule_tags": [
"ipblock"
],
"rules": [
{
"ruleActions": "deny",
"ruleMessages": "Unauthorized peer IP 45.61.188.131 in Vanguard - CSOC Blacklist",
"ruleTags": "IPBLOCK",
"rules": "IPBLOCK"
}
]
}
},
"client": {
"address": "81.2.69.144",
"geo": {
"city_name": "London",
"continent_name": "Europe",
"country_iso_code": "GB",
"country_name": "United Kingdom",
"location": {
"lat": 51.5142,
"lon": -0.0931
},
"region_iso_code": "GB-ENG",
"region_name": "England"
},
"ip": "81.2.69.144"
},
"ecs": {
"version": "8.11.0"
},
"event": {
"category": [
"network"
],
"id": "115683da",
"kind": "event",
"original": "{\"attackData\":{\"slowPostRate\":\"\",\"clientIP\":\"81.2.69.144\",\"configId\":\"92384723\",\"policyId\":\"prkg_252151\",\"ruleActions\":\"ZGVueQ%3d%3d\",\"ruleData\":\"\",\"ruleMessages\":\"VW5hdXRob3JpemVkIHBlZXIgSVAgNDUuNjEuMTg4LjEzMSBpbiBWYW5ndWFyZCAtIENTT0MgQmxhY2tsaXN0\",\"ruleSelectors\":\"\",\"ruleTags\":\"SVBCTE9DSw%3d%3d\",\"ruleVersions\":\"\",\"rules\":\"SVBCTE9DSw%3d%3d\"},\"format\":\"json\",\"geo\":{\"asn\":\"\",\"city\":\"MIAMI\",\"continent\":\"NA\",\"country\":\"US\",\"regionCode\":\"FL\"},\"httpMessage\":{\"bytes\":\"424\",\"host\":\"prodgateway.company.com\",\"method\":\"GET\",\"path\":\"/plugins/weathermap/editor.php\",\"port\":\"443\",\"protocol\":\"HTTP/1.1\",\"query\":\"plug=0\\u0026mapname=poc.conf\\u0026action=set_map_properties\\u0026param\\u0026param2\\u0026debug=existing\\u0026node_name\\u0026node_x\\u0026node_y\\u0026node_new_name\\u0026node_label\\u0026node_infourl\\u0026node_hover\\u0026node_iconfilename=--NONE--\\u0026link_name\\u0026link_bandwidth_in\\u0026link_bandwidth_out\\u0026link_target\\u0026link_width\\u0026link_infourl\\u0026link_hover\\u0026map_title=46ea1712d4b13b55b3f680cc5b8b54e8\\u0026map_legend=Traffic+Load\\u0026map_stamp=Created:+%b+%d+%Y+%H:%M:%S\\u0026map_linkdefaultwidth=7\",\"requestHeaders\":\"Host%3a%20prodgateway.company.com%0d%0aUser-Agent%3a%20Mozilla%2f5.0%20(Windows%20NT%2010.0%3b%20rv%3a109.0)%20Gecko%2f20100101%20Firefox%2f118.0%0d%0aConnection%3a%20close%0d%0aAccept%3a%20*%2f*%0d%0aAccept-Language%3a%20en%0d%0aAccept-Encoding%3a%20gzip%0d%0a\",\"requestId\":\"115683da\",\"responseHeaders\":\"Server%3a%20AkamaiGHost%0d%0aMime-Version%3a%201.0%0d%0aContent-Type%3a%20text%2fhtml%0d%0aContent-Length%3a%20424%0d%0aExpires%3a%20Sat,%2005%20Oct%202024%2017%3a59%3a35%20GMT%0d%0aDate%3a%20Sat,%2005%20Oct%202024%2017%3a59%3a35%20GMT%0d%0aConnection%3a%20close%0d%0aAkamai-GRN%3a%200.9b9a2d17.1728151175.115683da%0d%0aStrict-Transport-Security%3a%20max-age%3d15768000%20%3b%20includeSubDomains%0d%0a\",\"start\":\"1728151175\",\"status\":\"403\",\"tls\":\"tls1.3\"},\"type\":\"akamai_siem\",\"version\":\"1.0\"}",
"start": "2024-10-05T17:59:35.000Z"
},
"http": {
"request": {
"id": "115683da",
"method": "GET"
},
"response": {
"bytes": 424,
"status_code": 403
},
"version": "1.1"
},
"network": {
"protocol": "http",
"transport": "tcp"
},
"observer": {
"type": "proxy",
"vendor": "akamai"
},
"related": {
"ip": [
"81.2.69.144"
]
},
"source": {
"address": "81.2.69.144",
"geo": {
"city_name": "London",
"continent_name": "Europe",
"country_iso_code": "GB",
"country_name": "United Kingdom",
"location": {
"lat": 51.5142,
"lon": -0.0931
},
"region_iso_code": "GB-ENG",
"region_name": "England"
},
"ip": "81.2.69.144"
},
"tags": [
"preserve_original_event"
],
"url": {
"domain": "prodgateway.company.com",
"full": "prodgateway.company.com/plugins/weathermap/editor.php",
"path": "/plugins/weathermap/editor.php",
"port": 443,
"query": "plug=0&mapname=poc.conf&action=set_map_properties&param&param2&debug=existing&node_name&node_x&node_y&node_new_name&node_label&node_infourl&node_hover&node_iconfilename=--NONE--&link_name&link_bandwidth_in&link_bandwidth_out&link_target&link_width&link_infourl&link_hover&map_title=46ea1712d4b13b55b3f680cc5b8b54e8&map_legend=Traffic+Load&map_stamp=Created:+%b+%d+%Y+%H:%M:%S&map_linkdefaultwidth=7"
}
},
null
]
}
}
18 changes: 16 additions & 2 deletions packages/akamai/data_stream/siem/elasticsearch/ingest_pipeline/default.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,8 @@ processors:
target_field: source.as.number
type: long
ignore_missing: true
if: ctx.source?.as?.number == null
if: ctx.json.geo?.asn != '' && ctx.source?.as?.number == null
tag: convert_source_as_number
- rename:
field: source.as.organization_name
target_field: source.as.organization.name
Expand Down Expand Up @@ -388,6 +389,8 @@ processors:
target_field: akamai.siem.slow_post_rate
type: long
ignore_missing: true
if: ctx.json.attackData?.slowPostRate != ''
tag: convert_slow_post_rate
- rename:
field: json.attackData.clientReputation
target_field: akamai.siem.client_reputation
Expand All @@ -402,11 +405,15 @@ processors:
target_field: akamai.siem.bot.score
type: long
ignore_missing: true
if: ctx.json.botData?.botScore != ''
tag: convert_bot_score
- convert:
field: json.botData.responseSegment
target_field: akamai.siem.bot.response_segment
type: long
ignore_missing: true
if: ctx.json.botData?.responseSegment != ''
tag: convert_bot_response_segment
## Client Data
- rename:
field: json.clientData.appBundleId
Expand All @@ -421,6 +428,8 @@ processors:
target_field: akamai.siem.client_data.telemetry_type
type: long
ignore_missing: true
if: ctx.json.clientData?.telemetryType != ''
tag: convert_telemetry_type
- rename:
field: json.clientData.sdkVersion
target_field: akamai.siem.client_data.sdk_version
Expand All @@ -435,17 +444,22 @@ processors:
target_field: akamai.siem.user_risk.status
type: long
ignore_missing: true
if: ctx.json.userRiskData?.status != ''
tag: convert_user_risk_status
- convert:
field: json.userRiskData.score
target_field: akamai.siem.user_risk.score
type: long
if: ctx.json.userRiskData?.score != null && ctx.json.userRiskData.score != ''
ignore_missing: true
if: ctx.json.userRiskData?.score != ''
tag: convert_user_risk_score
- convert:
field: json.userRiskData.allow
target_field: akamai.siem.user_risk.allow
type: long
ignore_missing: true
if: ctx.json.userRiskData?.allow != ''
tag: convert_user_risk_allow
- kv:
if: ctx.json.userRiskData?.risk != ""
tag: kv_userRiskData_risk
Expand Down
2 changes: 1 addition & 1 deletion packages/akamai/manifest.yml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
name: akamai
title: Akamai
version: "2.27.0"
version: "2.27.1"
description: Collect logs from Akamai with Elastic Agent.
type: integration
format_version: "3.0.2"
Expand Down

0 comments on commit de2fbe9

Please sign in to comment.