diff --git a/aws/plugin.go b/aws/plugin.go index 36395c9..e19d457 100755 --- a/aws/plugin.go +++ b/aws/plugin.go @@ -7,6 +7,7 @@ import ( "github.com/turbot/tailpipe-plugin-aws/tables/alb_access_log" "github.com/turbot/tailpipe-plugin-aws/tables/cloudtrail_log" "github.com/turbot/tailpipe-plugin-aws/tables/nlb_access_log" + "github.com/turbot/tailpipe-plugin-aws/tables/network_firewall_log" "github.com/turbot/tailpipe-plugin-aws/tables/s3_server_access_log" "github.com/turbot/tailpipe-plugin-aws/tables/vpc_flow_log" "github.com/turbot/tailpipe-plugin-aws/tables/waf_traffic_log" @@ -29,6 +30,7 @@ func init() { table.RegisterTable[*s3_server_access_log.S3ServerAccessLog, *s3_server_access_log.S3ServerAccessLogTable]() table.RegisterTable[*vpc_flow_log.VpcFlowLog, *vpc_flow_log.VpcFlowLogTable]() table.RegisterTable[*waf_traffic_log.WafTrafficLog, *waf_traffic_log.WafTrafficLogTable]() + table.RegisterTable[*network_firewall_log.NetworkFirewallLog, *network_firewall_log.NetworkFirewallLogTable]() // register sources row_source.RegisterRowSource[*s3_bucket.AwsS3BucketSource]() diff --git a/docs/tables/aws_network_firewall_log/index.md b/docs/tables/aws_network_firewall_log/index.md new file mode 100644 index 0000000..03903d5 --- /dev/null +++ b/docs/tables/aws_network_firewall_log/index.md @@ -0,0 +1,233 @@ +--- +title: "Tailpipe Table: aws_network_firewall_log - Query AWS Network Firewall Logs" +description: "AWS Network Firewall logs capture information about traffic flowing through your AWS Network Firewall, including flow logs and alert information." +--- + +# Table: aws_network_firewall_log - Query AWS Network Firewall Logs + +The `aws_network_firewall_log` table allows you to query data from AWS Network Firewall logs. This table provides detailed insights into the traffic flowing through your AWS Network Firewall, including source and destination IP addresses, ports, protocols, and alert information. + +## Configure + +Create a [partition](https://tailpipe.io/docs/manage/partition) for `aws_network_firewall_log` ([examples](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_network_firewall_log#example-configurations)): + +```sh +vi ~/.tailpipe/config/aws.tpc +``` + +```hcl +connection "aws" "network_firewall_logging" { + profile = "my-network-firewall-logging" +} + +partition "aws_network_firewall_log" "my_logs" { + source "aws_s3_bucket" { + connection = connection.aws.network_firewall_logging + bucket = "aws-network-firewall-logs-bucket" + } +} +``` + +## Collect + +[Collect](https://tailpipe.io/docs/manage/collection) logs for all `aws_network_firewall_log` partitions: + +```sh +tailpipe collect aws_network_firewall_log +``` + +Or for a single partition: + +```sh +tailpipe collect aws_network_firewall_log.my_logs +``` + +## Query + +**[Explore example queries for this table →](https://hub.tailpipe.io/plugins/turbot/aws/queries/aws_network_firewall_log)** + +### Traffic by Protocol + +Analyze traffic by protocol to understand the distribution of network traffic. + +```sql +select + (event ->> 'proto') as protocol, + count(*) as traffic_count +from + aws_network_firewall_log +group by + protocol +order by + traffic_count desc; +``` + +### Destination Port Analysis + +Identify the most commonly accessed destination ports to understand traffic patterns. + +```sql +select + (event ->> 'dest_port') as destination_port, + count(*) as connection_count +from + aws_network_firewall_log +where + (event ->> 'dest_port') is not null +group by + destination_port +order by + connection_count desc +limit 10; +``` + +### Traffic by Application Protocol + +Analyze traffic by application protocol to understand what types of applications are generating network traffic. + +```sql +select + (event ->> 'app_proto') as app_protocol, + count(*) as connection_count +from + aws_network_firewall_log +group by + app_protocol +order by + connection_count desc; +``` + +### Recent Firewall Logs + +View the most recent firewall logs to analyze recent network traffic. + +```sql +select + event_timestamp, + (event ->> 'src_ip') as source_ip, + (event ->> 'src_port') as source_port, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'dest_port') as destination_port, + (event ->> 'proto') as protocol, + (event ->> 'app_proto') as app_protocol, + firewall_name, + availability_zone +from + aws_network_firewall_log +order by + event_timestamp desc +limit 20; +``` + +## Example Configurations + +### Collect logs from an S3 bucket + +Collect Network Firewall logs stored in an S3 bucket. + +```hcl +connection "aws" "network_firewall_logging" { + profile = "my-network-firewall-logging" +} + +partition "aws_network_firewall_log" "my_logs" { + source "aws_s3_bucket" { + connection = connection.aws.network_firewall_logging + bucket = "aws-network-firewall-logs-bucket" + } +} +``` + +### Collect logs from an S3 bucket with a prefix + +Collect Network Firewall logs stored in an S3 bucket using a prefix. + +```hcl +partition "aws_network_firewall_log" "my_logs_prefix" { + source "aws_s3_bucket" { + connection = connection.aws.network_firewall_logging + bucket = "aws-network-firewall-logs-bucket" + prefix = "my/prefix/" + } +} +``` + +### Collect logs from local files + +You can also collect Network Firewall logs from local files. + +```hcl +partition "aws_network_firewall_log" "local_logs" { + source "file" { + paths = ["/Users/myuser/network_firewall_logs"] + file_layout = "%{DATA}.json.gz" + } +} +``` + +### Collect logs for all accounts in an organization + +For a specific organization, collect logs for all accounts and regions. + +```hcl +partition "aws_network_firewall_log" "my_logs_org" { + source "aws_s3_bucket" { + connection = connection.aws.network_firewall_logging + bucket = "network-firewall-logs-bucket" + file_layout = "AWSLogs/o-aa111bb222/%{NUMBER:account_id}/network-firewall/(?flow|alert|tls)/%{DATA:region}/%{DATA:firewall_name}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}/%{NUMBER:account_id}_network-firewall_%{DATA:log_type}_%{DATA:region}_%{DATA:firewall_name}_%{DATA:timestamp}_%{DATA:hash}.log.gz" + } +} +``` + +### Collect logs for a single account + +For a specific account, collect logs for all regions. + +```hcl +partition "aws_network_firewall_log" "my_logs_account" { + source "aws_s3_bucket" { + connection = connection.aws.network_firewall_logging + bucket = "network-firewall-logs-bucket" + file_layout = "AWSLogs/(%{DATA:org_id}/)?123456789012/network-firewall/(?flow|alert|tls)/%{DATA:region}/%{DATA:firewall_name}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}/123456789012_network-firewall_%{DATA:log_type}_%{DATA:region}_%{DATA:firewall_name}_%{DATA:timestamp}_%{DATA:hash}.log.gz" + } +} +``` + +### Collect logs for a single region + +For all accounts, collect logs from `us-east-1`. + +```hcl +partition "aws_network_firewall_log" "my_logs_region" { + source "aws_s3_bucket" { + connection = connection.aws.network_firewall_logging + bucket = "network-firewall-logs-bucket" + file_layout = "AWSLogs/(%{DATA:org_id}/)?%{NUMBER:account_id}/network-firewall/(?flow|alert|tls)/us-east-1/%{DATA:firewall_name}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}/%{NUMBER:account_id}_network-firewall_%{DATA:log_type}_us-east-1_%{DATA:firewall_name}_%{DATA:timestamp}_%{DATA:hash}.log.gz" + } +} +``` + +### Filter for specific traffic type + +Use the filter argument in your partition to focus on specific traffic types. + +```hcl +partition "aws_network_firewall_log" "my_logs_filtered" { + filter = "event->>'app_proto' = 'tls'" + + source "aws_s3_bucket" { + connection = connection.aws.network_firewall_logging + bucket = "network-firewall-logs-bucket" + } +} +``` + +## Source Defaults + +### aws_s3_bucket + +This table sets the following defaults for the [aws_s3_bucket source](https://hub.tailpipe.io/plugins/turbot/aws/sources/aws_s3_bucket#arguments): + +| Argument | Default | +|--------------|---------| +| file_layout | `AWSLogs/(%{DATA:org_id}/)?%{NUMBER:account_id}/network-firewall/(?flow|alert|tls)/%{DATA:region}/%{DATA:firewall_name}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}/%{NUMBER:account_id}_network-firewall_%{DATA:log_type}_%{DATA:region}_%{DATA:firewall_name}_%{DATA:timestamp}_%{DATA:hash}.log.gz` | diff --git a/docs/tables/aws_network_firewall_log/queries.md b/docs/tables/aws_network_firewall_log/queries.md new file mode 100644 index 0000000..a7b1dc7 --- /dev/null +++ b/docs/tables/aws_network_firewall_log/queries.md @@ -0,0 +1,312 @@ +## Activity Examples + +### Daily Network Firewall Activity Trends + +Count Network Firewall log entries per day to identify network activity trends. This helps monitor overall firewall activity and detect unusual spikes in traffic. + +```sql +select + strftime(event_timestamp, '%Y-%m-%d') as traffic_date, + count(*) as log_count +from + aws_network_firewall_log +group by + traffic_date +order by + traffic_date asc; +``` + +```yaml +folder: Network Firewall +``` + +### Top 10 Source IPs Generating Traffic + +Identify the top 10 source IP addresses that generated the most network traffic. This helps detect potential high-traffic sources that may indicate misconfigured applications or suspicious activity. + +```sql +select + tp_source_ip, + count(*) as log_count +from + aws_network_firewall_log +where + tp_source_ip is not null +group by + tp_source_ip +order by + log_count desc +limit 10; +``` + +```yaml +folder: Network Firewall +``` + +### Traffic Distribution by Protocol + +Analyze traffic distribution by protocol to understand what types of network protocols are most frequently used in your environment. + +```sql +select + (event ->> 'proto') as protocol, + count(*) as traffic_count +from + aws_network_firewall_log +group by + protocol +order by + traffic_count desc; +``` + +```yaml +folder: Network Firewall +``` + +## Detection Examples + +### Identify Traffic from a Suspicious IP + +Check if a specific IP is sending or receiving traffic. This is useful for investigating potential threats or monitoring known suspicious IPs. + +```sql +select + event_timestamp, + (event ->> 'src_ip') as source_ip, + (event ->> 'src_port') as source_port, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'dest_port') as destination_port, + (event ->> 'proto') as protocol, + firewall_name, + availability_zone +from + aws_network_firewall_log +where + (event ->> 'src_ip') = '192.0.2.100' + or (event ->> 'dest_ip') = '192.0.2.100' +order by + event_timestamp desc; +``` + +```yaml +folder: Network Firewall +``` + +### Detect Potentially Suspicious DNS Traffic + +Identify DNS traffic that might indicate command and control (C2) communications or data exfiltration attempts over DNS. + +```sql +select + event_timestamp, + (event ->> 'src_ip') as source_ip, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'dest_port') as destination_port, + (event ->> 'proto') as protocol, + firewall_name, + availability_zone +from + aws_network_firewall_log +where + (event ->> 'app_proto') = 'dns' + and (event ->> 'dest_port') = '53' + and (event ->> 'proto') = 'UDP' +order by + event_timestamp desc; +``` + +```yaml +folder: Network Firewall +``` + +### Monitor Network Firewall Alerts + +Identify and analyze alert events generated by the Network Firewall. This helps monitor security rules and detect potential threats. + +```sql +select + event_timestamp, + (event ->> 'src_ip') as source_ip, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'alert' ->> 'action') as alert_action, + (event ->> 'alert' ->> 'signature') as alert_signature, + (event ->> 'alert' ->> 'severity') as alert_severity, + (event ->> 'alert' ->> 'category') as alert_category, + firewall_name, + availability_zone +from + aws_network_firewall_log +where + (event ->> 'alert') is not null +order by + event_timestamp desc; +``` + +```yaml +folder: Network Firewall +``` + +## Operational Examples + +### Monitor Traffic Through Specific Firewall + +Retrieve network traffic flows through a specific Network Firewall. This helps analyze firewall behavior and troubleshoot connectivity issues. + +```sql +select + event_timestamp, + (event ->> 'src_ip') as source_ip, + (event ->> 'src_port') as source_port, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'dest_port') as destination_port, + (event ->> 'proto') as protocol, + (event ->> 'app_proto') as app_protocol, + firewall_name, + availability_zone +from + aws_network_firewall_log +where + firewall_name = 'main-firewall' +order by + event_timestamp desc +limit 100; +``` + +```yaml +folder: Network Firewall +``` + +### Netflow Traffic Analysis + +Analyze netflow data to understand traffic patterns, including bytes transferred and packet counts. + +```sql +select + event_timestamp, + (event ->> 'src_ip') as source_ip, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'proto') as protocol, + (event ->> 'netflow' ->> 'bytes') as bytes, + (event ->> 'netflow' ->> 'packets') as packets, + (event ->> 'netflow' ->> 'start_time') as start_time, + (event ->> 'netflow' ->> 'end_time') as end_time, + firewall_name, + availability_zone +from + aws_network_firewall_log +where + (event ->> 'event_type') = 'netflow' + and (event ->> 'netflow') is not null +order by + event_timestamp desc +limit 100; +``` + +```yaml +folder: Network Firewall +``` + +### TLS Connection Analysis + +Monitor TLS connections and any related errors to identify potential SSL/TLS issues or suspicious encrypted traffic. + +```sql +select + event_timestamp, + (event ->> 'src_ip') as source_ip, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'sni') as server_name_indication, + (event ->> 'tls_inspected') as tls_inspected, + (event ->> 'tls_error' ->> 'error_message') as tls_error, + firewall_name, + availability_zone +from + aws_network_firewall_log +where + (event ->> 'app_proto') = 'tls' +order by + event_timestamp desc; +``` + +```yaml +folder: Network Firewall +``` + +## Volume Examples + +### Unusually Large Data Transfers + +Identify unusually large data transfers based on bytes transferred. This helps detect potential data exfiltration or abnormal network usage. + +```sql +select + event_timestamp, + (event ->> 'src_ip') as source_ip, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'netflow' ->> 'bytes') as bytes, + (event ->> 'netflow' ->> 'packets') as packets, + (event ->> 'proto') as protocol, + firewall_name, + availability_zone +from + aws_network_firewall_log +where + (event ->> 'netflow' ->> 'bytes')::bigint > 1000000 -- 1MB +order by + bytes desc; +``` + +```yaml +folder: Network Firewall +``` + +### High-Volume Network Traffic by Source IP + +Find network sources generating a high number of connections, which helps detect possible denial-of-service (DoS) attacks or heavy application usage. + +```sql +select + (event ->> 'src_ip') as source_ip, + count(*) as connection_count, + date_trunc('hour', event_timestamp) as traffic_hour +from + aws_network_firewall_log +where + event ->> 'src_ip' is not null +group by + source_ip, + traffic_hour +having + count(*) > 50 +order by + connection_count desc; +``` + +```yaml +folder: Network Firewall +``` + +### Destination Port Usage Analysis + +Analyze traffic by destination port to understand which services are most frequently accessed across your network. + +```sql +select + (event ->> 'dest_port') as destination_port, + (event ->> 'app_proto') as app_protocol, + count(*) as connection_count +from + aws_network_firewall_log +where + (event ->> 'dest_port') is not null +group by + destination_port, + app_protocol +order by + connection_count desc +limit 20; +``` + +```yaml +folder: Network Firewall +``` diff --git a/tables/network_firewall_log/network_firewall_log.go b/tables/network_firewall_log/network_firewall_log.go new file mode 100644 index 0000000..028d62c --- /dev/null +++ b/tables/network_firewall_log/network_firewall_log.go @@ -0,0 +1,76 @@ +package network_firewall_log + +import ( + "time" + + "github.com/turbot/tailpipe-plugin-sdk/schema" +) + +type NetworkFirewallAlert struct { + Action string `json:"action,omitempty"` + SignatureID int `json:"signature_id,omitempty"` + Rev int `json:"rev,omitempty"` + Signature string `json:"signature,omitempty"` + Category string `json:"category,omitempty"` + Severity int `json:"severity,omitempty"` +} + +type NetworkFirewallNetflow struct { + StartTime *time.Time `json:"start_time,omitempty"` + EndTime *time.Time `json:"end_time,omitempty"` + Bytes int `json:"bytes,omitempty"` + Packets int `json:"packets,omitempty"` + Age int `json:"age,omitempty"` + MinTtl int `json:"min_ttl,omitempty"` + MaxTtl int `json:"max_ttl,omitempty"` +} + +type RevocationCheck struct { + LeafCertFpr string `json:"leaf_cert_fpr,omitempty"` + Status string `json:"status,omitempty"` + Action string `json:"action,omitempty"` +} + +type TLSError struct { + ErrorMsg string `json:"error_message"` +} + +type NetworkFirewallEvent struct { + Timestamp *time.Time `json:"timestamp"` + FlowID int64 `json:"flow_id"` + EventType string `json:"event_type"` + SrcIP string `json:"src_ip"` + SrcPort int `json:"src_port"` + DestIP string `json:"dest_ip"` + DestPort int `json:"dest_port"` + Sni string `json:"sni,omitempty"` + Proto string `json:"proto,omitempty"` + AppProto *string `json:"app_proto,omitempty"` + Alert *NetworkFirewallAlert `json:"alert,omitempty"` + Netflow *NetworkFirewallNetflow `json:"netflow,omitempty"` + TLSInspected *bool `json:"tls_inspected,omitempty"` + TLSError *TLSError `json:"tls_error,omitempty"` + RevocationCheck *RevocationCheck `json:"revocation_check,omitempty"` +} + +type NetworkFirewallLog struct { + schema.CommonFields + + FirewallName string `json:"firewall_name"` + AvailabilityZone string `json:"availability_zone"` + EventTimestamp *time.Time `json:"event_timestamp"` + Event NetworkFirewallEvent `json:"event"` +} + +func (n *NetworkFirewallLog) GetColumnDescriptions() map[string]string { + return map[string]string{ + "firewall_name": "The name of the firewall associated with the log entry.", + "availability_zone": "The Availability Zone of the firewall endpoint that generated the log entry.", + "event_timestamp": "The epoch timestamp (in seconds) when the log was created (UTC).", + "event": "Detailed event information in Suricata EVE JSON format, including human-readable timestamp, event type, network packet details, and alert details if applicable.", + // Override table specific tp_* column descriptions + "tp_index": "The name of the firewall associated with the log entry.", + "tp_ips": "IP addresses extracted from the event (e.g., the source IP).", + "tp_usernames": "Not applicable for firewall logs.", + } +} diff --git a/tables/network_firewall_log/network_firewall_log_table.go b/tables/network_firewall_log/network_firewall_log_table.go new file mode 100644 index 0000000..0756d83 --- /dev/null +++ b/tables/network_firewall_log/network_firewall_log_table.go @@ -0,0 +1,80 @@ +package network_firewall_log + +import ( + "time" + + "github.com/rs/xid" + "github.com/turbot/pipe-fittings/v2/utils" + "github.com/turbot/tailpipe-plugin-aws/sources/s3_bucket" + "github.com/turbot/tailpipe-plugin-sdk/artifact_source" + "github.com/turbot/tailpipe-plugin-sdk/artifact_source_config" + "github.com/turbot/tailpipe-plugin-sdk/constants" + "github.com/turbot/tailpipe-plugin-sdk/row_source" + "github.com/turbot/tailpipe-plugin-sdk/schema" + "github.com/turbot/tailpipe-plugin-sdk/table" +) + +const NetworkFirewallLogTableIdentifier = "aws_network_firewall_log" + +// NetworkFirewallLogTable implements the table interface for AWS Network Firewall logs. +type NetworkFirewallLogTable struct{} + +// Identifier returns the unique table identifier. +func (c *NetworkFirewallLogTable) Identifier() string { + return NetworkFirewallLogTableIdentifier +} + +// GetSourceMetadata returns the artifact source configurations for the table. +func (c *NetworkFirewallLogTable) GetSourceMetadata() []*table.SourceMetadata[*NetworkFirewallLog] { + defaultS3ArtifactConfig := &artifact_source_config.ArtifactSourceConfigImpl{ + FileLayout: utils.ToStringPointer("AWSLogs/(%{DATA:org_id}/)?%{NUMBER:account_id}/network-firewall/(?flow|alert|tls)/%{DATA:region}/%{DATA:firewall_name}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}/%{NUMBER:account_id}_network-firewall_%{DATA:log_type}_%{DATA:region}_%{DATA:firewall_name}_%{DATA:timestamp}_%{DATA:hash}.log.gz"), + } + + return []*table.SourceMetadata[*NetworkFirewallLog]{ + { + SourceName: s3_bucket.AwsS3BucketSourceIdentifier, + // Use JSON mapping since the logs are structured as JSON. + Mapper: &NetworkFirewallMapper{}, + Options: []row_source.RowSourceOption{ + artifact_source.WithDefaultArtifactSourceConfig(defaultS3ArtifactConfig), + artifact_source.WithRowPerLine(), + }, + }, + { + SourceName: constants.ArtifactSourceIdentifier, + Mapper: &NetworkFirewallMapper{}, + Options: []row_source.RowSourceOption{ + artifact_source.WithRowPerLine(), + }, + }, + } +} + +// EnrichRow applies standard tailpipe enrichment to each log record. +func (c *NetworkFirewallLogTable) EnrichRow(row *NetworkFirewallLog, sourceEnrichmentFields schema.SourceEnrichment) (*NetworkFirewallLog, error) { + // Add common enrichment fields. + row.CommonFields = sourceEnrichmentFields.CommonFields + + // Standard tailpipe fields. + row.TpID = xid.New().String() + row.TpIngestTimestamp = time.Now() + // Convert the epoch event_timestamp (in seconds) to time.Time. + row.TpTimestamp = *row.EventTimestamp + row.TpDate = row.TpTimestamp.Truncate(24 * time.Hour) + + // Use the firewall name as an index. + row.TpIndex = row.FirewallName + + // If available, use the source IP from the event details. + if row.Event.SrcIP != "" { + row.TpSourceIP = &row.Event.SrcIP + row.TpIps = append(row.TpIps, row.Event.SrcIP) + } + + return row, nil +} + +// GetDescription returns a human-readable description of the table. +func (c *NetworkFirewallLogTable) GetDescription() string { + return "AWS Network Firewall logs capture detailed information about firewall events—including alert, flow, and TLS events produced by Suricata and a dedicated TLS engine. This table provides a structured representation of the log data." +} \ No newline at end of file diff --git a/tables/network_firewall_log/network_firewall_mapper.go b/tables/network_firewall_log/network_firewall_mapper.go new file mode 100644 index 0000000..d64adaf --- /dev/null +++ b/tables/network_firewall_log/network_firewall_mapper.go @@ -0,0 +1,190 @@ +package network_firewall_log + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "time" + + "github.com/turbot/tailpipe-plugin-sdk/table" +) + +type NetworkFirewallMapper struct { +} + +func (n *NetworkFirewallMapper) Identifier() string { + return "network_firewall_log_mapper" +} + +func (n *NetworkFirewallMapper) Map(_ context.Context, a any, _ ...table.MapOption[*NetworkFirewallLog]) (*NetworkFirewallLog, error) { + var jsonBytes []byte + + switch v := a.(type) { + case []byte: + jsonBytes = v + case string: + jsonBytes = []byte(v) + default: + return nil, fmt.Errorf("expected byte[] or string, got %T", a) + } + + var log NetworkFirewallLog + err := unmarshalNetworkFirewallLog(jsonBytes, &log) + if err != nil { + return nil, fmt.Errorf("error decoding JSON: %w; partial log: %+v", err, log) + } + + return &log, nil +} + +func unmarshalNetworkFirewallLog(data []byte, log *NetworkFirewallLog) error { + var temp struct { + FirewallName string `json:"firewall_name"` + AvailabilityZone string `json:"availability_zone"` + EventTimestamp string `json:"event_timestamp"` + Event struct { + Timestamp string `json:"timestamp"` + FlowID int64 `json:"flow_id"` + EventType string `json:"event_type"` + SrcIP string `json:"src_ip"` + SrcPort int `json:"src_port"` + DestIP string `json:"dest_ip"` + DestPort int `json:"dest_port"` + SNI string `json:"sni"` + TLSInspected *bool `json:"tls_inspected,omitempty"` + Proto string `json:"proto"` + AppProto string `json:"app_proto"` + TLSError struct { + ErrorMessage string `json:"error_message"` + } `json:"tls_error,omitempty"` + NetFlow struct { + Packets int64 `json:"pkts"` + Bytes int64 `json:"bytes"` + Start string `json:"start"` + End string `json:"end"` + Age int64 `json:"age"` + MinTTL int64 `json:"min_ttl"` + MaxTTL int64 `json:"max_ttl"` + } `json:"netflow"` + Alert struct { + Action string `json:"action"` + SignatureID int `json:"signature_id"` + Rev int `json:"rev"` + Signature string `json:"signature"` + Category string `json:"category"` + Severity int `json:"severity"` + } `json:"alert,omitempty"` + RevocationCheck struct { + LeafCertFpr string `json:"leaf_cert_fpr"` + Status string `json:"status"` + Action string `json:"action"` + } `json:"revocation_check,omitempty"` + } `json:"event"` + } + + // Unmarshal JSON directly into the NetworkFirewallLog struct. + if err := json.Unmarshal(data, &temp); err != nil { + return err + } + const layout = "2006-01-02T15:04:05.999999-0700" + // Assign values from temp struct + log.FirewallName = temp.FirewallName + log.AvailabilityZone = temp.AvailabilityZone + + // Parse and assign EventTimestamp + if temp.EventTimestamp != "" { + // For Unix timestamps, we must convert to integer first, then to time + unixSeconds, err := strconv.ParseInt(temp.EventTimestamp, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse event_timestamp as unix timestamp: %w", err) + } + timestamp := time.Unix(unixSeconds, 0) + log.EventTimestamp = ×tamp + } + + // Populate Event fields + if temp.Event.Timestamp != "" { + // Try parsing as Unix timestamp first + timestamp, err := time.Parse(layout, temp.Event.Timestamp) + if err != nil { + return fmt.Errorf("failed to parse event timestamp: %w", err) + } + log.Event.Timestamp = ×tamp + } + + log.Event.FlowID = temp.Event.FlowID + log.Event.EventType = temp.Event.EventType + log.Event.SrcIP = temp.Event.SrcIP + log.Event.SrcPort = temp.Event.SrcPort + log.Event.DestIP = temp.Event.DestIP + log.Event.DestPort = temp.Event.DestPort + log.Event.Sni = temp.Event.SNI + log.Event.Proto = temp.Event.Proto + + if temp.Event.AppProto != "" { + log.Event.AppProto = &temp.Event.AppProto + } + + log.Event.TLSInspected = temp.Event.TLSInspected + + // Handle TLSError if present + if temp.Event.TLSError.ErrorMessage != "" { + log.Event.TLSError = &TLSError{ + ErrorMsg: temp.Event.TLSError.ErrorMessage, + } + } + + // Handle Netflow + if temp.Event.NetFlow.Packets > 0 || temp.Event.NetFlow.Bytes > 0 { + netflow := &NetworkFirewallNetflow{ + Packets: int(temp.Event.NetFlow.Packets), + Bytes: int(temp.Event.NetFlow.Bytes), + Age: int(temp.Event.NetFlow.Age), + MinTtl: int(temp.Event.NetFlow.MinTTL), + MaxTtl: int(temp.Event.NetFlow.MaxTTL), + } + + // Parse start and end times + if temp.Event.NetFlow.Start != "" { + startTime, err := time.Parse(layout, temp.Event.NetFlow.Start) + if err != nil { + return fmt.Errorf("failed to parse netflow start time: %w", err) + } + netflow.StartTime = &startTime + } + + if temp.Event.NetFlow.End != "" { + endTime, err := time.Parse(layout, temp.Event.NetFlow.End) + if err != nil { + return fmt.Errorf("failed to parse netflow end time: %w", err) + } + netflow.EndTime = &endTime + } + + log.Event.Netflow = netflow + } + + // Handle Alert if present + if temp.Event.Alert.Action != "" || temp.Event.Alert.SignatureID != 0 { + log.Event.Alert = &NetworkFirewallAlert{ + Action: temp.Event.Alert.Action, + SignatureID: temp.Event.Alert.SignatureID, + Rev: temp.Event.Alert.Rev, + Signature: temp.Event.Alert.Signature, + Category: temp.Event.Alert.Category, + Severity: temp.Event.Alert.Severity, + } + } + + // Handle RevocationCheck if present + if temp.Event.RevocationCheck.LeafCertFpr != "" || temp.Event.RevocationCheck.Status != "" { + log.Event.RevocationCheck = &RevocationCheck{ + LeafCertFpr: temp.Event.RevocationCheck.LeafCertFpr, + Status: temp.Event.RevocationCheck.Status, + Action: temp.Event.RevocationCheck.Action, + } + } + return nil +} +