From ddb9b2a9f26e57378f6037a22bacb188caf56b90 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Tue, 4 Mar 2025 13:02:16 +0530 Subject: [PATCH 1/7] Add network firewall --- .../network_firewall_log.go | 53 ++++++++++++ .../network_firewall_log_table.go | 82 +++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 tables/network_firewall_log/network_firewall_log.go create mode 100644 tables/network_firewall_log/network_firewall_log_table.go diff --git a/tables/network_firewall_log/network_firewall_log.go b/tables/network_firewall_log/network_firewall_log.go new file mode 100644 index 00000000..1b8a1337 --- /dev/null +++ b/tables/network_firewall_log/network_firewall_log.go @@ -0,0 +1,53 @@ +package network_firewall_log + +import ( + "github.com/turbot/tailpipe-plugin-sdk/schema" +) + +type NetworkFirewallAlert struct { + Action string `json:"action"` + SignatureID int `json:"signature_id"` + Rev int `json:"rev"` + Signature string `json:"signature"` + Category string `json:"category"` + Severity int `json:"severity"` +} + +type NetworkFirewallEvent struct { + Timestamp string `json:"timestamp"` + FlowID int64 `json:"flow_id"` + EventType string `json:"event_type"` + SrcIP string `json:"src_ip"` + SrcPort int `json:"src_port"` + DestIP string `json:"dest_ip"` + DestPort int `json:"dest_port"` + Proto string `json:"proto"` + Alert *NetworkFirewallAlert `json:"alert,omitempty"` + TLSInspected *bool `json:"tls_inspected,omitempty"` + TLSError *string `json:"tls_error,omitempty"` + RevocationCheck *string `json:"revocation_check,omitempty"` + Bytes *int64 `json:"bytes,omitempty"` + Packets *int64 `json:"packets,omitempty"` +} + +type NetworkFirewallLog struct { + schema.CommonFields + + FirewallName string `json:"firewall_name"` + AvailabilityZone string `json:"availability_zone"` + EventTimestamp int64 `json:"event_timestamp"` + Event NetworkFirewallEvent `json:"event"` +} + +func (n *NetworkFirewallLog) GetColumnDescriptions() map[string]string { + return map[string]string{ + "firewall_name": "The name of the firewall associated with the log entry.", + "availability_zone": "The Availability Zone of the firewall endpoint that generated the log entry.", + "event_timestamp": "The epoch timestamp (in seconds) when the log was created (UTC).", + "event": "Detailed event information in Suricata EVE JSON format, including human-readable timestamp, event type, network packet details, and alert details if applicable.", + // Override table specific tp_* column descriptions + "tp_index": "The name of the firewall associated with the log entry.", + "tp_ips": "IP addresses extracted from the event (e.g., the source IP).", + "tp_usernames": "Not applicable for firewall logs.", + } +} \ No newline at end of file diff --git a/tables/network_firewall_log/network_firewall_log_table.go b/tables/network_firewall_log/network_firewall_log_table.go new file mode 100644 index 00000000..5bef9416 --- /dev/null +++ b/tables/network_firewall_log/network_firewall_log_table.go @@ -0,0 +1,82 @@ +package network_firewall_log + +import ( + "time" + + "github.com/rs/xid" + "github.com/turbot/pipe-fittings/v2/utils" + "github.com/turbot/tailpipe-plugin-aws/sources/s3_bucket" + "github.com/turbot/tailpipe-plugin-sdk/artifact_source" + "github.com/turbot/tailpipe-plugin-sdk/artifact_source_config" + "github.com/turbot/tailpipe-plugin-sdk/constants" + "github.com/turbot/tailpipe-plugin-sdk/mappers" + "github.com/turbot/tailpipe-plugin-sdk/row_source" + "github.com/turbot/tailpipe-plugin-sdk/schema" + "github.com/turbot/tailpipe-plugin-sdk/table" +) + +const NetworkFirewallLogTableIdentifier = "aws_network_firewall_log" + +// NetworkFirewallLogTable implements the table interface for AWS Network Firewall logs. +type NetworkFirewallLogTable struct{} + +// Identifier returns the unique table identifier. +func (c *NetworkFirewallLogTable) Identifier() string { + return NetworkFirewallLogTableIdentifier +} + +// GetSourceMetadata returns the artifact source configurations for the table. +func (c *NetworkFirewallLogTable) GetSourceMetadata() []*table.SourceMetadata[*NetworkFirewallLog] { + // Example file layout – adjust as needed. + defaultS3ArtifactConfig := &artifact_source_config.ArtifactSourceConfigImpl{ + FileLayout: utils.ToStringPointer("AWSLogs/(%{DATA:org_id}/)?%{NUMBER:account_id}/networkfirewall/%{DATA:region}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{DATA}.log"), + } + + return []*table.SourceMetadata[*NetworkFirewallLog]{ + { + SourceName: s3_bucket.AwsS3BucketSourceIdentifier, + // Use JSON mapping since the logs are structured as JSON. + Mapper: mappers.NewJSONMapper[*NetworkFirewallLog](), + Options: []row_source.RowSourceOption{ + artifact_source.WithDefaultArtifactSourceConfig(defaultS3ArtifactConfig), + artifact_source.WithRowPerLine(), + }, + }, + { + SourceName: constants.ArtifactSourceIdentifier, + Mapper: mappers.NewJSONMapper[*NetworkFirewallLog](), + Options: []row_source.RowSourceOption{ + artifact_source.WithRowPerLine(), + }, + }, + } +} + +// EnrichRow applies standard tailpipe enrichment to each log record. +func (c *NetworkFirewallLogTable) EnrichRow(row *NetworkFirewallLog, sourceEnrichmentFields schema.SourceEnrichment) (*NetworkFirewallLog, error) { + // Add common enrichment fields. + row.CommonFields = sourceEnrichmentFields.CommonFields + + // Standard tailpipe fields. + row.TpID = xid.New().String() + row.TpIngestTimestamp = time.Now() + // Convert the epoch event_timestamp (in seconds) to time.Time. + row.TpTimestamp = time.Unix(row.EventTimestamp, 0) + row.TpDate = row.TpTimestamp.Truncate(24 * time.Hour) + + // Use the firewall name as an index. + row.TpIndex = row.FirewallName + + // If available, use the source IP from the event details. + if row.Event.SrcIP != "" { + row.TpSourceIP = &row.Event.SrcIP + row.TpIps = append(row.TpIps, row.Event.SrcIP) + } + + return row, nil +} + +// GetDescription returns a human-readable description of the table. +func (c *NetworkFirewallLogTable) GetDescription() string { + return "AWS Network Firewall logs capture detailed information about firewall events—including alert, flow, and TLS events produced by Suricata and a dedicated TLS engine. This table provides a structured representation of the log data." +} \ No newline at end of file From 63f0a0b3c515c15d7bc2e44cdd0de383b1846c6c Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Wed, 5 Mar 2025 18:00:59 +0530 Subject: [PATCH 2/7] Add table aws_network_firewall_log --- .../network_firewall_log.go | 38 +++++++-------- .../network_firewall_log_table.go | 7 ++- .../network_firewall_mapper.go | 46 +++++++++++++++++++ 3 files changed, 69 insertions(+), 22 deletions(-) create mode 100644 tables/network_firewall_log/network_firewall_mapper.go diff --git a/tables/network_firewall_log/network_firewall_log.go b/tables/network_firewall_log/network_firewall_log.go index 1b8a1337..2156d5bb 100644 --- a/tables/network_firewall_log/network_firewall_log.go +++ b/tables/network_firewall_log/network_firewall_log.go @@ -1,6 +1,8 @@ package network_firewall_log import ( + "time" + "github.com/turbot/tailpipe-plugin-sdk/schema" ) @@ -14,20 +16,20 @@ type NetworkFirewallAlert struct { } type NetworkFirewallEvent struct { - Timestamp string `json:"timestamp"` - FlowID int64 `json:"flow_id"` - EventType string `json:"event_type"` - SrcIP string `json:"src_ip"` - SrcPort int `json:"src_port"` - DestIP string `json:"dest_ip"` - DestPort int `json:"dest_port"` - Proto string `json:"proto"` + Timestamp *time.Time `json:"timestamp"` + FlowID int64 `json:"flow_id"` + EventType string `json:"event_type"` + SrcIP string `json:"src_ip"` + SrcPort int `json:"src_port"` + DestIP string `json:"dest_ip"` + DestPort int `json:"dest_port"` + Proto string `json:"proto"` Alert *NetworkFirewallAlert `json:"alert,omitempty"` - TLSInspected *bool `json:"tls_inspected,omitempty"` - TLSError *string `json:"tls_error,omitempty"` - RevocationCheck *string `json:"revocation_check,omitempty"` - Bytes *int64 `json:"bytes,omitempty"` - Packets *int64 `json:"packets,omitempty"` + TLSInspected *bool `json:"tls_inspected,omitempty"` + TLSError *string `json:"tls_error,omitempty"` + RevocationCheck *string `json:"revocation_check,omitempty"` + Bytes *int64 `json:"bytes,omitempty"` + Packets *int64 `json:"packets,omitempty"` } type NetworkFirewallLog struct { @@ -35,7 +37,7 @@ type NetworkFirewallLog struct { FirewallName string `json:"firewall_name"` AvailabilityZone string `json:"availability_zone"` - EventTimestamp int64 `json:"event_timestamp"` + EventTimestamp *time.Time `json:"event_timestamp"` Event NetworkFirewallEvent `json:"event"` } @@ -46,8 +48,8 @@ func (n *NetworkFirewallLog) GetColumnDescriptions() map[string]string { "event_timestamp": "The epoch timestamp (in seconds) when the log was created (UTC).", "event": "Detailed event information in Suricata EVE JSON format, including human-readable timestamp, event type, network packet details, and alert details if applicable.", // Override table specific tp_* column descriptions - "tp_index": "The name of the firewall associated with the log entry.", - "tp_ips": "IP addresses extracted from the event (e.g., the source IP).", - "tp_usernames": "Not applicable for firewall logs.", + "tp_index": "The name of the firewall associated with the log entry.", + "tp_ips": "IP addresses extracted from the event (e.g., the source IP).", + "tp_usernames": "Not applicable for firewall logs.", } -} \ No newline at end of file +} diff --git a/tables/network_firewall_log/network_firewall_log_table.go b/tables/network_firewall_log/network_firewall_log_table.go index 5bef9416..1142ba8b 100644 --- a/tables/network_firewall_log/network_firewall_log_table.go +++ b/tables/network_firewall_log/network_firewall_log_table.go @@ -9,7 +9,6 @@ import ( "github.com/turbot/tailpipe-plugin-sdk/artifact_source" "github.com/turbot/tailpipe-plugin-sdk/artifact_source_config" "github.com/turbot/tailpipe-plugin-sdk/constants" - "github.com/turbot/tailpipe-plugin-sdk/mappers" "github.com/turbot/tailpipe-plugin-sdk/row_source" "github.com/turbot/tailpipe-plugin-sdk/schema" "github.com/turbot/tailpipe-plugin-sdk/table" @@ -36,7 +35,7 @@ func (c *NetworkFirewallLogTable) GetSourceMetadata() []*table.SourceMetadata[*N { SourceName: s3_bucket.AwsS3BucketSourceIdentifier, // Use JSON mapping since the logs are structured as JSON. - Mapper: mappers.NewJSONMapper[*NetworkFirewallLog](), + Mapper: &NetworkFirewallMapper{}, Options: []row_source.RowSourceOption{ artifact_source.WithDefaultArtifactSourceConfig(defaultS3ArtifactConfig), artifact_source.WithRowPerLine(), @@ -44,7 +43,7 @@ func (c *NetworkFirewallLogTable) GetSourceMetadata() []*table.SourceMetadata[*N }, { SourceName: constants.ArtifactSourceIdentifier, - Mapper: mappers.NewJSONMapper[*NetworkFirewallLog](), + Mapper: &NetworkFirewallMapper{}, Options: []row_source.RowSourceOption{ artifact_source.WithRowPerLine(), }, @@ -61,7 +60,7 @@ func (c *NetworkFirewallLogTable) EnrichRow(row *NetworkFirewallLog, sourceEnric row.TpID = xid.New().String() row.TpIngestTimestamp = time.Now() // Convert the epoch event_timestamp (in seconds) to time.Time. - row.TpTimestamp = time.Unix(row.EventTimestamp, 0) + row.TpTimestamp = *row.EventTimestamp row.TpDate = row.TpTimestamp.Truncate(24 * time.Hour) // Use the firewall name as an index. diff --git a/tables/network_firewall_log/network_firewall_mapper.go b/tables/network_firewall_log/network_firewall_mapper.go new file mode 100644 index 00000000..cc247f0c --- /dev/null +++ b/tables/network_firewall_log/network_firewall_mapper.go @@ -0,0 +1,46 @@ +package network_firewall_log + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/turbot/tailpipe-plugin-sdk/table" +) + +type NetworkFirewallMapper struct { +} + +func (n *NetworkFirewallMapper) Identifier() string { + return "network_firewall_log_mapper" +} + +func (n *NetworkFirewallMapper) Map(_ context.Context, a any, _ ...table.MapOption[*NetworkFirewallLog]) (*NetworkFirewallLog, error) { + var jsonBytes []byte + + switch v := a.(type) { + case []byte: + jsonBytes = v + case string: + jsonBytes = []byte(v) + default: + return nil, fmt.Errorf("expected byte[] or string, got %T", a) + } + + var log NetworkFirewallLog + err := unmarshalNetworkFirewallLog(jsonBytes, &log) + if err != nil { + return nil, fmt.Errorf("error decoding JSON: %w; partial log: %+v", err, log) + } + + return &log, nil +} + +func unmarshalNetworkFirewallLog(data []byte, log *NetworkFirewallLog) error { + + // Unmarshal JSON directly into the NetworkFirewallLog struct. + if err := json.Unmarshal(data, log); err != nil { + return err + } + return nil +} \ No newline at end of file From 90acea7c3f5979917414bcaca4a052ddf7c10972 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Fri, 7 Mar 2025 12:25:03 +0530 Subject: [PATCH 3/7] Update table --- aws/plugin.go | 2 ++ tables/network_firewall_log/network_firewall_log_table.go | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/aws/plugin.go b/aws/plugin.go index 67cfbff0..d3130948 100755 --- a/aws/plugin.go +++ b/aws/plugin.go @@ -5,6 +5,7 @@ import ( "github.com/turbot/tailpipe-plugin-aws/config" "github.com/turbot/tailpipe-plugin-aws/sources/s3_bucket" "github.com/turbot/tailpipe-plugin-aws/tables/cloudtrail_log" + "github.com/turbot/tailpipe-plugin-aws/tables/network_firewall_log" "github.com/turbot/tailpipe-plugin-aws/tables/s3_server_access_log" "github.com/turbot/tailpipe-plugin-sdk/plugin" "github.com/turbot/tailpipe-plugin-sdk/row_source" @@ -21,6 +22,7 @@ func init() { // 2. table implementation table.RegisterTable[*cloudtrail_log.CloudTrailLog, *cloudtrail_log.CloudTrailLogTable]() table.RegisterTable[*s3_server_access_log.S3ServerAccessLog, *s3_server_access_log.S3ServerAccessLogTable]() + table.RegisterTable[*network_firewall_log.NetworkFirewallLog, *network_firewall_log.NetworkFirewallLogTable]() // register sources row_source.RegisterRowSource[*s3_bucket.AwsS3BucketSource]() diff --git a/tables/network_firewall_log/network_firewall_log_table.go b/tables/network_firewall_log/network_firewall_log_table.go index 1142ba8b..86a604f7 100644 --- a/tables/network_firewall_log/network_firewall_log_table.go +++ b/tables/network_firewall_log/network_firewall_log_table.go @@ -28,7 +28,8 @@ func (c *NetworkFirewallLogTable) Identifier() string { func (c *NetworkFirewallLogTable) GetSourceMetadata() []*table.SourceMetadata[*NetworkFirewallLog] { // Example file layout – adjust as needed. defaultS3ArtifactConfig := &artifact_source_config.ArtifactSourceConfigImpl{ - FileLayout: utils.ToStringPointer("AWSLogs/(%{DATA:org_id}/)?%{NUMBER:account_id}/networkfirewall/%{DATA:region}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{DATA}.log"), + FileLayout: utils.ToStringPointer("AWSLogs/%{NUMBER:account_id}/network-firewall/(?flow|alert|tls)/%{DATA:region}/%{DATA:firewall_name}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}/%{NUMBER:account_id}_network-firewall_%{DATA:log_type}_%{DATA:region}_%{DATA:firewall_name}_%{DATA:timestamp}_%{DATA:hash}.log.gz"), + //s3-bucket-name/optional-s3-bucket-prefix/AWSLogs/aws-account-id/network-firewall/log-type/Region/firewall-name/timestamp/ } return []*table.SourceMetadata[*NetworkFirewallLog]{ From 3c380f8194eee635c191df1fa381c814e2557311 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Fri, 7 Mar 2025 20:09:38 +0530 Subject: [PATCH 4/7] Update aws_network_firewall_log mapper --- .../network_firewall_log.go | 61 ++++--- .../network_firewall_mapper.go | 169 +++++++++++++++++- 2 files changed, 207 insertions(+), 23 deletions(-) diff --git a/tables/network_firewall_log/network_firewall_log.go b/tables/network_firewall_log/network_firewall_log.go index 2156d5bb..028d62c7 100644 --- a/tables/network_firewall_log/network_firewall_log.go +++ b/tables/network_firewall_log/network_firewall_log.go @@ -7,29 +7,50 @@ import ( ) type NetworkFirewallAlert struct { - Action string `json:"action"` - SignatureID int `json:"signature_id"` - Rev int `json:"rev"` - Signature string `json:"signature"` - Category string `json:"category"` - Severity int `json:"severity"` + Action string `json:"action,omitempty"` + SignatureID int `json:"signature_id,omitempty"` + Rev int `json:"rev,omitempty"` + Signature string `json:"signature,omitempty"` + Category string `json:"category,omitempty"` + Severity int `json:"severity,omitempty"` +} + +type NetworkFirewallNetflow struct { + StartTime *time.Time `json:"start_time,omitempty"` + EndTime *time.Time `json:"end_time,omitempty"` + Bytes int `json:"bytes,omitempty"` + Packets int `json:"packets,omitempty"` + Age int `json:"age,omitempty"` + MinTtl int `json:"min_ttl,omitempty"` + MaxTtl int `json:"max_ttl,omitempty"` +} + +type RevocationCheck struct { + LeafCertFpr string `json:"leaf_cert_fpr,omitempty"` + Status string `json:"status,omitempty"` + Action string `json:"action,omitempty"` +} + +type TLSError struct { + ErrorMsg string `json:"error_message"` } type NetworkFirewallEvent struct { - Timestamp *time.Time `json:"timestamp"` - FlowID int64 `json:"flow_id"` - EventType string `json:"event_type"` - SrcIP string `json:"src_ip"` - SrcPort int `json:"src_port"` - DestIP string `json:"dest_ip"` - DestPort int `json:"dest_port"` - Proto string `json:"proto"` - Alert *NetworkFirewallAlert `json:"alert,omitempty"` - TLSInspected *bool `json:"tls_inspected,omitempty"` - TLSError *string `json:"tls_error,omitempty"` - RevocationCheck *string `json:"revocation_check,omitempty"` - Bytes *int64 `json:"bytes,omitempty"` - Packets *int64 `json:"packets,omitempty"` + Timestamp *time.Time `json:"timestamp"` + FlowID int64 `json:"flow_id"` + EventType string `json:"event_type"` + SrcIP string `json:"src_ip"` + SrcPort int `json:"src_port"` + DestIP string `json:"dest_ip"` + DestPort int `json:"dest_port"` + Sni string `json:"sni,omitempty"` + Proto string `json:"proto,omitempty"` + AppProto *string `json:"app_proto,omitempty"` + Alert *NetworkFirewallAlert `json:"alert,omitempty"` + Netflow *NetworkFirewallNetflow `json:"netflow,omitempty"` + TLSInspected *bool `json:"tls_inspected,omitempty"` + TLSError *TLSError `json:"tls_error,omitempty"` + RevocationCheck *RevocationCheck `json:"revocation_check,omitempty"` } type NetworkFirewallLog struct { diff --git a/tables/network_firewall_log/network_firewall_mapper.go b/tables/network_firewall_log/network_firewall_mapper.go index cc247f0c..8c05a583 100644 --- a/tables/network_firewall_log/network_firewall_mapper.go +++ b/tables/network_firewall_log/network_firewall_mapper.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "time" "github.com/turbot/tailpipe-plugin-sdk/table" ) @@ -37,10 +38,172 @@ func (n *NetworkFirewallMapper) Map(_ context.Context, a any, _ ...table.MapOpti } func unmarshalNetworkFirewallLog(data []byte, log *NetworkFirewallLog) error { - + var temp struct { + FirewallName string `json:"firewall_name"` + AvailabilityZone string `json:"availability_zone"` + EventTimestamp string `json:"event_timestamp"` + Event struct { + Timestamp string `json:"timestamp"` + FlowID int64 `json:"flow_id"` + EventType string `json:"event_type"` + SrcIP string `json:"src_ip"` + SrcPort int `json:"src_port"` + DestIP string `json:"dest_ip"` + DestPort int `json:"dest_port"` + SNI string `json:"sni"` + TLSInspected *bool `json:"tls_inspected,omitempty"` + Proto string `json:"proto"` + AppProto string `json:"app_proto"` + TLSError struct { + ErrorMessage string `json:"error_message"` + } `json:"tls_error,omitempty"` + NetFlow struct { + Packets int64 `json:"pkts"` + Bytes int64 `json:"bytes"` + Start string `json:"start"` + End string `json:"end"` + Age int64 `json:"age"` + MinTTL int64 `json:"min_ttl"` + MaxTTL int64 `json:"max_ttl"` + } `json:"netflow"` + Alert struct { + Action string `json:"action"` + SignatureID int `json:"signature_id"` + Rev int `json:"rev"` + Signature string `json:"signature"` + Category string `json:"category"` + Severity int `json:"severity"` + } `json:"alert,omitempty"` + RevocationCheck struct { + LeafCertFpr string `json:"leaf_cert_fpr"` + Status string `json:"status"` + Action string `json:"action"` + } `json:"revocation_check,omitempty"` + } `json:"event"` + } + // Unmarshal JSON directly into the NetworkFirewallLog struct. - if err := json.Unmarshal(data, log); err != nil { + if err := json.Unmarshal(data, &temp); err != nil { return err } + + // Assign values from temp struct + log.FirewallName = temp.FirewallName + log.AvailabilityZone = temp.AvailabilityZone + + // Parse and assign EventTimestamp + if temp.EventTimestamp != "" { + // Try parsing as Unix timestamp (seconds since epoch) + unixTimestamp, err := parseInt64(temp.EventTimestamp) + if err == nil { + timestamp := time.Unix(unixTimestamp, 0) + log.EventTimestamp = ×tamp + } else { + // Fallback to RFC3339 if it's not a Unix timestamp + timestamp, err := time.Parse(time.RFC3339, temp.EventTimestamp) + if err != nil { + return fmt.Errorf("failed to parse event_timestamp: %w", err) + } + log.EventTimestamp = ×tamp + } + } + + // Populate Event fields + if temp.Event.Timestamp != "" { + // Try parsing as Unix timestamp first + unixTimestamp, err := parseInt64(temp.Event.Timestamp) + if err == nil { + timestamp := time.Unix(unixTimestamp, 0) + log.Event.Timestamp = ×tamp + } else { + // Fallback to RFC3339 + timestamp, err := time.Parse(time.RFC3339, temp.Event.Timestamp) + if err != nil { + return fmt.Errorf("failed to parse event timestamp: %w", err) + } + log.Event.Timestamp = ×tamp + } + } + + log.Event.FlowID = temp.Event.FlowID + log.Event.EventType = temp.Event.EventType + log.Event.SrcIP = temp.Event.SrcIP + log.Event.SrcPort = temp.Event.SrcPort + log.Event.DestIP = temp.Event.DestIP + log.Event.DestPort = temp.Event.DestPort + log.Event.Sni = temp.Event.SNI + log.Event.Proto = temp.Event.Proto + + if temp.Event.AppProto != "" { + log.Event.AppProto = &temp.Event.AppProto + } + + log.Event.TLSInspected = temp.Event.TLSInspected + + // Handle TLSError if present + if temp.Event.TLSError.ErrorMessage != "" { + log.Event.TLSError = &TLSError{ + ErrorMsg: temp.Event.TLSError.ErrorMessage, + } + } + + // Handle Netflow + if temp.Event.NetFlow.Packets > 0 || temp.Event.NetFlow.Bytes > 0 { + netflow := &NetworkFirewallNetflow{ + Packets: int(temp.Event.NetFlow.Packets), + Bytes: int(temp.Event.NetFlow.Bytes), + Age: int(temp.Event.NetFlow.Age), + MinTtl: int(temp.Event.NetFlow.MinTTL), + MaxTtl: int(temp.Event.NetFlow.MaxTTL), + } + const layout = "2006-01-02T15:04:05.999999-0700" + + // Parse start and end times + if temp.Event.NetFlow.Start != "" { + startTime, err := time.Parse(layout, temp.Event.NetFlow.Start) + if err != nil { + return fmt.Errorf("failed to parse netflow start time: %w", err) + } + netflow.StartTime = &startTime + } + + if temp.Event.NetFlow.End != "" { + endTime, err := time.Parse(layout, temp.Event.NetFlow.End) + if err != nil { + return fmt.Errorf("failed to parse netflow end time: %w", err) + } + netflow.EndTime = &endTime + } + + log.Event.Netflow = netflow + } + + // Handle Alert if present + if temp.Event.Alert.Action != "" || temp.Event.Alert.SignatureID != 0 { + log.Event.Alert = &NetworkFirewallAlert{ + Action: temp.Event.Alert.Action, + SignatureID: temp.Event.Alert.SignatureID, + Rev: temp.Event.Alert.Rev, + Signature: temp.Event.Alert.Signature, + Category: temp.Event.Alert.Category, + Severity: temp.Event.Alert.Severity, + } + } + + // Handle RevocationCheck if present + if temp.Event.RevocationCheck.LeafCertFpr != "" || temp.Event.RevocationCheck.Status != "" { + log.Event.RevocationCheck = &RevocationCheck{ + LeafCertFpr: temp.Event.RevocationCheck.LeafCertFpr, + Status: temp.Event.RevocationCheck.Status, + Action: temp.Event.RevocationCheck.Action, + } + } return nil -} \ No newline at end of file +} + +// Helper function to parse a string to int64 +func parseInt64(value string) (int64, error) { + var result int64 + _, err := fmt.Sscanf(value, "%d", &result) + return result, err +} From a45e59e55470236d097107c1c55a2024de949a41 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Sun, 9 Mar 2025 21:23:41 +0530 Subject: [PATCH 5/7] Cleanup remove comments and update timestamp code --- .../network_firewall_log_table.go | 2 - .../network_firewall_mapper.go | 43 ++++++------------- 2 files changed, 12 insertions(+), 33 deletions(-) diff --git a/tables/network_firewall_log/network_firewall_log_table.go b/tables/network_firewall_log/network_firewall_log_table.go index 86a604f7..b345b8e9 100644 --- a/tables/network_firewall_log/network_firewall_log_table.go +++ b/tables/network_firewall_log/network_firewall_log_table.go @@ -26,10 +26,8 @@ func (c *NetworkFirewallLogTable) Identifier() string { // GetSourceMetadata returns the artifact source configurations for the table. func (c *NetworkFirewallLogTable) GetSourceMetadata() []*table.SourceMetadata[*NetworkFirewallLog] { - // Example file layout – adjust as needed. defaultS3ArtifactConfig := &artifact_source_config.ArtifactSourceConfigImpl{ FileLayout: utils.ToStringPointer("AWSLogs/%{NUMBER:account_id}/network-firewall/(?flow|alert|tls)/%{DATA:region}/%{DATA:firewall_name}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}/%{NUMBER:account_id}_network-firewall_%{DATA:log_type}_%{DATA:region}_%{DATA:firewall_name}_%{DATA:timestamp}_%{DATA:hash}.log.gz"), - //s3-bucket-name/optional-s3-bucket-prefix/AWSLogs/aws-account-id/network-firewall/log-type/Region/firewall-name/timestamp/ } return []*table.SourceMetadata[*NetworkFirewallLog]{ diff --git a/tables/network_firewall_log/network_firewall_mapper.go b/tables/network_firewall_log/network_firewall_mapper.go index 8c05a583..d64adafb 100644 --- a/tables/network_firewall_log/network_firewall_mapper.go +++ b/tables/network_firewall_log/network_firewall_mapper.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strconv" "time" "github.com/turbot/tailpipe-plugin-sdk/table" @@ -86,43 +87,30 @@ func unmarshalNetworkFirewallLog(data []byte, log *NetworkFirewallLog) error { if err := json.Unmarshal(data, &temp); err != nil { return err } - + const layout = "2006-01-02T15:04:05.999999-0700" // Assign values from temp struct log.FirewallName = temp.FirewallName log.AvailabilityZone = temp.AvailabilityZone // Parse and assign EventTimestamp if temp.EventTimestamp != "" { - // Try parsing as Unix timestamp (seconds since epoch) - unixTimestamp, err := parseInt64(temp.EventTimestamp) - if err == nil { - timestamp := time.Unix(unixTimestamp, 0) - log.EventTimestamp = ×tamp - } else { - // Fallback to RFC3339 if it's not a Unix timestamp - timestamp, err := time.Parse(time.RFC3339, temp.EventTimestamp) - if err != nil { - return fmt.Errorf("failed to parse event_timestamp: %w", err) - } - log.EventTimestamp = ×tamp + // For Unix timestamps, we must convert to integer first, then to time + unixSeconds, err := strconv.ParseInt(temp.EventTimestamp, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse event_timestamp as unix timestamp: %w", err) } + timestamp := time.Unix(unixSeconds, 0) + log.EventTimestamp = ×tamp } // Populate Event fields if temp.Event.Timestamp != "" { // Try parsing as Unix timestamp first - unixTimestamp, err := parseInt64(temp.Event.Timestamp) - if err == nil { - timestamp := time.Unix(unixTimestamp, 0) - log.Event.Timestamp = ×tamp - } else { - // Fallback to RFC3339 - timestamp, err := time.Parse(time.RFC3339, temp.Event.Timestamp) - if err != nil { - return fmt.Errorf("failed to parse event timestamp: %w", err) - } - log.Event.Timestamp = ×tamp + timestamp, err := time.Parse(layout, temp.Event.Timestamp) + if err != nil { + return fmt.Errorf("failed to parse event timestamp: %w", err) } + log.Event.Timestamp = ×tamp } log.Event.FlowID = temp.Event.FlowID @@ -156,7 +144,6 @@ func unmarshalNetworkFirewallLog(data []byte, log *NetworkFirewallLog) error { MinTtl: int(temp.Event.NetFlow.MinTTL), MaxTtl: int(temp.Event.NetFlow.MaxTTL), } - const layout = "2006-01-02T15:04:05.999999-0700" // Parse start and end times if temp.Event.NetFlow.Start != "" { @@ -201,9 +188,3 @@ func unmarshalNetworkFirewallLog(data []byte, log *NetworkFirewallLog) error { return nil } -// Helper function to parse a string to int64 -func parseInt64(value string) (int64, error) { - var result int64 - _, err := fmt.Sscanf(value, "%d", &result) - return result, err -} From 81ff82ab5175e1eb1a2bd6ccb28245874be464a3 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Mon, 10 Mar 2025 17:25:29 +0530 Subject: [PATCH 6/7] Add queries.md, index.md for aws_network_firewall_log --- docs/tables/aws_network_firewall_log/index.md | 211 ++++++++++++ .../aws_network_firewall_log/queries.md | 305 ++++++++++++++++++ .../network_firewall_log_table.go | 2 +- 3 files changed, 517 insertions(+), 1 deletion(-) create mode 100644 docs/tables/aws_network_firewall_log/index.md create mode 100644 docs/tables/aws_network_firewall_log/queries.md diff --git a/docs/tables/aws_network_firewall_log/index.md b/docs/tables/aws_network_firewall_log/index.md new file mode 100644 index 00000000..4bfa7509 --- /dev/null +++ b/docs/tables/aws_network_firewall_log/index.md @@ -0,0 +1,211 @@ +--- +title: "Tailpipe Table: aws_network_firewall_log - Query AWS Network Firewall Logs" +description: "AWS Network Firewall logs capture information about traffic flowing through your AWS Network Firewall, including flow logs and alert information." +--- + +# Table: aws_network_firewall_log - Query AWS Network Firewall Logs + +The `aws_network_firewall_log` table allows you to query data from AWS Network Firewall logs. This table provides detailed insights into the traffic flowing through your AWS Network Firewall, including source and destination IP addresses, ports, protocols, and alert information. + +## Configure + +Create a [partition](https://tailpipe.io/docs/manage/partition) for `aws_network_firewall_log` ([examples](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_network_firewall_log#example-configurations)): + +```sh +vi ~/.tailpipe/config/aws.tpc +``` + +```hcl +connection "aws" "network_firewall_logging" { + profile = "my-network-firewall-logging" +} + +partition "aws_network_firewall_log" "my_logs" { + source "aws_s3_bucket" { + connection = connection.aws.network_firewall_logging + bucket = "aws-network-firewall-logs-bucket" + } +} +``` + +## Collect + +[Collect](https://tailpipe.io/docs/manage/collection) logs for all `aws_network_firewall_log` partitions: + +```sh +tailpipe collect aws_network_firewall_log +``` + +Or for a single partition: + +```sh +tailpipe collect aws_network_firewall_log.my_logs +``` + +## Query + +**[Explore example queries for this table →](https://hub.tailpipe.io/plugins/turbot/aws/queries/aws_network_firewall_log)** + +### Traffic by Protocol + +Analyze traffic by protocol to understand the distribution of network traffic. + +```sql +select + event ->> 'proto' as protocol, + count(*) as traffic_count +from + aws_network_firewall_log +group by + protocol +order by + traffic_count desc; +``` + +### Destination Port Analysis + +Identify the most commonly accessed destination ports to understand traffic patterns. + +```sql +select + event ->> 'dest_port' as destination_port, + count(*) as connection_count +from + aws_network_firewall_log +where + event ->> 'dest_port' is not null +group by + destination_port +order by + connection_count desc +limit 10; +``` + +### Traffic by Application Protocol + +Analyze traffic by application protocol to understand what types of applications are generating network traffic. + +```sql +select + event ->> 'app_proto' as app_protocol, + count(*) as connection_count +from + aws_network_firewall_log +group by + app_protocol +order by + connection_count desc; +``` + +## Example Configurations + +### Collect logs from an S3 bucket + +Collect Network Firewall logs stored in an S3 bucket. + +```hcl +connection "aws" "network_firewall_logging" { + profile = "my-network-firewall-logging" +} + +partition "aws_network_firewall_log" "my_logs" { + source "aws_s3_bucket" { + connection = connection.aws.network_firewall_logging + bucket = "aws-network-firewall-logs-bucket" + } +} +``` + +### Collect logs from an S3 bucket with a prefix + +Collect Network Firewall logs stored in an S3 bucket using a prefix. + +```hcl +partition "aws_network_firewall_log" "my_logs_prefix" { + source "aws_s3_bucket" { + connection = connection.aws.network_firewall_logging + bucket = "aws-network-firewall-logs-bucket" + prefix = "my/prefix/" + } +} +``` + +### Collect logs from local files + +You can also collect Network Firewall logs from local files. + +```hcl +partition "aws_network_firewall_log" "local_logs" { + source "file" { + paths = ["/Users/myuser/network_firewall_logs"] + file_layout = "%{DATA}.json.gz" + } +} +``` + +### Collect logs for all accounts in an organization + +For a specific organization, collect logs for all accounts and regions. + +```hcl +partition "aws_network_firewall_log" "my_logs_org" { + source "aws_s3_bucket" { + connection = connection.aws.network_firewall_logging + bucket = "network-firewall-logs-bucket" + file_layout = "AWSLogs/o-aa111bb222/%{NUMBER:account_id}/network-firewall/(?flow|alert|tls)/%{DATA:region}/%{DATA:firewall_name}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}/%{NUMBER:account_id}_network-firewall_%{DATA:log_type}_%{DATA:region}_%{DATA:firewall_name}_%{DATA:timestamp}_%{DATA:hash}.log.gz" + } +} +``` + +### Collect logs for a single account + +For a specific account, collect logs for all regions. + +```hcl +partition "aws_network_firewall_log" "my_logs_account" { + source "aws_s3_bucket" { + connection = connection.aws.network_firewall_logging + bucket = "network-firewall-logs-bucket" + file_layout = "AWSLogs/(%{DATA:org_id}/)?123456789012/network-firewall/(?flow|alert|tls)/%{DATA:region}/%{DATA:firewall_name}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}/123456789012_network-firewall_%{DATA:log_type}_%{DATA:region}_%{DATA:firewall_name}_%{DATA:timestamp}_%{DATA:hash}.log.gz" + } +} +``` + +### Collect logs for a single region + +For all accounts, collect logs from `us-east-1`. + +```hcl +partition "aws_network_firewall_log" "my_logs_region" { + source "aws_s3_bucket" { + connection = connection.aws.network_firewall_logging + bucket = "network-firewall-logs-bucket" + file_layout = "AWSLogs/(%{DATA:org_id}/)?%{NUMBER:account_id}/network-firewall/(?flow|alert|tls)/us-east-1/%{DATA:firewall_name}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}/%{NUMBER:account_id}_network-firewall_%{DATA:log_type}_us-east-1_%{DATA:firewall_name}_%{DATA:timestamp}_%{DATA:hash}.log.gz" + } +} +``` + +### Filter for specific traffic type + +Use the filter argument in your partition to focus on specific traffic types. + +```hcl +partition "aws_network_firewall_log" "my_logs_filtered" { + filter = "event->>'app_proto' = 'tls'" + + source "aws_s3_bucket" { + connection = connection.aws.network_firewall_logging + bucket = "network-firewall-logs-bucket" + } +} +``` + +## Source Defaults + +### aws_s3_bucket + +This table sets the following defaults for the [aws_s3_bucket source](https://hub.tailpipe.io/plugins/turbot/aws/sources/aws_s3_bucket#arguments): + +| Argument | Default | +|--------------|---------| +| file_layout | `AWSLogs/(%{DATA:org_id}/)?%{NUMBER:account_id}/network-firewall/(?flow|alert|tls)/%{DATA:region}/%{DATA:firewall_name}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}/%{NUMBER:account_id}_network-firewall_%{DATA:log_type}_%{DATA:region}_%{DATA:firewall_name}_%{DATA:timestamp}_%{DATA:hash}.log.gz` | diff --git a/docs/tables/aws_network_firewall_log/queries.md b/docs/tables/aws_network_firewall_log/queries.md new file mode 100644 index 00000000..88076f7f --- /dev/null +++ b/docs/tables/aws_network_firewall_log/queries.md @@ -0,0 +1,305 @@ +## Activity Examples + +### Daily Network Firewall Activity Trends + +Count Network Firewall log entries per day to identify network activity trends. This helps monitor overall firewall activity and detect unusual spikes in traffic. + +```sql +select + strftime(event_timestamp, '%Y-%m-%d') as traffic_date, + count(*) as log_count +from + aws_network_firewall_log +group by + traffic_date +order by + traffic_date asc; +``` + +```yaml +folder: Network Firewall +``` + +### Top 10 Source IPs Generating Traffic + +Identify the top 10 source IP addresses that generated the most network traffic. This helps detect potential high-traffic sources that may indicate misconfigured applications or suspicious activity. + +```sql +select + tp_source_ip, + count(*) as log_count +from + aws_network_firewall_log +where + tp_source_ip is not null +group by + tp_source_ip +order by + log_count desc +limit 10; +``` + +```yaml +folder: Network Firewall +``` + +### Traffic Distribution by Protocol + +Analyze traffic distribution by protocol to understand what types of network protocols are most frequently used in your environment. + +```sql +select + event ->> 'proto' as protocol, + count(*) as traffic_count +from + aws_network_firewall_log +group by + protocol +order by + traffic_count desc; +``` + +```yaml +folder: Network Firewall +``` + +## Detection Examples + +### Identify Traffic from a Suspicious IP + +Check if a specific IP is sending or receiving traffic. This is useful for investigating potential threats or monitoring known suspicious IPs. + +```sql +select + event_timestamp, + event ->> 'src_ip' as source_ip, + event ->> 'src_port' as source_port, + event ->> 'dest_ip' as destination_ip, + event ->> 'dest_port' as destination_port, + event ->> 'proto' as protocol, + firewall_name +from + aws_network_firewall_log +where + event ->> 'src_ip' = '192.0.2.100' + or event ->> 'dest_ip' = '192.0.2.100' +order by + event_timestamp desc; +``` + +```yaml +folder: Network Firewall +``` + +### Detect Potentially Suspicious DNS Traffic + +Identify DNS traffic that might indicate command and control (C2) communications or data exfiltration attempts over DNS. + +```sql +select + event_timestamp, + event ->> 'src_ip' as source_ip, + event ->> 'dest_ip' as destination_ip, + event ->> 'dest_port' as destination_port, + event ->> 'proto' as protocol, + firewall_name +from + aws_network_firewall_log +where + event ->> 'app_proto' = 'dns' + and event ->> 'dest_port' = '53' + and event ->> 'proto' = 'UDP' +order by + event_timestamp desc; +``` + +```yaml +folder: Network Firewall +``` + +### Monitor Network Firewall Alerts + +Identify and analyze alert events generated by the Network Firewall. This helps monitor security rules and detect potential threats. + +```sql +select + event_timestamp, + event ->> 'src_ip' as source_ip, + event ->> 'dest_ip' as destination_ip, + event ->> 'alert' ->> 'action' as alert_action, + event ->> 'alert' ->> 'signature' as alert_signature, + event ->> 'alert' ->> 'severity' as alert_severity, + event ->> 'alert' ->> 'category' as alert_category, + firewall_name +from + aws_network_firewall_log +where + event ->> 'alert' is not null +order by + event_timestamp desc; +``` + +```yaml +folder: Network Firewall +``` + +## Operational Examples + +### Monitor Traffic Through Specific Firewall + +Retrieve network traffic flows through a specific Network Firewall. This helps analyze firewall behavior and troubleshoot connectivity issues. + +```sql +select + event_timestamp, + event ->> 'src_ip' as source_ip, + event ->> 'src_port' as source_port, + event ->> 'dest_ip' as destination_ip, + event ->> 'dest_port' as destination_port, + event ->> 'proto' as protocol, + event ->> 'app_proto' as app_protocol, + availability_zone +from + aws_network_firewall_log +where + firewall_name = 'main-firewall' +order by + event_timestamp desc +limit 100; +``` + +```yaml +folder: Network Firewall +``` + +### Netflow Traffic Analysis + +Analyze netflow data to understand traffic patterns, including bytes transferred and packet counts. + +```sql +select + event_timestamp, + event ->> 'src_ip' as source_ip, + event ->> 'dest_ip' as destination_ip, + event ->> 'proto' as protocol, + event ->> 'netflow' ->> 'bytes' as bytes, + event ->> 'netflow' ->> 'packets' as packets, + event ->> 'netflow' ->> 'start_time' as start_time, + event ->> 'netflow' ->> 'end_time' as end_time, + firewall_name +from + aws_network_firewall_log +where + event ->> 'event_type' = 'netflow' + and event ->> 'netflow' is not null +order by + event_timestamp desc +limit 100; +``` + +```yaml +folder: Network Firewall +``` + +### TLS Connection Analysis + +Monitor TLS connections and any related errors to identify potential SSL/TLS issues or suspicious encrypted traffic. + +```sql +select + event_timestamp, + event ->> 'src_ip' as source_ip, + event ->> 'dest_ip' as destination_ip, + event ->> 'sni' as server_name_indication, + event ->> 'tls_inspected' as tls_inspected, + event ->> 'tls_error' ->> 'error_message' as tls_error, + firewall_name +from + aws_network_firewall_log +where + event ->> 'app_proto' = 'tls' +order by + event_timestamp desc; +``` + +```yaml +folder: Network Firewall +``` + +## Volume Examples + +### Unusually Large Data Transfers + +Identify unusually large data transfers based on bytes transferred. This helps detect potential data exfiltration or abnormal network usage. + +```sql +select + event_timestamp, + event ->> 'src_ip' as source_ip, + event ->> 'dest_ip' as destination_ip, + event ->> 'netflow' ->> 'bytes' as bytes, + event ->> 'netflow' ->> 'packets' as packets, + event ->> 'proto' as protocol, + firewall_name +from + aws_network_firewall_log +where + (event ->> 'netflow' ->> 'bytes')::bigint > 1000000 -- 1MB +order by + bytes desc; +``` + +```yaml +folder: Network Firewall +``` + +### High-Volume Network Traffic by Source IP + +Find network sources generating a high number of connections, which helps detect possible denial-of-service (DoS) attacks or heavy application usage. + +```sql +select + event ->> 'src_ip' as source_ip, + count(*) as connection_count, + date_trunc('hour', event_timestamp) as traffic_hour +from + aws_network_firewall_log +where + event ->> 'src_ip' is not null +group by + source_ip, + traffic_hour +having + count(*) > 50 +order by + connection_count desc; +``` + +```yaml +folder: Network Firewall +``` + +### Destination Port Usage Analysis + +Analyze traffic by destination port to understand which services are most frequently accessed across your network. + +```sql +select + event ->> 'dest_port' as destination_port, + event ->> 'app_proto' as app_protocol, + count(*) as connection_count +from + aws_network_firewall_log +where + event ->> 'dest_port' is not null +group by + destination_port, + app_protocol +order by + connection_count desc +limit 20; +``` + +```yaml +folder: Network Firewall +``` diff --git a/tables/network_firewall_log/network_firewall_log_table.go b/tables/network_firewall_log/network_firewall_log_table.go index b345b8e9..0756d832 100644 --- a/tables/network_firewall_log/network_firewall_log_table.go +++ b/tables/network_firewall_log/network_firewall_log_table.go @@ -27,7 +27,7 @@ func (c *NetworkFirewallLogTable) Identifier() string { // GetSourceMetadata returns the artifact source configurations for the table. func (c *NetworkFirewallLogTable) GetSourceMetadata() []*table.SourceMetadata[*NetworkFirewallLog] { defaultS3ArtifactConfig := &artifact_source_config.ArtifactSourceConfigImpl{ - FileLayout: utils.ToStringPointer("AWSLogs/%{NUMBER:account_id}/network-firewall/(?flow|alert|tls)/%{DATA:region}/%{DATA:firewall_name}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}/%{NUMBER:account_id}_network-firewall_%{DATA:log_type}_%{DATA:region}_%{DATA:firewall_name}_%{DATA:timestamp}_%{DATA:hash}.log.gz"), + FileLayout: utils.ToStringPointer("AWSLogs/(%{DATA:org_id}/)?%{NUMBER:account_id}/network-firewall/(?flow|alert|tls)/%{DATA:region}/%{DATA:firewall_name}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{HOUR:hour}/%{NUMBER:account_id}_network-firewall_%{DATA:log_type}_%{DATA:region}_%{DATA:firewall_name}_%{DATA:timestamp}_%{DATA:hash}.log.gz"), } return []*table.SourceMetadata[*NetworkFirewallLog]{ From ad401734f2fbe3d09a8323b4b2a9bce3fd1a35c5 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Mon, 31 Mar 2025 15:17:06 +0530 Subject: [PATCH 7/7] Refactor SQL queries in aws_network_firewall_log documentation to improve readability by adding parentheses around JSON extraction. --- docs/tables/aws_network_firewall_log/index.md | 30 ++++- .../aws_network_firewall_log/queries.md | 123 +++++++++--------- 2 files changed, 91 insertions(+), 62 deletions(-) diff --git a/docs/tables/aws_network_firewall_log/index.md b/docs/tables/aws_network_firewall_log/index.md index 4bfa7509..03903d56 100644 --- a/docs/tables/aws_network_firewall_log/index.md +++ b/docs/tables/aws_network_firewall_log/index.md @@ -52,7 +52,7 @@ Analyze traffic by protocol to understand the distribution of network traffic. ```sql select - event ->> 'proto' as protocol, + (event ->> 'proto') as protocol, count(*) as traffic_count from aws_network_firewall_log @@ -68,12 +68,12 @@ Identify the most commonly accessed destination ports to understand traffic patt ```sql select - event ->> 'dest_port' as destination_port, + (event ->> 'dest_port') as destination_port, count(*) as connection_count from aws_network_firewall_log where - event ->> 'dest_port' is not null + (event ->> 'dest_port') is not null group by destination_port order by @@ -87,7 +87,7 @@ Analyze traffic by application protocol to understand what types of applications ```sql select - event ->> 'app_proto' as app_protocol, + (event ->> 'app_proto') as app_protocol, count(*) as connection_count from aws_network_firewall_log @@ -97,6 +97,28 @@ order by connection_count desc; ``` +### Recent Firewall Logs + +View the most recent firewall logs to analyze recent network traffic. + +```sql +select + event_timestamp, + (event ->> 'src_ip') as source_ip, + (event ->> 'src_port') as source_port, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'dest_port') as destination_port, + (event ->> 'proto') as protocol, + (event ->> 'app_proto') as app_protocol, + firewall_name, + availability_zone +from + aws_network_firewall_log +order by + event_timestamp desc +limit 20; +``` + ## Example Configurations ### Collect logs from an S3 bucket diff --git a/docs/tables/aws_network_firewall_log/queries.md b/docs/tables/aws_network_firewall_log/queries.md index 88076f7f..a7b1dc73 100644 --- a/docs/tables/aws_network_firewall_log/queries.md +++ b/docs/tables/aws_network_firewall_log/queries.md @@ -49,7 +49,7 @@ Analyze traffic distribution by protocol to understand what types of network pro ```sql select - event ->> 'proto' as protocol, + (event ->> 'proto') as protocol, count(*) as traffic_count from aws_network_firewall_log @@ -72,17 +72,18 @@ Check if a specific IP is sending or receiving traffic. This is useful for inves ```sql select event_timestamp, - event ->> 'src_ip' as source_ip, - event ->> 'src_port' as source_port, - event ->> 'dest_ip' as destination_ip, - event ->> 'dest_port' as destination_port, - event ->> 'proto' as protocol, - firewall_name + (event ->> 'src_ip') as source_ip, + (event ->> 'src_port') as source_port, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'dest_port') as destination_port, + (event ->> 'proto') as protocol, + firewall_name, + availability_zone from aws_network_firewall_log where - event ->> 'src_ip' = '192.0.2.100' - or event ->> 'dest_ip' = '192.0.2.100' + (event ->> 'src_ip') = '192.0.2.100' + or (event ->> 'dest_ip') = '192.0.2.100' order by event_timestamp desc; ``` @@ -98,17 +99,18 @@ Identify DNS traffic that might indicate command and control (C2) communications ```sql select event_timestamp, - event ->> 'src_ip' as source_ip, - event ->> 'dest_ip' as destination_ip, - event ->> 'dest_port' as destination_port, - event ->> 'proto' as protocol, - firewall_name + (event ->> 'src_ip') as source_ip, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'dest_port') as destination_port, + (event ->> 'proto') as protocol, + firewall_name, + availability_zone from aws_network_firewall_log where - event ->> 'app_proto' = 'dns' - and event ->> 'dest_port' = '53' - and event ->> 'proto' = 'UDP' + (event ->> 'app_proto') = 'dns' + and (event ->> 'dest_port') = '53' + and (event ->> 'proto') = 'UDP' order by event_timestamp desc; ``` @@ -124,17 +126,18 @@ Identify and analyze alert events generated by the Network Firewall. This helps ```sql select event_timestamp, - event ->> 'src_ip' as source_ip, - event ->> 'dest_ip' as destination_ip, - event ->> 'alert' ->> 'action' as alert_action, - event ->> 'alert' ->> 'signature' as alert_signature, - event ->> 'alert' ->> 'severity' as alert_severity, - event ->> 'alert' ->> 'category' as alert_category, - firewall_name + (event ->> 'src_ip') as source_ip, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'alert' ->> 'action') as alert_action, + (event ->> 'alert' ->> 'signature') as alert_signature, + (event ->> 'alert' ->> 'severity') as alert_severity, + (event ->> 'alert' ->> 'category') as alert_category, + firewall_name, + availability_zone from aws_network_firewall_log where - event ->> 'alert' is not null + (event ->> 'alert') is not null order by event_timestamp desc; ``` @@ -152,12 +155,13 @@ Retrieve network traffic flows through a specific Network Firewall. This helps a ```sql select event_timestamp, - event ->> 'src_ip' as source_ip, - event ->> 'src_port' as source_port, - event ->> 'dest_ip' as destination_ip, - event ->> 'dest_port' as destination_port, - event ->> 'proto' as protocol, - event ->> 'app_proto' as app_protocol, + (event ->> 'src_ip') as source_ip, + (event ->> 'src_port') as source_port, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'dest_port') as destination_port, + (event ->> 'proto') as protocol, + (event ->> 'app_proto') as app_protocol, + firewall_name, availability_zone from aws_network_firewall_log @@ -179,19 +183,20 @@ Analyze netflow data to understand traffic patterns, including bytes transferred ```sql select event_timestamp, - event ->> 'src_ip' as source_ip, - event ->> 'dest_ip' as destination_ip, - event ->> 'proto' as protocol, - event ->> 'netflow' ->> 'bytes' as bytes, - event ->> 'netflow' ->> 'packets' as packets, - event ->> 'netflow' ->> 'start_time' as start_time, - event ->> 'netflow' ->> 'end_time' as end_time, - firewall_name + (event ->> 'src_ip') as source_ip, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'proto') as protocol, + (event ->> 'netflow' ->> 'bytes') as bytes, + (event ->> 'netflow' ->> 'packets') as packets, + (event ->> 'netflow' ->> 'start_time') as start_time, + (event ->> 'netflow' ->> 'end_time') as end_time, + firewall_name, + availability_zone from aws_network_firewall_log where - event ->> 'event_type' = 'netflow' - and event ->> 'netflow' is not null + (event ->> 'event_type') = 'netflow' + and (event ->> 'netflow') is not null order by event_timestamp desc limit 100; @@ -208,16 +213,17 @@ Monitor TLS connections and any related errors to identify potential SSL/TLS iss ```sql select event_timestamp, - event ->> 'src_ip' as source_ip, - event ->> 'dest_ip' as destination_ip, - event ->> 'sni' as server_name_indication, - event ->> 'tls_inspected' as tls_inspected, - event ->> 'tls_error' ->> 'error_message' as tls_error, - firewall_name + (event ->> 'src_ip') as source_ip, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'sni') as server_name_indication, + (event ->> 'tls_inspected') as tls_inspected, + (event ->> 'tls_error' ->> 'error_message') as tls_error, + firewall_name, + availability_zone from aws_network_firewall_log where - event ->> 'app_proto' = 'tls' + (event ->> 'app_proto') = 'tls' order by event_timestamp desc; ``` @@ -235,12 +241,13 @@ Identify unusually large data transfers based on bytes transferred. This helps d ```sql select event_timestamp, - event ->> 'src_ip' as source_ip, - event ->> 'dest_ip' as destination_ip, - event ->> 'netflow' ->> 'bytes' as bytes, - event ->> 'netflow' ->> 'packets' as packets, - event ->> 'proto' as protocol, - firewall_name + (event ->> 'src_ip') as source_ip, + (event ->> 'dest_ip') as destination_ip, + (event ->> 'netflow' ->> 'bytes') as bytes, + (event ->> 'netflow' ->> 'packets') as packets, + (event ->> 'proto') as protocol, + firewall_name, + availability_zone from aws_network_firewall_log where @@ -259,7 +266,7 @@ Find network sources generating a high number of connections, which helps detect ```sql select - event ->> 'src_ip' as source_ip, + (event ->> 'src_ip') as source_ip, count(*) as connection_count, date_trunc('hour', event_timestamp) as traffic_hour from @@ -285,13 +292,13 @@ Analyze traffic by destination port to understand which services are most freque ```sql select - event ->> 'dest_port' as destination_port, - event ->> 'app_proto' as app_protocol, + (event ->> 'dest_port') as destination_port, + (event ->> 'app_proto') as app_protocol, count(*) as connection_count from aws_network_firewall_log where - event ->> 'dest_port' is not null + (event ->> 'dest_port') is not null group by destination_port, app_protocol