From 9a831c68362feb6970ecbd4ec847ccf429a46a47 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Mon, 7 Apr 2025 14:56:42 +0530 Subject: [PATCH 01/12] Add table `aws_securityhub_finding` --- aws/plugin.go | 2 + docs/sources/aws_s3_bucket.md | 1 + .../securityhub_finding.go | 217 ++++++++++++++++++ .../securityhub_finding_mapper.go | 38 +++ .../securityhub_finding_table.go | 71 ++++++ 5 files changed, 329 insertions(+) create mode 100644 tables/securityhub_finding/securityhub_finding.go create mode 100644 tables/securityhub_finding/securityhub_finding_mapper.go create mode 100644 tables/securityhub_finding/securityhub_finding_table.go diff --git a/aws/plugin.go b/aws/plugin.go index 2d3a36c..217318d 100755 --- a/aws/plugin.go +++ b/aws/plugin.go @@ -16,6 +16,7 @@ import ( "github.com/turbot/tailpipe-plugin-aws/tables/s3_server_access_log" "github.com/turbot/tailpipe-plugin-aws/tables/vpc_flow_log" "github.com/turbot/tailpipe-plugin-aws/tables/waf_traffic_log" + "github.com/turbot/tailpipe-plugin-aws/tables/securityhub_finding" "github.com/turbot/tailpipe-plugin-sdk/plugin" "github.com/turbot/tailpipe-plugin-sdk/row_source" "github.com/turbot/tailpipe-plugin-sdk/table" @@ -38,6 +39,7 @@ func init() { table.RegisterTable[*guardduty_finding.GuardDutyFinding, *guardduty_finding.GuardDutyFindingTable]() table.RegisterTable[*nlb_access_log.NlbAccessLog, *nlb_access_log.NlbAccessLogTable]() table.RegisterTable[*s3_server_access_log.S3ServerAccessLog, *s3_server_access_log.S3ServerAccessLogTable]() + table.RegisterTable[*securityhub_finding.SecurityHubFinding, *securityhub_finding.SecurityHubFindingTable]() table.RegisterTable[*vpc_flow_log.VpcFlowLog, *vpc_flow_log.VpcFlowLogTable]() table.RegisterTable[*waf_traffic_log.WafTrafficLog, *waf_traffic_log.WafTrafficLogTable]() diff --git a/docs/sources/aws_s3_bucket.md b/docs/sources/aws_s3_bucket.md index 2d25877..0d1a4dc 100644 --- a/docs/sources/aws_s3_bucket.md +++ b/docs/sources/aws_s3_bucket.md @@ -77,6 +77,7 @@ The following tables define their own default values for certain source argument - **[aws_guardduty_finding](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_guardduty_finding#aws_s3_bucket)** - **[aws_nlb_access_log](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_nlb_access_log#aws_s3_bucket)** - **[aws_s3_server_access_log](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_s3_server_access_log#aws_s3_bucket)** +- **[aws_securityhub_finding](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_securityhub_finding#aws_s3_bucket)** - **[aws_cost_and_usage_focus](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_cost_and_usage_focus#aws_s3_bucket)** - **[aws_cost_and_usage_report](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_cost_and_usage_report#aws_s3_bucket)** - **[aws_cost_optimization_recommendation](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_cost_optimization_recommendation#aws_s3_bucket)** diff --git a/tables/securityhub_finding/securityhub_finding.go b/tables/securityhub_finding/securityhub_finding.go new file mode 100644 index 0000000..360d306 --- /dev/null +++ b/tables/securityhub_finding/securityhub_finding.go @@ -0,0 +1,217 @@ +package securityhub_finding + +import ( + "time" + + "github.com/turbot/tailpipe-plugin-sdk/schema" +) + +type SecurityHubFinding struct { + schema.CommonFields + + // Top level fields + Version *string `json:"version,omitempty"` + ID *string `json:"id,omitempty"` + DetailType *string `json:"detail_type,omitempty"` + Source *string `json:"source,omitempty"` + Account *string `json:"account,omitempty"` + Time *time.Time `json:"time,omitempty"` + Region *string `json:"region,omitempty"` + Resources []*string `json:"resources,omitempty"` + Detail *DetailFindingsData `json:"detail,omitempty"` +} + +// DetailFindingsData maps the `detail` field containing findings +type DetailFindingsData struct { + Findings []*Finding `json:"findings,omitempty" parquet:"name=findings, type=JSON"` + AwsRegion *string `json:"awsRegion" parquet:"name=aws_region"` + EventCategory *string `json:"eventCategory" parquet:"name=event_category"` + EventID *string `json:"eventID" parquet:"name=event_id"` + EventName *string `json:"eventName" parquet:"name=event_name"` + EventSource *string `json:"eventSource" parquet:"name=event_source"` + EventTime time.Time `json:"eventTime" parquet:"name=event_time"` + EventType *string `json:"eventType" parquet:"name=event_type"` + EventVersion *string `json:"eventVersion" parquet:"name=event_version"` + ManagementEvent bool `json:"managementEvent" parquet:"name=management_event"` + ReadOnly bool `json:"readOnly" parquet:"name=read_only"` + RecipientAccountID *string `json:"recipientAccountId" parquet:"name=recipient_account_id"` + RequestID *string `json:"requestID" parquet:"name=request_id"` + RequestParameters *map[string]interface{} `json:"requestParameters" parquet:"name=request_parameters, type=JSON"` + ResponseElements *map[string]interface{} `json:"responseElements" parquet:"name=response_elements, type=JSON"` + SourceIPAddress *string `json:"sourceIPAddress" parquet:"name=source_ip_address"` + UserAgent *string `json:"userAgent" parquet:"name=user_agent"` + UserIdentity SecurityHubUserIdentity `json:"userIdentity" parquet:"name=user_identity, type=JSON"` +} + +// Finding maps the individual findings in the detail +type Finding struct { + ProductArn *string `json:"ProductArn,omitempty"` + Types []*string `json:"Types,omitempty"` + Description *string `json:"Description,omitempty"` + Compliance *Compliance `json:"Compliance,omitempty"` + ProductName *string `json:"ProductName,omitempty"` + FirstObservedAt *time.Time `json:"FirstObservedAt,omitempty"` + CreatedAt *time.Time `json:"CreatedAt,omitempty"` + LastObservedAt *time.Time `json:"LastObservedAt,omitempty"` + CompanyName *string `json:"CompanyName,omitempty"` + FindingProviderFields *FindingProviderFields `json:"FindingProviderFields,omitempty"` + ProductFields map[string]string `json:"ProductFields,omitempty"` + Remediation *Remediation `json:"Remediation,omitempty"` + SchemaVersion *string `json:"SchemaVersion,omitempty"` + GeneratorId *string `json:"GeneratorId,omitempty"` + RecordState *string `json:"RecordState,omitempty"` + Title *string `json:"Title,omitempty"` + Workflow *Workflow `json:"Workflow,omitempty"` + Severity *Severity `json:"Severity,omitempty"` + UpdatedAt *time.Time `json:"UpdatedAt,omitempty"` + WorkflowState *string `json:"WorkflowState,omitempty"` + AwsAccountId *string `json:"AwsAccountId,omitempty"` + Region *string `json:"Region,omitempty"` + Id *string `json:"Id,omitempty"` + Resources []*FindingResource `json:"Resources,omitempty"` + ProcessedAt *time.Time `json:"ProcessedAt,omitempty"` +} + +// Supporting structs for nested fields +type Compliance struct { + Status *string `json:"Status,omitempty"` + SecurityControlId *string `json:"SecurityControlId,omitempty"` + AssociatedStandards []*AssociatedStandard `json:"AssociatedStandards,omitempty"` + SecurityControlParameters []*SecurityControlParameter `json:"SecurityControlParameters,omitempty"` +} + +type AssociatedStandard struct { + StandardsId *string `json:"StandardsId,omitempty"` +} + +type SecurityControlParameter struct { + Value []string `json:"Value,omitempty"` + Name *string `json:"Name,omitempty"` +} + +type FindingProviderFields struct { + Types []*string `json:"Types,omitempty"` + Severity *Severity `json:"Severity,omitempty"` +} + +type Remediation struct { + Recommendation *Recommendation `json:"Recommendation,omitempty"` +} + +type Recommendation struct { + Text *string `json:"Text,omitempty"` + Url *string `json:"Url,omitempty"` +} + +type Workflow struct { + Status *string `json:"Status,omitempty"` +} + +type Severity struct { + Normalized *int `json:"Normalized,omitempty"` + Label *string `json:"Label,omitempty"` + Original *string `json:"Original,omitempty"` +} + +type FindingResource struct { + Partition *string `json:"Partition,omitempty"` + Type *string `json:"Type,omitempty"` + Details *ResourceDetails `json:"Details,omitempty"` + Region *string `json:"Region,omitempty"` + Id *string `json:"Id,omitempty"` +} + +type ResourceDetails struct { + AwsLambdaFunction *AwsLambdaFunction `json:"AwsLambdaFunction,omitempty"` +} + +type AwsLambdaFunction struct { + LastModified *time.Time `json:"LastModified,omitempty"` + Role *string `json:"Role,omitempty"` + FunctionName *string `json:"FunctionName,omitempty"` + MemorySize *int `json:"MemorySize,omitempty"` + Runtime *string `json:"Runtime,omitempty"` + TracingConfig *TracingConfig `json:"TracingConfig,omitempty"` + Version *string `json:"Version,omitempty"` + Timeout *int `json:"Timeout,omitempty"` + Handler *string `json:"Handler,omitempty"` + CodeSha256 *string `json:"CodeSha256,omitempty"` + RevisionId *string `json:"RevisionId,omitempty"` +} + +type TracingConfig struct { + Mode *string `json:"Mode,omitempty"` +} + +type SecurityHubUserIdentity struct { + AccessKeyID *string `json:"accessKeyId"` + AccountID *string `json:"accountId"` + Arn *string `json:"arn"` + PrincipalID *string `json:"principalId"` + SessionContext SecurityHubSessionContext `json:"sessionContext"` + Type *string `json:"type"` +} + +type SecurityHubSessionContext struct { + Attributes Attributes `json:"attributes"` + SessionIssuer SecurityHubSessionIssuer `json:"sessionIssuer"` +} + +type Attributes struct { + CreationDate time.Time `json:"creationDate"` + MfaAuthenticated *string `json:"mfaAuthenticated"` +} + +type SecurityHubSessionIssuer struct { + AccountID string `json:"accountId"` + Arn string `json:"arn"` + PrincipalID string `json:"principalId"` + Type string `json:"type"` + UserName string `json:"userName"` +} + +func (c *SecurityHubFinding) GetColumnDescriptions() map[string]string { + return map[string]string{ + // Top level fields + "version": "The version of the event format.", + "id": "The unique identifier for the event.", + "detail_type": "The type of the event detail.", + "source": "The service or system that generated the event.", + "account": "The AWS account ID where the finding was generated.", + "time": "The timestamp when the event was generated.", + "region": "The AWS region where the finding was generated.", + "resources": "The list of AWS resources associated with the finding.", + + // Detail fields + "detail": "The detailed information about the security finding.", + + // Finding fields + "product_arn": "The ARN of the AWS security product that generated the finding.", + "types": "The list of types assigned to the finding.", + "description": "A detailed description of the security finding.", + "compliance": "Information about the finding's compliance status.", + "product_name": "The name of the security product that generated the finding.", + "first_observed_at": "The timestamp when the finding was first observed.", + "created_at": "The timestamp when the finding was created.", + "last_observed_at": "The timestamp when the finding was last observed.", + "company_name": "The name of the company that provides the security product.", + "product_fields": "Additional fields provided by the security product.", + "remediation": "Recommended steps to remediate the finding.", + "schema_version": "The version of the finding format schema.", + "generator_id": "The identifier of the system that generated the finding.", + "record_state": "The current state of the finding record.", + "title": "A short human-readable title for the finding.", + "workflow": "Information about the finding's workflow status.", + "severity": "The severity level of the finding.", + "updated_at": "The timestamp when the finding was last updated.", + "workflow_state": "The current state of the finding in the workflow.", + "aws_account_id": "The AWS account ID associated with the finding.", + "processed_at": "The timestamp when the finding was processed.", + + // Tailpipe-specific metadata fields + "tp_akas": "The list of AWS ARNs associated with the finding.", + "tp_index": "The AWS account ID where the finding was generated.", + "tp_timestamp": "The timestamp when the finding was generated.", + "tp_date": "The date when the finding was generated, truncated to day.", + } +} diff --git a/tables/securityhub_finding/securityhub_finding_mapper.go b/tables/securityhub_finding/securityhub_finding_mapper.go new file mode 100644 index 0000000..0d0a6d4 --- /dev/null +++ b/tables/securityhub_finding/securityhub_finding_mapper.go @@ -0,0 +1,38 @@ +package securityhub_finding + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/turbot/tailpipe-plugin-sdk/mappers" +) + +type SecurityHubFindingMapper struct { +} + +func (m *SecurityHubFindingMapper) Identifier() string { + return "security_hub_finding_mapper" +} + +func (m *SecurityHubFindingMapper) Map(_ context.Context, a any, _ ...mappers.MapOption[*SecurityHubFinding]) (*SecurityHubFinding, error) { + var b SecurityHubFinding + + switch data := a.(type) { + case []byte: + if err := json.Unmarshal(data, &b); err != nil { + return nil, fmt.Errorf("error unmarshalling row data: %w", err) + } + case string: + if err := json.Unmarshal([]byte(data), &b); err != nil { + return nil, fmt.Errorf("error unmarshalling row data: %w", err) + } + case SecurityHubFinding: + b = data + default: + return nil, fmt.Errorf("expected byte[], string or SecurityHubFinding, got %T", a) + } + + return &b, nil + +} \ No newline at end of file diff --git a/tables/securityhub_finding/securityhub_finding_table.go b/tables/securityhub_finding/securityhub_finding_table.go new file mode 100644 index 0000000..1dc442d --- /dev/null +++ b/tables/securityhub_finding/securityhub_finding_table.go @@ -0,0 +1,71 @@ +package securityhub_finding + +import ( + "time" + + "github.com/rs/xid" + "github.com/turbot/pipe-fittings/v2/utils" + "github.com/turbot/tailpipe-plugin-aws/sources/s3_bucket" + "github.com/turbot/tailpipe-plugin-aws/tables" + "github.com/turbot/tailpipe-plugin-sdk/artifact_source" + "github.com/turbot/tailpipe-plugin-sdk/artifact_source_config" + "github.com/turbot/tailpipe-plugin-sdk/constants" + "github.com/turbot/tailpipe-plugin-sdk/row_source" + "github.com/turbot/tailpipe-plugin-sdk/schema" + "github.com/turbot/tailpipe-plugin-sdk/table" +) + +const SecurityHubFindingTableIdentifier = "aws_securityhub_finding" + +type SecurityHubFindingTable struct{} + +func (c *SecurityHubFindingTable) Identifier() string { + return SecurityHubFindingTableIdentifier +} + +func (c *SecurityHubFindingTable) GetSourceMetadata() []*table.SourceMetadata[*SecurityHubFinding] { + defaultS3ArtifactConfig := &artifact_source_config.ArtifactSourceConfigImpl{ + FileLayout: utils.ToStringPointer("AWSLogs/(%{DATA:org_id}/)?%{NUMBER:account_id}/SecurityHub/%{DATA:region}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{DATA}.json.gz"), + } + + return []*table.SourceMetadata[*SecurityHubFinding]{ + { + // S3 artifact source + SourceName: s3_bucket.AwsS3BucketSourceIdentifier, + Mapper: &SecurityHubFindingMapper{}, + Options: []row_source.RowSourceOption{ + artifact_source.WithDefaultArtifactSourceConfig(defaultS3ArtifactConfig), + }, + }, + { + SourceName: constants.ArtifactSourceIdentifier, + Mapper: &SecurityHubFindingMapper{}, + }, + } +} + +func (c *SecurityHubFindingTable) EnrichRow(row *SecurityHubFinding, sourceEnrichmentFields schema.SourceEnrichment) (*SecurityHubFinding, error) { + row.CommonFields = sourceEnrichmentFields.CommonFields + + row.TpID = xid.New().String() + row.TpIngestTimestamp = time.Now() + + for _, resource := range row.Resources { + newAkas := tables.AwsAkasFromArn(*resource) + row.TpAkas = append(row.TpAkas, newAkas...) + } + + if row.Time != nil { + row.TpTimestamp = *row.Time + row.TpDate = row.Time.Truncate(24 * time.Hour) + } + if row.Account != nil { + row.TpIndex = *row.Account + } + + return row, nil +} + +func (c *SecurityHubFindingTable) GetDescription() string { + return "AWS Security Hub findings provide detailed information about potential security issues and compliance violations detected across your AWS accounts and resources. This table captures comprehensive security findings from various AWS security services and partner integrations, including details about the affected resources, severity levels, compliance status, and recommended remediation steps." +} From 54153cb5a6a9f4dcf34cf1b4ff4211bf3da2c78f Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Wed, 16 Apr 2025 18:47:55 +0530 Subject: [PATCH 02/12] Update aws_securityhub_finding table --- tables/securityhub_finding/securityhub_finding_table.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tables/securityhub_finding/securityhub_finding_table.go b/tables/securityhub_finding/securityhub_finding_table.go index 1dc442d..c720f44 100644 --- a/tables/securityhub_finding/securityhub_finding_table.go +++ b/tables/securityhub_finding/securityhub_finding_table.go @@ -23,7 +23,7 @@ func (c *SecurityHubFindingTable) Identifier() string { return SecurityHubFindingTableIdentifier } -func (c *SecurityHubFindingTable) GetSourceMetadata() []*table.SourceMetadata[*SecurityHubFinding] { +func (c *SecurityHubFindingTable) GetSourceMetadata() ([]*table.SourceMetadata[*SecurityHubFinding], error) { defaultS3ArtifactConfig := &artifact_source_config.ArtifactSourceConfigImpl{ FileLayout: utils.ToStringPointer("AWSLogs/(%{DATA:org_id}/)?%{NUMBER:account_id}/SecurityHub/%{DATA:region}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{DATA}.json.gz"), } @@ -35,13 +35,17 @@ func (c *SecurityHubFindingTable) GetSourceMetadata() []*table.SourceMetadata[*S Mapper: &SecurityHubFindingMapper{}, Options: []row_source.RowSourceOption{ artifact_source.WithDefaultArtifactSourceConfig(defaultS3ArtifactConfig), + artifact_source.WithRowPerLine(), }, }, { SourceName: constants.ArtifactSourceIdentifier, Mapper: &SecurityHubFindingMapper{}, + Options: []row_source.RowSourceOption{ + artifact_source.WithRowPerLine(), + }, }, - } + }, nil } func (c *SecurityHubFindingTable) EnrichRow(row *SecurityHubFinding, sourceEnrichmentFields schema.SourceEnrichment) (*SecurityHubFinding, error) { From 4622d467584fd1a89abc1200dbdf98d53d97fcca Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Wed, 16 Apr 2025 20:43:35 +0530 Subject: [PATCH 03/12] Remove redundant row option from `SecurityHubFindingTable` configuration --- tables/securityhub_finding/securityhub_finding_table.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tables/securityhub_finding/securityhub_finding_table.go b/tables/securityhub_finding/securityhub_finding_table.go index c720f44..4cf2c4f 100644 --- a/tables/securityhub_finding/securityhub_finding_table.go +++ b/tables/securityhub_finding/securityhub_finding_table.go @@ -35,7 +35,6 @@ func (c *SecurityHubFindingTable) GetSourceMetadata() ([]*table.SourceMetadata[* Mapper: &SecurityHubFindingMapper{}, Options: []row_source.RowSourceOption{ artifact_source.WithDefaultArtifactSourceConfig(defaultS3ArtifactConfig), - artifact_source.WithRowPerLine(), }, }, { From 7cf6e1737f12c0f936eed4bb5ed7d094b4991814 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Thu, 17 Apr 2025 19:09:27 +0530 Subject: [PATCH 04/12] Update SecurityHubFinding --- .../securityhub_finding.go | 85 +++---------------- .../securityhub_finding_table.go | 3 - 2 files changed, 11 insertions(+), 77 deletions(-) diff --git a/tables/securityhub_finding/securityhub_finding.go b/tables/securityhub_finding/securityhub_finding.go index 360d306..4ce781d 100644 --- a/tables/securityhub_finding/securityhub_finding.go +++ b/tables/securityhub_finding/securityhub_finding.go @@ -3,6 +3,7 @@ package securityhub_finding import ( "time" + "github.com/aws/aws-sdk-go-v2/service/securityhub/types" "github.com/turbot/tailpipe-plugin-sdk/schema" ) @@ -10,20 +11,20 @@ type SecurityHubFinding struct { schema.CommonFields // Top level fields - Version *string `json:"version,omitempty"` - ID *string `json:"id,omitempty"` - DetailType *string `json:"detail_type,omitempty"` - Source *string `json:"source,omitempty"` - Account *string `json:"account,omitempty"` - Time *time.Time `json:"time,omitempty"` - Region *string `json:"region,omitempty"` - Resources []*string `json:"resources,omitempty"` - Detail *DetailFindingsData `json:"detail,omitempty"` + Version *string `json:"version,omitempty"` + ID *string `json:"id,omitempty"` + DetailType *string `json:"detail_type,omitempty"` + Source *string `json:"source,omitempty"` + Account *string `json:"account,omitempty"` + Time *time.Time `json:"time,omitempty"` + Region *string `json:"region,omitempty"` + Resources []*string `json:"resources,omitempty"` + Detail *DetailFindingsData `json:"detail,omitempty" parquet:"name=detail, type=JSON"` + Findings *types.AwsSecurityFinding `json:"findings,omitempty" parquet:"name=findings, type=JSON"` } // DetailFindingsData maps the `detail` field containing findings type DetailFindingsData struct { - Findings []*Finding `json:"findings,omitempty" parquet:"name=findings, type=JSON"` AwsRegion *string `json:"awsRegion" parquet:"name=aws_region"` EventCategory *string `json:"eventCategory" parquet:"name=event_category"` EventID *string `json:"eventID" parquet:"name=event_id"` @@ -43,43 +44,6 @@ type DetailFindingsData struct { UserIdentity SecurityHubUserIdentity `json:"userIdentity" parquet:"name=user_identity, type=JSON"` } -// Finding maps the individual findings in the detail -type Finding struct { - ProductArn *string `json:"ProductArn,omitempty"` - Types []*string `json:"Types,omitempty"` - Description *string `json:"Description,omitempty"` - Compliance *Compliance `json:"Compliance,omitempty"` - ProductName *string `json:"ProductName,omitempty"` - FirstObservedAt *time.Time `json:"FirstObservedAt,omitempty"` - CreatedAt *time.Time `json:"CreatedAt,omitempty"` - LastObservedAt *time.Time `json:"LastObservedAt,omitempty"` - CompanyName *string `json:"CompanyName,omitempty"` - FindingProviderFields *FindingProviderFields `json:"FindingProviderFields,omitempty"` - ProductFields map[string]string `json:"ProductFields,omitempty"` - Remediation *Remediation `json:"Remediation,omitempty"` - SchemaVersion *string `json:"SchemaVersion,omitempty"` - GeneratorId *string `json:"GeneratorId,omitempty"` - RecordState *string `json:"RecordState,omitempty"` - Title *string `json:"Title,omitempty"` - Workflow *Workflow `json:"Workflow,omitempty"` - Severity *Severity `json:"Severity,omitempty"` - UpdatedAt *time.Time `json:"UpdatedAt,omitempty"` - WorkflowState *string `json:"WorkflowState,omitempty"` - AwsAccountId *string `json:"AwsAccountId,omitempty"` - Region *string `json:"Region,omitempty"` - Id *string `json:"Id,omitempty"` - Resources []*FindingResource `json:"Resources,omitempty"` - ProcessedAt *time.Time `json:"ProcessedAt,omitempty"` -} - -// Supporting structs for nested fields -type Compliance struct { - Status *string `json:"Status,omitempty"` - SecurityControlId *string `json:"SecurityControlId,omitempty"` - AssociatedStandards []*AssociatedStandard `json:"AssociatedStandards,omitempty"` - SecurityControlParameters []*SecurityControlParameter `json:"SecurityControlParameters,omitempty"` -} - type AssociatedStandard struct { StandardsId *string `json:"StandardsId,omitempty"` } @@ -89,38 +53,11 @@ type SecurityControlParameter struct { Name *string `json:"Name,omitempty"` } -type FindingProviderFields struct { - Types []*string `json:"Types,omitempty"` - Severity *Severity `json:"Severity,omitempty"` -} - -type Remediation struct { - Recommendation *Recommendation `json:"Recommendation,omitempty"` -} - type Recommendation struct { Text *string `json:"Text,omitempty"` Url *string `json:"Url,omitempty"` } -type Workflow struct { - Status *string `json:"Status,omitempty"` -} - -type Severity struct { - Normalized *int `json:"Normalized,omitempty"` - Label *string `json:"Label,omitempty"` - Original *string `json:"Original,omitempty"` -} - -type FindingResource struct { - Partition *string `json:"Partition,omitempty"` - Type *string `json:"Type,omitempty"` - Details *ResourceDetails `json:"Details,omitempty"` - Region *string `json:"Region,omitempty"` - Id *string `json:"Id,omitempty"` -} - type ResourceDetails struct { AwsLambdaFunction *AwsLambdaFunction `json:"AwsLambdaFunction,omitempty"` } diff --git a/tables/securityhub_finding/securityhub_finding_table.go b/tables/securityhub_finding/securityhub_finding_table.go index 4cf2c4f..222e208 100644 --- a/tables/securityhub_finding/securityhub_finding_table.go +++ b/tables/securityhub_finding/securityhub_finding_table.go @@ -40,9 +40,6 @@ func (c *SecurityHubFindingTable) GetSourceMetadata() ([]*table.SourceMetadata[* { SourceName: constants.ArtifactSourceIdentifier, Mapper: &SecurityHubFindingMapper{}, - Options: []row_source.RowSourceOption{ - artifact_source.WithRowPerLine(), - }, }, }, nil } From 3ecf82948a7c00a324615f02a0956516cb7b1b91 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Fri, 18 Apr 2025 11:14:22 +0530 Subject: [PATCH 05/12] Update security hub finding --- .../securityhub_finding.go | 49 +++++++++++++++---- .../securityhub_finding_extractor.go | 47 ++++++++++++++++++ 2 files changed, 86 insertions(+), 10 deletions(-) create mode 100644 tables/securityhub_finding/securityhub_finding_extractor.go diff --git a/tables/securityhub_finding/securityhub_finding.go b/tables/securityhub_finding/securityhub_finding.go index 4ce781d..d632597 100644 --- a/tables/securityhub_finding/securityhub_finding.go +++ b/tables/securityhub_finding/securityhub_finding.go @@ -11,16 +11,45 @@ type SecurityHubFinding struct { schema.CommonFields // Top level fields - Version *string `json:"version,omitempty"` - ID *string `json:"id,omitempty"` - DetailType *string `json:"detail_type,omitempty"` - Source *string `json:"source,omitempty"` - Account *string `json:"account,omitempty"` - Time *time.Time `json:"time,omitempty"` - Region *string `json:"region,omitempty"` - Resources []*string `json:"resources,omitempty"` - Detail *DetailFindingsData `json:"detail,omitempty" parquet:"name=detail, type=JSON"` - Findings *types.AwsSecurityFinding `json:"findings,omitempty" parquet:"name=findings, type=JSON"` + Version *string `json:"version,omitempty"` + ID *string `json:"id,omitempty"` + DetailType *string `json:"detail_type,omitempty"` + Source *string `json:"source,omitempty"` + Account *string `json:"account,omitempty"` + Time *time.Time `json:"time,omitempty"` + Region *string `json:"region,omitempty"` + // Detail *DetailFindingsData `json:"detail,omitempty" parquet:"name=detail, type=JSON"` + // Finding array schema + AwsAccountId *string `json:"awsAccountId" parquet:"name=aws_account_id"` + CreatedAt *string `json:"createdAt" parquet:"name=created_at"` + Description *string `json:"description" parquet:"name=description"` + GeneratorId *string `json:"generatorId" parquet:"name=generator_id"` + FindingId *string `json:"findingId" parquet:"name=finding_id"` + ProductArn *string `json:"productArn" parquet:"name=product_arn"` + ProductFields *map[string]interface{} `json:"productFields" parquet:"name=product_fields, type=JSON"` + ProductName *string `json:"productName" parquet:"name=product_name"` + Remediation *string `json:"remediation" parquet:"name=remediation"` + Resources []types.Resource `json:"resources" parquet:"name=resources, type=JSON"` + SchemaVersion *string `json:"schemaVersion" parquet:"name=schema_version"` + Title *string `json:"title" parquet:"name=title"` + // It is for schema only + AwsRegion *string `json:"awsRegion" parquet:"name=aws_region"` + EventCategory *string `json:"eventCategory" parquet:"name=event_category"` + EventID *string `json:"eventID" parquet:"name=event_id"` + EventName *string `json:"eventName" parquet:"name=event_name"` + EventSource *string `json:"eventSource" parquet:"name=event_source"` + EventTime time.Time `json:"eventTime" parquet:"name=event_time"` + EventType *string `json:"eventType" parquet:"name=event_type"` + EventVersion *string `json:"eventVersion" parquet:"name=event_version"` + ManagementEvent bool `json:"managementEvent" parquet:"name=management_event"` + ReadOnly bool `json:"readOnly" parquet:"name=read_only"` + RecipientAccountID *string `json:"recipientAccountId" parquet:"name=recipient_account_id"` + RequestID *string `json:"requestID" parquet:"name=request_id"` + RequestParameters *map[string]interface{} `json:"requestParameters" parquet:"name=request_parameters, type=JSON"` + ResponseElements *map[string]interface{} `json:"responseElements" parquet:"name=response_elements, type=JSON"` + SourceIPAddress *string `json:"sourceIPAddress" parquet:"name=source_ip_address"` + UserAgent *string `json:"userAgent" parquet:"name=user_agent"` + UserIdentity SecurityHubUserIdentity `json:"userIdentity" parquet:"name=user_identity, type=JSON"` } // DetailFindingsData maps the `detail` field containing findings diff --git a/tables/securityhub_finding/securityhub_finding_extractor.go b/tables/securityhub_finding/securityhub_finding_extractor.go new file mode 100644 index 0000000..49b6e0d --- /dev/null +++ b/tables/securityhub_finding/securityhub_finding_extractor.go @@ -0,0 +1,47 @@ +package securityhub_finding + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + + "github.com/turbot/tailpipe-plugin-sdk/artifact_source" +) + +// SecurityHubFindingExtractor is an extractor that receives JSON serialised CloudTrailLogBatch objects +// and extracts SecurityHubFinding records from them +type SecurityHubFindingExtractor struct { +} + +// NewCloudTrailLogExtractor creates a new SecurityHubFindingExtractor +func NewCloudTrailLogExtractor() artifact_source.Extractor { + return &SecurityHubFindingExtractor{} +} + +func (c *SecurityHubFindingExtractor) Identifier() string { + return "cloudtrail_log_extractor" +} + +// Extract unmarshalls the artifact data as an CloudTrailLogBatch and returns the SecurityHubFinding records +func (c *SecurityHubFindingExtractor) Extract(_ context.Context, a any) ([]any, error) { + // the expected input type is a JSON byte[] deserializable to CloudTrailLogBatch + jsonBytes, ok := a.([]byte) + if !ok { + return nil, fmt.Errorf("expected byte[], got %T", a) + } + + // decode json ito CloudTrailLogBatch + var log SecurityHubFinding + err := json.Unmarshal(jsonBytes, &log) + if err != nil { + return nil, fmt.Errorf("error decoding json: %w", err) + } + + slog.Debug("SecurityHubFindingExtractor", "record count", len(log.Findings)) + var res = make([]any, len(log.Findings)) + for i, record := range log.Findings { + res[i] = &record + } + return res, nil +} From e9ba805b1036288360349b23a6782b8c4eb3769f Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Fri, 18 Apr 2025 19:00:09 +0530 Subject: [PATCH 06/12] added all the finding fields to top level --- .../securityhub_finding.go | 244 ++++++++++-------- 1 file changed, 142 insertions(+), 102 deletions(-) diff --git a/tables/securityhub_finding/securityhub_finding.go b/tables/securityhub_finding/securityhub_finding.go index d632597..fbace03 100644 --- a/tables/securityhub_finding/securityhub_finding.go +++ b/tables/securityhub_finding/securityhub_finding.go @@ -11,50 +11,59 @@ type SecurityHubFinding struct { schema.CommonFields // Top level fields - Version *string `json:"version,omitempty"` - ID *string `json:"id,omitempty"` - DetailType *string `json:"detail_type,omitempty"` - Source *string `json:"source,omitempty"` - Account *string `json:"account,omitempty"` - Time *time.Time `json:"time,omitempty"` - Region *string `json:"region,omitempty"` - // Detail *DetailFindingsData `json:"detail,omitempty" parquet:"name=detail, type=JSON"` - // Finding array schema - AwsAccountId *string `json:"awsAccountId" parquet:"name=aws_account_id"` - CreatedAt *string `json:"createdAt" parquet:"name=created_at"` - Description *string `json:"description" parquet:"name=description"` - GeneratorId *string `json:"generatorId" parquet:"name=generator_id"` - FindingId *string `json:"findingId" parquet:"name=finding_id"` - ProductArn *string `json:"productArn" parquet:"name=product_arn"` - ProductFields *map[string]interface{} `json:"productFields" parquet:"name=product_fields, type=JSON"` - ProductName *string `json:"productName" parquet:"name=product_name"` - Remediation *string `json:"remediation" parquet:"name=remediation"` - Resources []types.Resource `json:"resources" parquet:"name=resources, type=JSON"` - SchemaVersion *string `json:"schemaVersion" parquet:"name=schema_version"` - Title *string `json:"title" parquet:"name=title"` - // It is for schema only - AwsRegion *string `json:"awsRegion" parquet:"name=aws_region"` - EventCategory *string `json:"eventCategory" parquet:"name=event_category"` - EventID *string `json:"eventID" parquet:"name=event_id"` - EventName *string `json:"eventName" parquet:"name=event_name"` - EventSource *string `json:"eventSource" parquet:"name=event_source"` - EventTime time.Time `json:"eventTime" parquet:"name=event_time"` - EventType *string `json:"eventType" parquet:"name=event_type"` - EventVersion *string `json:"eventVersion" parquet:"name=event_version"` - ManagementEvent bool `json:"managementEvent" parquet:"name=management_event"` - ReadOnly bool `json:"readOnly" parquet:"name=read_only"` - RecipientAccountID *string `json:"recipientAccountId" parquet:"name=recipient_account_id"` - RequestID *string `json:"requestID" parquet:"name=request_id"` - RequestParameters *map[string]interface{} `json:"requestParameters" parquet:"name=request_parameters, type=JSON"` - ResponseElements *map[string]interface{} `json:"responseElements" parquet:"name=response_elements, type=JSON"` - SourceIPAddress *string `json:"sourceIPAddress" parquet:"name=source_ip_address"` - UserAgent *string `json:"userAgent" parquet:"name=user_agent"` - UserIdentity SecurityHubUserIdentity `json:"userIdentity" parquet:"name=user_identity, type=JSON"` -} + Version *string `json:"version,omitempty"` + ID *string `json:"id,omitempty"` + DetailType *string `json:"detail_type,omitempty"` + Source *string `json:"source,omitempty"` + Account *string `json:"account,omitempty"` + Time *time.Time `json:"time,omitempty"` + Region *string `json:"region,omitempty"` -// DetailFindingsData maps the `detail` field containing findings -type DetailFindingsData struct { - AwsRegion *string `json:"awsRegion" parquet:"name=aws_region"` + // Finding array schema + AwsAccountName *string `json:"aws_account_name" parquet:"name=aws_account_name"` + CompanyName *string `json:"company_name" parquet:"name=company_name"` + Compliance *types.Compliance `json:"compliance" parquet:"name=compliance"` + Confidence *int32 `json:"confidence" parquet:"name=confidence"` + CreatedAt *string `json:"createdAt" parquet:"name=created_at"` + Criticality *int32 `json:"criticality parquet:"name=criticality"` + Description *string `json:"description" parquet:"name=description"` + FirstObservedAt *string `json:"first_observed_at" parquet:"name=first_observed_at"` + GeneratorId *string `json:"generatorId" parquet:"name=generator_id"` + GeneratorDetails *types.GeneratorDetails `json:"generator_details" parquet:"name=generator_details"` + FindingId *string `json:"findingId" parquet:"name=finding_id"` + FindingRegion *string `json:"findingRegion" parquet:"name=finding_region"` + LastObservedAt *string `json:"last_observed_at" parquet:"name=last_observed_at"` + Malware []types.Malware `json:"malware" parquet:"name=malware"` + Network *types.Network `json:"network" parquet:"name=network"` + NetworkPath []types.NetworkPathComponent `json:"network_path" parquet:"name=network_path"` + Note *types.Note `json:"note" parquet:"name=note"` + PatchSummary *types.PatchSummary `json:"patch_summary" parquet:"name=patch_summary"` + Process *types.ProcessDetails `json:"process" parquet:"name=process"` + ProcessedAt *string `json:"processed_at" parquet:"name=processed_at"` + ProductArn *string `json:"product_arn" parquet:"name=product_arn"` + ProductFields map[string]string `json:"product_fields" parquet:"name=product_fields"` + ProductName *string `json:"product_name" parquet:"name=product_name"` + RecordState types.RecordState `json:"record_state" parquet:"name=record_state"` + RelatedFindings []types.RelatedFinding `json:"related_findings" parquet:"name=related_findings"` + Remediation *types.Remediation `json:"remediation" parquet:"name=remediation"` + Resources []types.Resource `json:"resources" parquet:"name=resources"` + Action *types.Action `json:"action" parquet:"name=action"` + Sample *bool `json:"sample" parquet:"name=sample"` + SchemaVersion *string `json:"schema_version" parquet:"name=schema_version"` + Severity *types.Severity `json:"severity" parquet:"name=severity"` + SourceUrl *string `json:"source_url" parquet:"name=source_url"` + ThreatIntelIndicators []types.ThreatIntelIndicator `json:"threat_intel_indicators" parquet:"name=threat_intel_indicators"` + Threats []types.Threat `json:"threats" parquet:"name=threats"` + Title *string `json:"title" parquet:"name=title"` + Types []string `json:"types" parquet:"name=types"` + UpdatedAt *string `json:"updated_at" parquet:"name=updated_at"` + UserDefinedFields map[string]string `json:"user_defined_fields" parquet:"name=user_defined_fields"` + VerificationState types.VerificationState `json:"verification_state" parquet:"name=verification_state"` + Vulnerabilities []types.Vulnerability `json:"vulnerabilities" parquet:"name=vulnerabilities"` + Workflow *types.Workflow `json:"workflow" parquet:"name=workflow"` + WorkflowState types.WorkflowState `json:"workflow_state" parquet:"name=workflow_state"` + + // Event fields EventCategory *string `json:"eventCategory" parquet:"name=event_category"` EventID *string `json:"eventID" parquet:"name=event_id"` EventName *string `json:"eventName" parquet:"name=event_name"` @@ -73,40 +82,36 @@ type DetailFindingsData struct { UserIdentity SecurityHubUserIdentity `json:"userIdentity" parquet:"name=user_identity, type=JSON"` } -type AssociatedStandard struct { - StandardsId *string `json:"StandardsId,omitempty"` -} - -type SecurityControlParameter struct { - Value []string `json:"Value,omitempty"` - Name *string `json:"Name,omitempty"` -} - -type Recommendation struct { - Text *string `json:"Text,omitempty"` - Url *string `json:"Url,omitempty"` -} - -type ResourceDetails struct { - AwsLambdaFunction *AwsLambdaFunction `json:"AwsLambdaFunction,omitempty"` -} - -type AwsLambdaFunction struct { - LastModified *time.Time `json:"LastModified,omitempty"` - Role *string `json:"Role,omitempty"` - FunctionName *string `json:"FunctionName,omitempty"` - MemorySize *int `json:"MemorySize,omitempty"` - Runtime *string `json:"Runtime,omitempty"` - TracingConfig *TracingConfig `json:"TracingConfig,omitempty"` - Version *string `json:"Version,omitempty"` - Timeout *int `json:"Timeout,omitempty"` - Handler *string `json:"Handler,omitempty"` - CodeSha256 *string `json:"CodeSha256,omitempty"` - RevisionId *string `json:"RevisionId,omitempty"` -} - -type TracingConfig struct { - Mode *string `json:"Mode,omitempty"` +// DetailFindingsData maps the `detail` field containing findings +// The following struct will be used for only parse the log lines +type DetailFindingsData struct { + Version *string `json:"version,omitempty"` + ID *string `json:"id,omitempty"` + DetailType *string `json:"detail_type,omitempty"` + Source *string `json:"source,omitempty"` + Account *string `json:"account,omitempty"` + Time *time.Time `json:"time,omitempty"` + Region *string `json:"region,omitempty"` + Detail struct { + Findings []types.AwsSecurityFinding `json:"findings" parquet:"name=findings, type=JSON"` + AwsRegion *string `json:"awsRegion" parquet:"name=aws_region"` + EventCategory *string `json:"eventCategory" parquet:"name=event_category"` + EventID *string `json:"eventID" parquet:"name=event_id"` + EventName *string `json:"eventName" parquet:"name=event_name"` + EventSource *string `json:"eventSource" parquet:"name=event_source"` + EventTime time.Time `json:"eventTime" parquet:"name=event_time"` + EventType *string `json:"eventType" parquet:"name=event_type"` + EventVersion *string `json:"eventVersion" parquet:"name=event_version"` + ManagementEvent bool `json:"managementEvent" parquet:"name=management_event"` + ReadOnly bool `json:"readOnly" parquet:"name=read_only"` + RecipientAccountID *string `json:"recipientAccountId" parquet:"name=recipient_account_id"` + RequestID *string `json:"requestID" parquet:"name=request_id"` + RequestParameters *map[string]interface{} `json:"requestParameters" parquet:"name=request_parameters, type=JSON"` + ResponseElements *map[string]interface{} `json:"responseElements" parquet:"name=response_elements, type=JSON"` + SourceIPAddress *string `json:"sourceIPAddress" parquet:"name=source_ip_address"` + UserAgent *string `json:"userAgent" parquet:"name=user_agent"` + UserIdentity SecurityHubUserIdentity `json:"userIdentity" parquet:"name=user_identity, type=JSON"` + } `json:"detail" parquet:"name=detail, type=JSON"` } type SecurityHubUserIdentity struct { @@ -146,33 +151,68 @@ func (c *SecurityHubFinding) GetColumnDescriptions() map[string]string { "account": "The AWS account ID where the finding was generated.", "time": "The timestamp when the event was generated.", "region": "The AWS region where the finding was generated.", - "resources": "The list of AWS resources associated with the finding.", - - // Detail fields - "detail": "The detailed information about the security finding.", // Finding fields - "product_arn": "The ARN of the AWS security product that generated the finding.", - "types": "The list of types assigned to the finding.", - "description": "A detailed description of the security finding.", - "compliance": "Information about the finding's compliance status.", - "product_name": "The name of the security product that generated the finding.", - "first_observed_at": "The timestamp when the finding was first observed.", - "created_at": "The timestamp when the finding was created.", - "last_observed_at": "The timestamp when the finding was last observed.", - "company_name": "The name of the company that provides the security product.", - "product_fields": "Additional fields provided by the security product.", - "remediation": "Recommended steps to remediate the finding.", - "schema_version": "The version of the finding format schema.", - "generator_id": "The identifier of the system that generated the finding.", - "record_state": "The current state of the finding record.", - "title": "A short human-readable title for the finding.", - "workflow": "Information about the finding's workflow status.", - "severity": "The severity level of the finding.", - "updated_at": "The timestamp when the finding was last updated.", - "workflow_state": "The current state of the finding in the workflow.", - "aws_account_id": "The AWS account ID associated with the finding.", - "processed_at": "The timestamp when the finding was processed.", + "aws_account_name": "The name of the AWS account from which a finding was generated.", + "company_name": "The name of the company for the product that generated the finding. Security Hub populates this attribute automatically for each finding.", + "compliance": "Contains security standard-related finding details for findings generated from compliance checks against specific rules in supported security standards.", + "confidence": "The likelihood that a finding accurately identifies the behavior or issue that it was intended to identify. Scored on a 0-100 basis.", + "created_at": "The timestamp when the security findings provider created the potential security issue that a finding captured.", + "criticality": "The level of importance assigned to the resources associated with the finding. A score of 0 means no criticality, and 100 is reserved for the most critical resources.", + "description": "A detailed description of the security finding.", + "first_observed_at": "The timestamp when the security findings provider first observed the potential security issue that a finding captured.", + "generator_id": "The identifier for the solution-specific component (a discrete unit of logic) that generated a finding.", + "generator_details": "Metadata for the Amazon CodeGuru detector associated with a finding, particularly for Lambda function-related findings.", + "finding_id": "The security findings provider-specific identifier for a finding.", + "finding_region": "The AWS region from which the finding was generated.", + "last_observed_at": "The timestamp when the security findings provider most recently observed the potential security issue that a finding captured.", + "malware": "A list of malware related to a finding.", + "network": "The details of network-related information about a finding.", + "network_path": "Information about a network path that is relevant to a finding, with each entry representing a component of that path.", + "note": "A user-defined note added to a finding.", + "patch_summary": "An overview of the patch compliance status for an instance against a selected compliance standard.", + "process": "The details of process-related information about a finding.", + "processed_at": "The timestamp when Security Hub received a finding and began to process it.", + "product_arn": "The ARN generated by Security Hub that uniquely identifies a product that generates findings.", + "product_fields": "Additional solution-specific details that aren't part of the defined AwsSecurityFinding format. Can contain up to 50 key-value pairs.", + "product_name": "The name of the product that generated the finding. Security Hub populates this attribute automatically for each finding.", + "record_state": "The record state of a finding.", + "related_findings": "A list of related findings.", + "remediation": "A data type that describes the remediation options for a finding.", + "resources": "A set of resource data types that describe the resources that the finding refers to.", + "action": "Details about an action that affects or that was taken on a resource.", + "sample": "Indicates whether the finding is a sample finding.", + "schema_version": "The schema version that a finding is formatted for.", + "severity": "The severity level of the finding.", + "source_url": "A URL that links to a page about the current finding in the security findings provider's solution.", + "threat_intel_indicators": "Threat intelligence details related to a finding.", + "threats": "Details about the threat detected in a security finding and the file paths that were affected by the threat.", + "title": "A short human-readable title for the finding.", + "types": "One or more finding types in the format of namespace/category/classifier that classify a finding.", + "updated_at": "The timestamp when the security findings provider last updated the finding record.", + "user_defined_fields": "A list of name/value string pairs associated with the finding. These are custom, user-defined fields added to a finding.", + "verification_state": "Indicates the veracity of a finding.", + "vulnerabilities": "A list of vulnerabilities associated with the findings.", + "workflow": "Information about the status of the investigation into a finding.", + "workflow_state": "The workflow state of a finding.", + + // Event fields + "event_category": "The category of the event.", + "event_id": "The unique identifier for the event.", + "event_name": "The name of the event.", + "event_source": "The source of the event.", + "event_time": "The timestamp when the event was generated.", + "event_type": "The type of the event.", + "event_version": "The version of the event.", + "management_event": "Indicates if the event is a management event.", + "read_only": "Indicates if the event is read-only.", + "recipient_account_id": "The AWS account ID of the recipient of the event.", + "request_id": "The unique identifier for the request.", + "request_parameters": "The parameters of the request.", + "response_elements": "The elements of the response.", + "source_ip_address": "The IP address of the source of the event.", + "user_agent": "The user agent of the user who made the request.", + "user_identity": "The identity of the user who made the request.", // Tailpipe-specific metadata fields "tp_akas": "The list of AWS ARNs associated with the finding.", From 322725d1cfefeffe31466c37c14870c07d8e3a50 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Fri, 18 Apr 2025 19:29:09 +0530 Subject: [PATCH 07/12] Update SecurityHubFindingMapper to handle pointer types and update SecurityHubFindingTable to include artifact extractor --- .../securityhub_finding/securityhub_finding_mapper.go | 4 ++++ .../securityhub_finding/securityhub_finding_table.go | 10 +++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tables/securityhub_finding/securityhub_finding_mapper.go b/tables/securityhub_finding/securityhub_finding_mapper.go index 0d0a6d4..4b761cb 100644 --- a/tables/securityhub_finding/securityhub_finding_mapper.go +++ b/tables/securityhub_finding/securityhub_finding_mapper.go @@ -29,6 +29,10 @@ func (m *SecurityHubFindingMapper) Map(_ context.Context, a any, _ ...mappers.Ma } case SecurityHubFinding: b = data + return &b, nil + case *SecurityHubFinding: + b = *data + return &b, nil default: return nil, fmt.Errorf("expected byte[], string or SecurityHubFinding, got %T", a) } diff --git a/tables/securityhub_finding/securityhub_finding_table.go b/tables/securityhub_finding/securityhub_finding_table.go index 222e208..478ee4a 100644 --- a/tables/securityhub_finding/securityhub_finding_table.go +++ b/tables/securityhub_finding/securityhub_finding_table.go @@ -35,11 +35,15 @@ func (c *SecurityHubFindingTable) GetSourceMetadata() ([]*table.SourceMetadata[* Mapper: &SecurityHubFindingMapper{}, Options: []row_source.RowSourceOption{ artifact_source.WithDefaultArtifactSourceConfig(defaultS3ArtifactConfig), + artifact_source.WithArtifactExtractor(NewSecurityHubFindingExtractor()), }, }, { SourceName: constants.ArtifactSourceIdentifier, Mapper: &SecurityHubFindingMapper{}, + Options: []row_source.RowSourceOption{ + artifact_source.WithArtifactExtractor(NewSecurityHubFindingExtractor()), + }, }, }, nil } @@ -50,9 +54,9 @@ func (c *SecurityHubFindingTable) EnrichRow(row *SecurityHubFinding, sourceEnric row.TpID = xid.New().String() row.TpIngestTimestamp = time.Now() - for _, resource := range row.Resources { - newAkas := tables.AwsAkasFromArn(*resource) - row.TpAkas = append(row.TpAkas, newAkas...) + if row.ProductArn != nil { + akas := tables.AwsAkasFromArn(*row.ProductArn) + row.TpAkas = append(row.TpAkas, akas...) } if row.Time != nil { From 8f70ea02085bf5d54e4381c34a381a3d10bf3dd1 Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Fri, 18 Apr 2025 19:29:31 +0530 Subject: [PATCH 08/12] Update extractor --- .../securityhub_finding_extractor.go | 126 +++++++++++++++--- 1 file changed, 111 insertions(+), 15 deletions(-) diff --git a/tables/securityhub_finding/securityhub_finding_extractor.go b/tables/securityhub_finding/securityhub_finding_extractor.go index 49b6e0d..491a111 100644 --- a/tables/securityhub_finding/securityhub_finding_extractor.go +++ b/tables/securityhub_finding/securityhub_finding_extractor.go @@ -9,39 +9,135 @@ import ( "github.com/turbot/tailpipe-plugin-sdk/artifact_source" ) -// SecurityHubFindingExtractor is an extractor that receives JSON serialised CloudTrailLogBatch objects +// SecurityHubFindingExtractor is an extractor that receives JSON serialised SecurityHub findings // and extracts SecurityHubFinding records from them type SecurityHubFindingExtractor struct { } -// NewCloudTrailLogExtractor creates a new SecurityHubFindingExtractor -func NewCloudTrailLogExtractor() artifact_source.Extractor { +// NewSecurityHubFindingExtractor creates a new SecurityHubFindingExtractor +func NewSecurityHubFindingExtractor() artifact_source.Extractor { return &SecurityHubFindingExtractor{} } func (c *SecurityHubFindingExtractor) Identifier() string { - return "cloudtrail_log_extractor" + return "securityhub_finding_extractor" } -// Extract unmarshalls the artifact data as an CloudTrailLogBatch and returns the SecurityHubFinding records +// Extract unmarshalls the artifact data as SecurityHub findings and returns the SecurityHubFinding records func (c *SecurityHubFindingExtractor) Extract(_ context.Context, a any) ([]any, error) { - // the expected input type is a JSON byte[] deserializable to CloudTrailLogBatch - jsonBytes, ok := a.([]byte) - if !ok { - return nil, fmt.Errorf("expected byte[], got %T", a) + // the expected input type is a JSON byte[] deserializable to DetailFindingsData + var jsonBytes []byte + + switch v := a.(type) { + case []byte: + jsonBytes = v + case string: + jsonBytes = []byte(v) + default: + return nil, fmt.Errorf("expected []byte or string, got %T", a) + } + + // First, we need to remap certain JSON fields due to naming conventions + // DetailFindingsData expects "detail-type" to be mapped to "detail_type" + var rawEvent map[string]json.RawMessage + if err := json.Unmarshal(jsonBytes, &rawEvent); err != nil { + return nil, fmt.Errorf("error decoding json: %w", err) + } + + // Handle kebab-case to snake_case for detail-type + if detailType, ok := rawEvent["detail-type"]; ok { + rawEvent["detail_type"] = detailType + delete(rawEvent, "detail-type") } - // decode json ito CloudTrailLogBatch - var log SecurityHubFinding - err := json.Unmarshal(jsonBytes, &log) + // Re-encode the modified JSON + modifiedJSON, err := json.Marshal(rawEvent) if err != nil { + return nil, fmt.Errorf("error re-encoding json: %w", err) + } + + // decode json into DetailFindingsData + var event DetailFindingsData + err = json.Unmarshal(modifiedJSON, &event) + if err != nil { + slog.Debug("Error decoding SecurityHub finding", "error", err, "sample_start", string(jsonBytes[:min(len(jsonBytes), 500)])) return nil, fmt.Errorf("error decoding json: %w", err) } - slog.Debug("SecurityHubFindingExtractor", "record count", len(log.Findings)) - var res = make([]any, len(log.Findings)) - for i, record := range log.Findings { + slog.Debug("SecurityHubFindingExtractor", "record count", len(event.Detail.Findings)) + + findings := toMapSecurityHubFinding(event) + var res = make([]any, len(findings)) + for i, record := range findings { res[i] = &record } return res, nil } + +func toMapSecurityHubFinding(event DetailFindingsData) []SecurityHubFinding { + var findings []SecurityHubFinding + + for _, finding := range event.Detail.Findings { + f := SecurityHubFinding{} + + // Event metadata + f.Version = event.Version + f.ID = event.ID + f.DetailType = event.DetailType + f.Source = event.Source + f.Account = event.Account + f.Time = event.Time + f.Region = event.Region + + // Finding details from AWS security finding + if finding.CreatedAt != nil { + // CreatedAt is a string in SecurityHubFinding + createdAtStr := *finding.CreatedAt + f.CreatedAt = &createdAtStr + } + if finding.Description != nil { + f.Description = finding.Description + } + if finding.GeneratorId != nil { + f.GeneratorId = finding.GeneratorId + } + if finding.Id != nil { + f.FindingId = finding.Id + } + if finding.ProductArn != nil { + f.ProductArn = finding.ProductArn + } + if finding.ProductName != nil { + f.ProductName = finding.ProductName + } + if finding.Title != nil { + f.Title = finding.Title + } + if finding.SchemaVersion != nil { + f.SchemaVersion = finding.SchemaVersion + } + + // Map ProductFields + if finding.ProductFields != nil { + productFields := make(map[string]string) + for k, v := range finding.ProductFields { + productFields[k] = v + } + f.ProductFields = productFields + } + + // Map Resources + if len(finding.Resources) > 0 { + f.Resources = finding.Resources + } + + // Map Remediation + if finding.Remediation != nil && finding.Remediation.Recommendation != nil { + f.Remediation = finding.Remediation + } + + findings = append(findings, f) + } + + return findings +} From ce8987a98824e0e115ad788feac8bef65074734d Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Fri, 18 Apr 2025 22:14:19 +0530 Subject: [PATCH 09/12] remove event fields --- .../securityhub_finding.go | 64 +------------------ 1 file changed, 1 insertion(+), 63 deletions(-) diff --git a/tables/securityhub_finding/securityhub_finding.go b/tables/securityhub_finding/securityhub_finding.go index fbace03..7fe2bde 100644 --- a/tables/securityhub_finding/securityhub_finding.go +++ b/tables/securityhub_finding/securityhub_finding.go @@ -62,24 +62,6 @@ type SecurityHubFinding struct { Vulnerabilities []types.Vulnerability `json:"vulnerabilities" parquet:"name=vulnerabilities"` Workflow *types.Workflow `json:"workflow" parquet:"name=workflow"` WorkflowState types.WorkflowState `json:"workflow_state" parquet:"name=workflow_state"` - - // Event fields - EventCategory *string `json:"eventCategory" parquet:"name=event_category"` - EventID *string `json:"eventID" parquet:"name=event_id"` - EventName *string `json:"eventName" parquet:"name=event_name"` - EventSource *string `json:"eventSource" parquet:"name=event_source"` - EventTime time.Time `json:"eventTime" parquet:"name=event_time"` - EventType *string `json:"eventType" parquet:"name=event_type"` - EventVersion *string `json:"eventVersion" parquet:"name=event_version"` - ManagementEvent bool `json:"managementEvent" parquet:"name=management_event"` - ReadOnly bool `json:"readOnly" parquet:"name=read_only"` - RecipientAccountID *string `json:"recipientAccountId" parquet:"name=recipient_account_id"` - RequestID *string `json:"requestID" parquet:"name=request_id"` - RequestParameters *map[string]interface{} `json:"requestParameters" parquet:"name=request_parameters, type=JSON"` - ResponseElements *map[string]interface{} `json:"responseElements" parquet:"name=response_elements, type=JSON"` - SourceIPAddress *string `json:"sourceIPAddress" parquet:"name=source_ip_address"` - UserAgent *string `json:"userAgent" parquet:"name=user_agent"` - UserIdentity SecurityHubUserIdentity `json:"userIdentity" parquet:"name=user_identity, type=JSON"` } // DetailFindingsData maps the `detail` field containing findings @@ -93,54 +75,10 @@ type DetailFindingsData struct { Time *time.Time `json:"time,omitempty"` Region *string `json:"region,omitempty"` Detail struct { - Findings []types.AwsSecurityFinding `json:"findings" parquet:"name=findings, type=JSON"` - AwsRegion *string `json:"awsRegion" parquet:"name=aws_region"` - EventCategory *string `json:"eventCategory" parquet:"name=event_category"` - EventID *string `json:"eventID" parquet:"name=event_id"` - EventName *string `json:"eventName" parquet:"name=event_name"` - EventSource *string `json:"eventSource" parquet:"name=event_source"` - EventTime time.Time `json:"eventTime" parquet:"name=event_time"` - EventType *string `json:"eventType" parquet:"name=event_type"` - EventVersion *string `json:"eventVersion" parquet:"name=event_version"` - ManagementEvent bool `json:"managementEvent" parquet:"name=management_event"` - ReadOnly bool `json:"readOnly" parquet:"name=read_only"` - RecipientAccountID *string `json:"recipientAccountId" parquet:"name=recipient_account_id"` - RequestID *string `json:"requestID" parquet:"name=request_id"` - RequestParameters *map[string]interface{} `json:"requestParameters" parquet:"name=request_parameters, type=JSON"` - ResponseElements *map[string]interface{} `json:"responseElements" parquet:"name=response_elements, type=JSON"` - SourceIPAddress *string `json:"sourceIPAddress" parquet:"name=source_ip_address"` - UserAgent *string `json:"userAgent" parquet:"name=user_agent"` - UserIdentity SecurityHubUserIdentity `json:"userIdentity" parquet:"name=user_identity, type=JSON"` + Findings []types.AwsSecurityFinding `json:"findings" parquet:"name=findings, type=JSON"` } `json:"detail" parquet:"name=detail, type=JSON"` } -type SecurityHubUserIdentity struct { - AccessKeyID *string `json:"accessKeyId"` - AccountID *string `json:"accountId"` - Arn *string `json:"arn"` - PrincipalID *string `json:"principalId"` - SessionContext SecurityHubSessionContext `json:"sessionContext"` - Type *string `json:"type"` -} - -type SecurityHubSessionContext struct { - Attributes Attributes `json:"attributes"` - SessionIssuer SecurityHubSessionIssuer `json:"sessionIssuer"` -} - -type Attributes struct { - CreationDate time.Time `json:"creationDate"` - MfaAuthenticated *string `json:"mfaAuthenticated"` -} - -type SecurityHubSessionIssuer struct { - AccountID string `json:"accountId"` - Arn string `json:"arn"` - PrincipalID string `json:"principalId"` - Type string `json:"type"` - UserName string `json:"userName"` -} - func (c *SecurityHubFinding) GetColumnDescriptions() map[string]string { return map[string]string{ // Top level fields From 2a12fd67ad5a579426c92df059ec99110cdab8fb Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Fri, 18 Apr 2025 23:06:35 +0530 Subject: [PATCH 10/12] Update SecurityHubFindingExtractor and securityhub_finding --- .../securityhub_finding.go | 18 ---- .../securityhub_finding_extractor.go | 102 +++++++++++++++++- 2 files changed, 99 insertions(+), 21 deletions(-) diff --git a/tables/securityhub_finding/securityhub_finding.go b/tables/securityhub_finding/securityhub_finding.go index 7fe2bde..dc649a4 100644 --- a/tables/securityhub_finding/securityhub_finding.go +++ b/tables/securityhub_finding/securityhub_finding.go @@ -134,24 +134,6 @@ func (c *SecurityHubFinding) GetColumnDescriptions() map[string]string { "workflow": "Information about the status of the investigation into a finding.", "workflow_state": "The workflow state of a finding.", - // Event fields - "event_category": "The category of the event.", - "event_id": "The unique identifier for the event.", - "event_name": "The name of the event.", - "event_source": "The source of the event.", - "event_time": "The timestamp when the event was generated.", - "event_type": "The type of the event.", - "event_version": "The version of the event.", - "management_event": "Indicates if the event is a management event.", - "read_only": "Indicates if the event is read-only.", - "recipient_account_id": "The AWS account ID of the recipient of the event.", - "request_id": "The unique identifier for the request.", - "request_parameters": "The parameters of the request.", - "response_elements": "The elements of the response.", - "source_ip_address": "The IP address of the source of the event.", - "user_agent": "The user agent of the user who made the request.", - "user_identity": "The identity of the user who made the request.", - // Tailpipe-specific metadata fields "tp_akas": "The list of AWS ARNs associated with the finding.", "tp_index": "The AWS account ID where the finding was generated.", diff --git a/tables/securityhub_finding/securityhub_finding_extractor.go b/tables/securityhub_finding/securityhub_finding_extractor.go index 491a111..a13de40 100644 --- a/tables/securityhub_finding/securityhub_finding_extractor.go +++ b/tables/securityhub_finding/securityhub_finding_extractor.go @@ -90,32 +90,128 @@ func toMapSecurityHubFinding(event DetailFindingsData) []SecurityHubFinding { f.Region = event.Region // Finding details from AWS security finding + if finding.AwsAccountName != nil { + f.AwsAccountName = finding.AwsAccountName + } + if finding.CompanyName != nil { + f.CompanyName = finding.CompanyName + } + if finding.Compliance != nil { + f.Compliance = finding.Compliance + } + if finding.Confidence != nil { + f.Confidence = finding.Confidence + } if finding.CreatedAt != nil { - // CreatedAt is a string in SecurityHubFinding createdAtStr := *finding.CreatedAt f.CreatedAt = &createdAtStr } + if finding.Criticality != nil { + f.Criticality = finding.Criticality + } if finding.Description != nil { f.Description = finding.Description } + if finding.FirstObservedAt != nil { + f.FirstObservedAt = finding.FirstObservedAt + } if finding.GeneratorId != nil { f.GeneratorId = finding.GeneratorId } + if finding.GeneratorDetails != nil { + f.GeneratorDetails = finding.GeneratorDetails + } if finding.Id != nil { f.FindingId = finding.Id } + if finding.Region != nil { + f.FindingRegion = finding.Region + } + if finding.LastObservedAt != nil { + f.LastObservedAt = finding.LastObservedAt + } + if finding.Malware != nil { + f.Malware = finding.Malware + } + if finding.Network != nil { + f.Network = finding.Network + } + if finding.NetworkPath != nil { + f.NetworkPath = finding.NetworkPath + } + if finding.Note != nil { + f.Note = finding.Note + } + if finding.PatchSummary != nil { + f.PatchSummary = finding.PatchSummary + } + if finding.Process != nil { + f.Process = finding.Process + } + if finding.ProcessedAt != nil { + f.ProcessedAt = finding.ProcessedAt + } if finding.ProductArn != nil { f.ProductArn = finding.ProductArn } if finding.ProductName != nil { f.ProductName = finding.ProductName } - if finding.Title != nil { - f.Title = finding.Title + if finding.RecordState != "" { + f.RecordState = finding.RecordState + } + if finding.RelatedFindings != nil { + f.RelatedFindings = finding.RelatedFindings + } + if finding.Action != nil { + f.Action = finding.Action + } + if finding.Sample != nil { + f.Sample = finding.Sample } if finding.SchemaVersion != nil { f.SchemaVersion = finding.SchemaVersion } + if finding.Severity != nil { + f.Severity = finding.Severity + } + if finding.SourceUrl != nil { + f.SourceUrl = finding.SourceUrl + } + if finding.ThreatIntelIndicators != nil { + f.ThreatIntelIndicators = finding.ThreatIntelIndicators + } + if finding.Threats != nil { + f.Threats = finding.Threats + } + if finding.Title != nil { + f.Title = finding.Title + } + if finding.Types != nil { + f.Types = finding.Types + } + if finding.UpdatedAt != nil { + f.UpdatedAt = finding.UpdatedAt + } + if finding.UserDefinedFields != nil { + userDefinedFields := make(map[string]string) + for k, v := range finding.UserDefinedFields { + userDefinedFields[k] = v + } + f.UserDefinedFields = userDefinedFields + } + if finding.VerificationState != "" { + f.VerificationState = finding.VerificationState + } + if finding.Vulnerabilities != nil { + f.Vulnerabilities = finding.Vulnerabilities + } + if finding.Workflow != nil { + f.Workflow = finding.Workflow + } + if finding.WorkflowState != "" { + f.WorkflowState = finding.WorkflowState + } // Map ProductFields if finding.ProductFields != nil { From ee7b785d51ad80a5dd29f0bbc3f182e3eab1490a Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Mon, 21 Apr 2025 13:44:25 +0530 Subject: [PATCH 11/12] Add docs for aws_securityhub_finding --- docs/tables/aws_securityhub_finding/index.md | 224 ++++++++++++++++++ .../tables/aws_securityhub_finding/queries.md | 217 +++++++++++++++++ 2 files changed, 441 insertions(+) create mode 100644 docs/tables/aws_securityhub_finding/index.md create mode 100644 docs/tables/aws_securityhub_finding/queries.md diff --git a/docs/tables/aws_securityhub_finding/index.md b/docs/tables/aws_securityhub_finding/index.md new file mode 100644 index 0000000..4669818 --- /dev/null +++ b/docs/tables/aws_securityhub_finding/index.md @@ -0,0 +1,224 @@ +--- +title: "Tailpipe Table: aws_securityhub_finding - Query AWS Security Hub Findings" +description: "AWS Security Hub findings provide comprehensive security findings from various AWS security services and partner integrations, including details about potential security issues and compliance violations." +--- + +# Table: aws_securityhub_finding - Query AWS Security Hub Findings + +The `aws_securityhub_finding` table allows you to query data from [AWS Security Hub findings](https://docs.aws.amazon.com/securityhub/latest/userguide/securityhub-findings.html). This table provides detailed information about potential security issues and compliance violations detected across your AWS accounts and resources, including severity levels, compliance status, affected resources, and recommended remediation steps. + +## Configure + +Create a [partition](https://tailpipe.io/docs/manage/partition) for `aws_securityhub_finding` ([examples](https://hub.tailpipe.io/plugins/turbot/aws/tables/aws_securityhub_finding#example-configurations)): + +```sh +vi ~/.tailpipe/config/aws.tpc +``` + +```hcl +connection "aws" "security_account" { + profile = "my-security-account" +} + +partition "aws_securityhub_finding" "my_findings" { + source "aws_s3_bucket" { + connection = connection.aws.security_account + bucket = "aws-securityhub-findings-bucket" + } +} +``` + +## Collect + +[Collect](https://tailpipe.io/docs/manage/collection) findings for all `aws_securityhub_finding` partitions: + +```sh +tailpipe collect aws_securityhub_finding +``` + +Or for a single partition: + +```sh +tailpipe collect aws_securityhub_finding.my_findings +``` + +## Query + +**[Explore example queries for this table →](https://hub.tailpipe.io/plugins/turbot/aws/queries/aws_securityhub_finding)** + +### High Severity Findings + +List all high severity security findings with detailed resource information. + +```sql +select + tp_timestamp, + title, + types, + severity, + description, + tp_index as account_id, + region, + resources, + remediation.recommendation.text as remediation_text +from + aws_securityhub_finding +where + severity.normalized >= 70 +order by + severity.normalized desc, + tp_timestamp desc; +``` + +### Findings by Type + +Group findings by type with severity and temporal information. + +```sql +select + types, + count(*) as finding_count, + round(avg(severity.normalized), 2) as avg_severity +from + aws_securityhub_finding +group by + types +order by + finding_count desc; +``` + +### Recent Findings with Resource Details + +Examine recent security findings with comprehensive resource and remediation information. + +```sql +select + tp_timestamp, + title, + types, + severity, + resources, + tp_index as account_id, + region, + workflow_state, + remediation.recommendation.text as remediation_text +from + aws_securityhub_finding +where + created_at > current_date - interval '7 days' +order by + tp_timestamp desc; +``` + +## Example Configurations + +### Collect findings from an S3 bucket + +Collect Security Hub findings stored in an S3 bucket using the default log file format. + +```hcl +connection "aws" "security_account" { + profile = "my-security-account" +} + +partition "aws_securityhub_finding" "my_findings" { + source "aws_s3_bucket" { + connection = connection.aws.security_account + bucket = "aws-securityhub-findings-bucket" + } +} +``` + +### Collect findings from an S3 bucket with a prefix + +Collect Security Hub findings stored in an S3 bucket using a prefix. + +```hcl +partition "aws_securityhub_finding" "my_findings_prefix" { + source "aws_s3_bucket" { + connection = connection.aws.security_account + bucket = "aws-securityhub-findings-bucket" + prefix = "my/prefix/" + } +} +``` + +### Collect findings from local files + +You can also collect Security Hub findings from local files. + +```hcl +partition "aws_securityhub_finding" "local_findings" { + source "file" { + paths = ["/Users/myuser/securityhub_findings"] + file_layout = `%{DATA}.jsonl.gz` + } +} +``` + +### Filter high severity findings only + +Use the filter argument in your partition to focus on high severity findings, reducing the size of local storage. + +```hcl +partition "aws_securityhub_finding" "high_severity_findings" { + filter = "severity.normalized >= 70" + + source "aws_s3_bucket" { + connection = connection.aws.security_account + bucket = "aws-securityhub-findings-bucket" + } +} +``` + +### Collect findings for all accounts in an organization + +For a specific organization, collect findings for all accounts and regions. + +```hcl +partition "aws_securityhub_finding" "my_findings_org" { + source "aws_s3_bucket" { + connection = connection.aws.security_account + bucket = "securityhub-findings-bucket" + file_layout = `AWSLogs/o-aa111bb222/%{NUMBER:account_id}/SecurityHub/%{DATA:region}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{DATA}.jsonl.gz` + } +} +``` + +### Collect findings for a single account + +For a specific account, collect findings for all regions. + +```hcl +partition "aws_securityhub_finding" "my_findings_account" { + source "aws_s3_bucket" { + connection = connection.aws.security_account + bucket = "securityhub-findings-bucket" + file_layout = `AWSLogs/(%{DATA:org_id}/)?123456789012/SecurityHub/%{DATA:region}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{DATA}.jsonl.gz` + } +} +``` + +### Collect findings for a single region + +For all accounts, collect findings from us-east-1. + +```hcl +partition "aws_securityhub_finding" "my_findings_region" { + source "aws_s3_bucket" { + connection = connection.aws.security_account + bucket = "securityhub-findings-bucket" + file_layout = `AWSLogs/(%{DATA:org_id}/)?%{NUMBER:account_id}/SecurityHub/us-east-1/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{DATA}.jsonl.gz` + } +} +``` + +## Source Defaults + +### aws_s3_bucket + +This table sets the following defaults for the [aws_s3_bucket source](https://hub.tailpipe.io/plugins/turbot/aws/sources/aws_s3_bucket#arguments): + +| Argument | Default | +|--------------|---------| +| file_layout | `AWSLogs/(%{DATA:org_id}/)?%{NUMBER:account_id}/SecurityHub/%{DATA:region_path}/%{YEAR:year}/%{MONTHNUM:month}/%{MONTHDAY:day}/%{DATA}.jsonl.gz` | \ No newline at end of file diff --git a/docs/tables/aws_securityhub_finding/queries.md b/docs/tables/aws_securityhub_finding/queries.md new file mode 100644 index 0000000..96d0b11 --- /dev/null +++ b/docs/tables/aws_securityhub_finding/queries.md @@ -0,0 +1,217 @@ +## Activity Examples + +### Daily Activity Trends + +Analyze the daily distribution of Security Hub findings to identify security patterns and potential security issues over time. + +```sql +select + strftime(tp_timestamp, '%Y-%m-%d') as finding_date, + count(*) as finding_count, + round(avg(severity.normalized), 2) as avg_severity +from + aws_securityhub_finding +group by + finding_date +order by + finding_date asc; +``` + +```yaml +folder: SecurityHub +``` + +### Recent Findings Analysis + +Analyze recent security findings with detailed resource and severity information. + +```sql +select + tp_timestamp, + title, + types, + severity, + resources, + tp_index as account_id, + region, + workflow_state, + remediation.recommendation.text as remediation_text +from + aws_securityhub_finding +where + tp_timestamp > current_date - interval '7 days' +order by + severity.normalized desc, + tp_timestamp desc; +``` + +```yaml +folder: SecurityHub +``` + +### Top 10 Finding Types + +Generate a ranked list of the most prevalent Security Hub finding types with severity information. + +```sql +select + types, + count(*) as finding_count, + round(avg(severity.normalized), 2) as avg_severity +from + aws_securityhub_finding +group by + types +order by + finding_count desc +limit 10; +``` + +```yaml +folder: SecurityHub +``` + + + +### Findings by Account and Region + +Analyze security findings across your AWS organization with detailed severity information. + +```sql +select + tp_index as account_id, + region, + count(*) as finding_count, + round(avg(severity.normalized), 2) as avg_severity, + sum(case when severity.normalized >= 90 then 1 else 0 end) as critical_severity_count, + sum(case when severity.normalized >= 70 and severity.normalized < 90 then 1 else 0 end) as high_severity_count, + sum(case when severity.normalized >= 40 and severity.normalized < 70 then 1 else 0 end) as medium_severity_count, + sum(case when severity.normalized >= 1 and severity.normalized < 40 then 1 else 0 end) as low_severity_count, + sum(case when severity.normalized = 0 then 1 else 0 end) as informational_severity_count +from + aws_securityhub_finding +group by + account_id, + region +order by + critical_severity_count desc; +``` + +```yaml +folder: SecurityHub +``` + +### Findings by Severity Level + +Categorize Security Hub findings into severity bands with detailed counts and percentages. + +```sql +select + case + when severity.normalized >= 90 then 'Critical (90-100)' + when severity.normalized >= 70 then 'High (70-89)' + when severity.normalized >= 40 then 'Medium (40-69)' + when severity.normalized >= 1 then 'Low (1-39)' + else 'Informational (0)' + end as severity_level, + count(*) as finding_count, + round(count(*) * 100.0 / sum(count(*)) over(), 2) as percentage +from + aws_securityhub_finding +group by + severity_level +order by + case severity_level + when 'Critical (90-100)' then 1 + when 'High (70-89)' then 2 + when 'Medium (40-69)' then 3 + when 'Low (1-39)' then 4 + else 5 + end; +``` + +```yaml +folder: SecurityHub +``` + +## Compliance Examples + +### Compliance Status Overview + +Monitor compliance status with detailed severity information. + +```sql +select + compliance.status, + compliance.security_control_id, + count(*) as finding_count, + round(avg(severity.normalized), 2) as avg_severity +from + aws_securityhub_finding +where + compliance is not null +group by + compliance.status, + compliance.security_control_id +order by + finding_count desc; +``` + +```yaml +folder: SecurityHub +``` + +## Detection Examples + +### Detect High Severity Findings with Remediation + + ```sql +select + tp_timestamp, + title, + types, + severity, + description, + tp_index as account_id, + region, + resources, + remediation.recommendation.text as remediation_text +from + aws_securityhub_finding +where + severity.normalized >= 70 +order by + severity.normalized desc, + tp_timestamp desc; +``` + +```yaml +folder: SecurityHub +``` + +### Lambda Function Security Issues + +Identify security issues in Lambda functions, focusing on public access. + +```sql +select + tp_timestamp, + title, + severity.normalized as severity, + json_extract(resources, '$[0].id') as function_arn, + json_extract(resources, '$[0].details.awslambdafunction.runtime') as runtime, + workflow_state +from + aws_securityhub_finding +where + json_extract(resources, '$[0].type') = '"AwsLambdaFunction"' + and title ilike '%public access%' + and severity.normalized >= 70 +order by + severity desc, + tp_timestamp desc; +``` + +```yaml +folder: SecurityHub +``` From f1411d364c69559b31464c4dd22cef8b2644d48c Mon Sep 17 00:00:00 2001 From: Priyanka Chatterjee Date: Mon, 21 Apr 2025 19:55:03 +0530 Subject: [PATCH 12/12] Update tables/securityhub_finding/securityhub_finding.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- tables/securityhub_finding/securityhub_finding.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tables/securityhub_finding/securityhub_finding.go b/tables/securityhub_finding/securityhub_finding.go index dc649a4..680435a 100644 --- a/tables/securityhub_finding/securityhub_finding.go +++ b/tables/securityhub_finding/securityhub_finding.go @@ -25,7 +25,7 @@ type SecurityHubFinding struct { Compliance *types.Compliance `json:"compliance" parquet:"name=compliance"` Confidence *int32 `json:"confidence" parquet:"name=confidence"` CreatedAt *string `json:"createdAt" parquet:"name=created_at"` - Criticality *int32 `json:"criticality parquet:"name=criticality"` + Criticality *int32 `json:"criticality" parquet:"name=criticality"` Description *string `json:"description" parquet:"name=description"` FirstObservedAt *string `json:"first_observed_at" parquet:"name=first_observed_at"` GeneratorId *string `json:"generatorId" parquet:"name=generator_id"`