diff --git a/config.go b/config.go index f323a7d4f..4268850b7 100644 --- a/config.go +++ b/config.go @@ -437,7 +437,10 @@ func getConfig() *types.Configuration { Alertmanager: types.AlertmanagerOutputConfig{ExtraLabels: make(map[string]string), ExtraAnnotations: make(map[string]string), CustomSeverityMap: make(map[types.PriorityType]string), CustomHeaders: make(map[string]string)}, CloudEvents: types.CloudEventsOutputConfig{Extensions: make(map[string]string)}, GCP: types.GcpOutputConfig{PubSub: types.GcpPubSub{CustomAttributes: make(map[string]string)}}, - OTLP: types.OTLPOutputConfig{Traces: types.OTLPTraces{ExtraEnvVars: make(map[string]string)}}, + OTLP: types.OTLPOutputConfig{ + Traces: types.OTLPTraces{ExtraEnvVars: make(map[string]string)}, + Metrics: types.OTLPMetrics{ExtraEnvVars: make(map[string]string)}, + }, } configFile := kingpin.Flag("config-file", "config file").Short('c').ExistingFile() @@ -557,7 +560,7 @@ func getConfig() *types.Configuration { v.SetDefault("OTLP.Traces.Endpoint", "") v.SetDefault("OTLP.Traces.Protocol", "http/json") // NOTE: we don't need to parse the OTLP.Traces.Headers field, as use it to - // set OTEL_EXPORTER_OTLP_TRACES_HEADERS (at otlp_init.go), which is then + // set OTEL_EXPORTER_OTLP_TRACES_HEADERS (at otlp_traces_init.go), which is then // parsed by the OTLP SDK libs, see // https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/#otel_exporter_otlp_traces_headers v.SetDefault("OTLP.Traces.Headers", "") @@ -569,6 +572,15 @@ func getConfig() *types.Configuration { // it to 1000ms by default, override-able via OTLP_DURATION environment variable. v.SetDefault("OTLP.Traces.Duration", 1000) + v.SetDefault("OTLP.Metrics.Endpoint", "") + v.SetDefault("OTLP.Metrics.Protocol", "grpc") + // NOTE: we don't need to parse the OTLP.Metrics.Headers field, as use it to set OTEL_EXPORTER_OTLP_METRICS_HEADERS + // (at otlp_metrics.go), which is then parsed by the OTLP SDK libs. + v.SetDefault("OTLP.Metrics.Headers", "") + v.SetDefault("OTLP.Metrics.Timeout", 10000) + v.SetDefault("OTLP.Metrics.MinimumPriority", "") + v.SetDefault("OTLP.Metrics.CheckCert", true) + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) v.AutomaticEnv() if *configFile != "" { @@ -596,6 +608,7 @@ func getConfig() *types.Configuration { v.GetStringMapString("AlertManager.CustomSeverityMap") v.GetStringMapString("GCP.PubSub.CustomAttributes") v.GetStringMapString("OTLP.Traces.ExtraEnvVars") + v.GetStringMapString("OTLP.Metrics.ExtraEnvVars") c.Elasticsearch.CustomHeaders = v.GetStringMapString("Elasticsearch.CustomHeaders") @@ -750,6 +763,21 @@ func getConfig() *types.Configuration { } } + if value, present := os.LookupEnv("OTLP_METRICS_EXTRAENVVARS"); present { + extraEnvVars := strings.Split(value, ",") + for _, extraEnvVarData := range extraEnvVars { + envName, envValue, found := strings.Cut(extraEnvVarData, ":") + envName, envValue = strings.TrimSpace(envName), strings.TrimSpace(envValue) + if !promKVNameRegex.MatchString(envName) { + log.Printf("[ERROR] : OTLPMetrics - Extra Env Var name '%v' is not valid", envName) + } else if found { + c.OTLP.Metrics.ExtraEnvVars[envName] = envValue + } else { + c.OTLP.Metrics.ExtraEnvVars[envName] = "" + } + } + } + if c.AWS.SecurityLake.Interval < 5 { c.AWS.SecurityLake.Interval = 5 } @@ -881,6 +909,7 @@ func getConfig() *types.Configuration { c.OpenObserve.MinimumPriority = checkPriority(c.OpenObserve.MinimumPriority) c.Dynatrace.MinimumPriority = checkPriority(c.Dynatrace.MinimumPriority) c.SumoLogic.MinimumPriority = checkPriority(c.SumoLogic.MinimumPriority) + c.OTLP.Metrics.MinimumPriority = checkPriority(c.OTLP.Metrics.MinimumPriority) c.Talon.MinimumPriority = checkPriority(c.Talon.MinimumPriority) c.Slack.MessageFormatTemplate = getMessageFormatTemplate("Slack", c.Slack.MessageFormat) diff --git a/config_example.yaml b/config_example.yaml index dd2514a7f..0bcc52614 100644 --- a/config_example.yaml +++ b/config_example.yaml @@ -543,6 +543,17 @@ otlp: # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) # checkcert: true # Set if you want to skip TLS certificate validation (default: true) + metrics: + # endpoint: "" # OTLP endpoint, typically in the form http{s}://{domain or ip}:4318/v1/metrics + # protocol: "" # OTLP transport protocol to be used for metrics data; it can be "grpc" or "http/protobuf" (default: "grpc") + # timeout: "" # OTLP timeout for outgoing metrics in milliseconds (default: "" which uses SDK default: 10000) + # headers: "" # List of headers to apply to all outgoing metrics in the form of "some-key=some-value,other-key=other-value" (default: "") + # extraenvvars: # Extra env vars (override the other settings) (default: "") + # OTEL_EXPORTER_OTLP_METRICS_TIMEOUT: 10000 + # OTEL_EXPORTER_OTLP_TIMEOUT: 10000 + # minimumpriority: "" # Minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default: "") + # checkcert: true # Set to false if you want to skip TLS certificate validation (only with https) (default: true) + talon: # address: "" # Falco talon address, if not empty, Falco Talon output is enabled # checkcert: false # check if ssl certificate of the output is valid (default: true) diff --git a/docs/outputs/images/otlp_metrics-prom_view.png b/docs/outputs/images/otlp_metrics-prom_view.png new file mode 100644 index 000000000..5778ed2a5 Binary files /dev/null and b/docs/outputs/images/otlp_metrics-prom_view.png differ diff --git a/docs/outputs/otlp_metrics.md b/docs/outputs/otlp_metrics.md new file mode 100644 index 000000000..83b5a20e0 --- /dev/null +++ b/docs/outputs/otlp_metrics.md @@ -0,0 +1,195 @@ +# OTEL Metrics + +- **Category**: Metrics/Observability +- **Website**: + +## Table of content + +- [OTEL Metrics](#otel-metrics) + - [Table of content](#table-of-content) + - [Configuration](#configuration) + - [Example of config.yaml](#example-of-configyaml) + - [Additional info](#additional-info) + - [Running a whole stack with docker-compose](#running-a-whole-stack-with-docker-compose) + +## Configuration + +| Setting | Env var | Default value | Description | +|--------------------------------|--------------------------------|--------------------|-------------------------------------------------------------------------------------------------------------------------------------| +| `otlp.metrics.endpoint` | `OTLP_METRICS_ENDPOINT` | | OTLP endpoint, typically in the form http{s}://{domain or ip}:4318/v1/metrics | +| `otlp.metrics.protocol` | `OTLP_METRICS_PROTOCOL` | `grpc` | OTLP transport protocol to be used for metrics data; it can be `"grpc"` or `"http/protobuf"` | +| `otlp.metrics.timeout` | `OTLP_METRICS_TIMEOUT` | `10000` (from SDK) | OTLP timeout for outgoing metrics in milliseconds | +| `otlp.metrics.headers` | `OTLP_METRICS_HEADERS` | `""` | List of headers to apply to all outgoing metrics in the form of `some-key=some-value,other-key=other-value` | +| `otlp.metrics.extraenvvars` | `OTLP_METRICS_EXTRAENVVARS` | `""` | Extra env vars (override the other settings) | +| `otlp.metrics.minimumpriority` | `OTLP_METRICS_MINIMUMPRIORITY` | `""` (=`debug`) | Minimum priority of event for using this output, order is `emergency,alert,critical,error,warning,notice,informational,debug or ""` | +| `otlp.metrics.checkcert` | `OTLP_METRICS_CHECKCERT` | `true` | Set to false if you want to skip TLS certificate validation (only with https) | + +> [!NOTE] +For the extra Env Vars values see [standard `OTEL_*` environment variables](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/): + +## Example of config.yaml + +```yaml +otlp: + metrics: + # endpoint: "" # OTLP endpoint, typically in the form http{s}://{domain or ip}:4318/v1/metrics + # protocol: "" # OTLP transport protocol to be used for metrics data; it can be "grpc" or "http/protobuf" (default: "grpc") + # timeout: "" # OTLP timeout for outgoing metrics in milliseconds (default: "" which uses SDK default: 10000) + # headers: "" # List of headers to apply to all outgoing metrics in the form of "some-key=some-value,other-key=other-value" (default: "") + # extraenvvars: # Extra env vars (override the other settings) (default: "") + # OTEL_EXPORTER_OTLP_METRICS_TIMEOUT: 10000 + # OTEL_EXPORTER_OTLP_TIMEOUT: 10000 + # minimumpriority: "" # Minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default: "") + # checkcert: true # Set to false if you want to skip TLS certificate validation (only with https) (default: true) +``` + +## Additional info + +> [!NOTE] +The OTLP Metrics are only available for the source: `syscalls`. + +## Running a whole stack with docker-compose + +Below `docker-compose` file runs a stack of: + +- `falco` +- `falcosidekick` +- `prometheus` as metrics backend +- OTEL collector to collect OTEL metrics from `falcosidekick` and let prometheus scrape them +- `events-generator` to generate arbitrary Falco events + +### Requirements + +A local Linux kernel capable of running `falco`--modern-bpf`, see . + +### Configuration files + +You need to create these files: + +- `./docker-compose.yaml`: minimal docker-compose configuration + +```yaml +--- +services: + falco: + image: falcosecurity/falco:0.39.0 + privileged: true + volumes: + - /var/run/docker.sock:/host/var/run/docker.sock + - /dev:/host/dev + - /proc:/host/proc:ro + - /boot:/host/boot:ro + - /lib/modules:/host/lib/modules:ro + - /usr:/host/usr:ro + - /etc/falco:/host/etc:ro + command: [ + "/usr/bin/falco" , + "-o", "json_output=true", + "-o", "http_output.enabled=true", + "-o", "http_output.url=http://sidekick:2801", # Set the HTTP output url to Falco sidekick endpoint + "-o", "http_output.insecure=true" + ] + + sidekick: + image: falcosidekick:latest + ports: + - "2801:2801" # Expose default port towards Falco instance + environment: + - OTLP_METRICS_ENDPOINT=http://otel-collector:4317 + - OTLP_METRICS_CHECKCERT=false + + otel-collector: + image: otel/opentelemetry-collector-contrib + volumes: + - ./config.yaml:/etc/otelcol-contrib/config.yaml + ports: + - "4317:4317" # Expose OTLP gRPC port + + prometheus: + image: prom/prometheus:latest + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" # Expose port to access Prometheus expression browser + + event-generator: + image: falcosecurity/event-generator + command: run + restart: always + trigger: + image: alpine + command: [ # Alternate reads to /etc/shadow with creations of symlinks from it + "sh", + "-c", + "while true; do cat /etc/shadow > /dev/null; sleep 5; ln -s /etc/shadow shadow; rm shadow; sleep 5; done" + ] +``` + +> `./docker-compose.yaml` mentions the `falcosidekick:latest` docker image, that must be locally available before +> bringing up the stack. You can build it from source by cloning the repository and issuing the building commands: +> ```shell +> git clone https://github.com/falcosecurity/falcosidekick.git +> cd falcosidekick +> go build . && docker build . -t falcosidekick:latest +> ``` + +- `./config.yaml`: minimal OTEL collector configuration + +```yaml +--- +receivers: + otlp: + protocols: + grpc: + endpoint: "0.0.0.0:4317" + +exporters: + prometheus: + endpoint: "0.0.0.0:9090" + +service: + pipelines: + metrics: + receivers: [otlp] + processors: [] + exporters: [prometheus] +``` + +- `./prometheus.yml`: minimal prometheus configuration + +```yaml +global: + scrape_interval: 5s + +scrape_configs: + - job_name: 'otel-collector' + static_configs: + - targets: ['otel-collector:9090'] +``` + +### Run it + +To bring up the stack, and see the results on prometheus expression browser: + +1. Bring up the stack + + ```shell + docker compose up + ``` + +2. Navigate to to start browsing the local prometheus expression browser + +3. Navigate to the `Graph` tab and adjust the time interval to be comparable to the stack uptime (e.g.: 15 minutes) + +5. To get information regarding the `falcosecurity_falco_rules_matches_total` metric, you can enter a simple query like +`falcosecurity_falco_rules_matches_total` or `sum by (rule) (falcosecurity_falco_rules_matches_total)` and press +`Execute` + +6. Explore the obtained results + ![Falco metrics view](images/otlp_metrics-prom_view.png) + +1. Bring down the stack + + ```shell + docker compose down + ``` diff --git a/go.mod b/go.mod index b8f75fdaf..d1915b791 100644 --- a/go.mod +++ b/go.mod @@ -33,9 +33,13 @@ require ( github.com/xitongsys/parquet-go v1.6.2 github.com/xitongsys/parquet-go-source v0.0.0-20240122235623-d6294584ab18 go.opentelemetry.io/otel v1.30.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 + go.opentelemetry.io/otel/metric v1.30.0 go.opentelemetry.io/otel/sdk v1.30.0 + go.opentelemetry.io/otel/sdk/metric v1.30.0 go.opentelemetry.io/otel/trace v1.30.0 golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 @@ -144,8 +148,6 @@ require ( go.opentelemetry.io/contrib/detectors/gcp v1.29.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.29.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect diff --git a/go.sum b/go.sum index bfda79a80..421465c9a 100644 --- a/go.sum +++ b/go.sum @@ -861,6 +861,10 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+n go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0 h1:WypxHH02KX2poqqbaadmkMYalGyy/vil4HE4PM4nRJc= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0/go.mod h1:U79SV99vtvGSEBeeHnpgGJfTsnsdkWLpPN/CcHAzBSI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0 h1:VrMAbeJz4gnVDg2zEzjHG4dEH86j4jO6VYB+NgtGD8s= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.30.0/go.mod h1:qqN/uFdpeitTvm+JDqqnjm517pmQRYxTORbETHq5tOc= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 h1:lsInsfvhVIfOI6qHVyysXMNDnjO9Npvl7tlDPJFBVd4= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0/go.mod h1:KQsVNh4OjgjTG0G6EiNi1jVpnaeeKsKMRwbLN+f1+8M= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 h1:umZgi92IyxfXd/l4kaDhnKgY8rnN/cZcF1LKc6I8OQ8= @@ -869,8 +873,8 @@ go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4Q go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= -go.opentelemetry.io/otel/sdk/metric v1.29.0 h1:K2CfmJohnRgvZ9UAj2/FhIf/okdWcNdBwe1m8xFXiSY= -go.opentelemetry.io/otel/sdk/metric v1.29.0/go.mod h1:6zZLdCl2fkauYoZIOn/soQIDSWFmNSRcICarHfuhNJQ= +go.opentelemetry.io/otel/sdk/metric v1.30.0 h1:QJLT8Pe11jyHBHfSAgYH7kEmT24eX792jZO1bo4BXkM= +go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y= go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= diff --git a/handlers.go b/handlers.go index b5669393c..5e2f6baf7 100644 --- a/handlers.go +++ b/handlers.go @@ -475,7 +475,13 @@ func forwardEvent(falcopayload types.FalcoPayload) { } if config.OTLP.Traces.Endpoint != "" && (falcopayload.Priority >= types.Priority(config.OTLP.Traces.MinimumPriority)) && (falcopayload.Source == syscall || falcopayload.Source == syscalls) { - go otlpClient.OTLPTracesPost(falcopayload) + go otlpTracesClient.OTLPTracesPost(falcopayload) + } + + if config.OTLP.Metrics.Endpoint != "" && + (falcopayload.Priority) >= types.Priority(config.OTLP.Metrics.MinimumPriority) && + (falcopayload.Source == syscall || falcopayload.Source == syscalls) { + go otlpMetricsClient.OTLPMetricsPost(falcopayload) } if config.Talon.Address != "" && (falcopayload.Priority >= types.Priority(config.Talon.MinimumPriority) || falcopayload.Rule == testRule) { diff --git a/main.go b/main.go index 559c49d43..a0899cf3e 100644 --- a/main.go +++ b/main.go @@ -78,7 +78,8 @@ var ( n8nClient *outputs.Client openObserveClient *outputs.Client dynatraceClient *outputs.Client - otlpClient *outputs.Client + otlpTracesClient *outputs.Client + otlpMetricsClient *outputs.Client talonClient *outputs.Client statsdClient, dogstatsdClient *statsd.Client @@ -801,12 +802,24 @@ func init() { if config.OTLP.Traces.Endpoint != "" { var err error - otlpClient, err = outputs.NewOtlpTracesClient(config, stats, promStats, statsdClient, dogstatsdClient) + otlpTracesClient, err = outputs.NewOtlpTracesClient(config, stats, promStats, statsdClient, dogstatsdClient) if err != nil { config.OTLP.Traces.Endpoint = "" } else { outputs.EnabledOutputs = append(outputs.EnabledOutputs, "OTLPTraces") - shutDownFuncs = append(shutDownFuncs, otlpClient.ShutDownFunc) + shutDownFuncs = append(shutDownFuncs, otlpTracesClient.ShutDownFunc) + } + } + + if config.OTLP.Metrics.Endpoint != "" { + var err error + otlpMetricsClient, err = outputs.NewOTLPMetricsClient(context.Background(), config, stats, promStats, + statsdClient, dogstatsdClient) + if err != nil { + config.OTLP.Metrics.Endpoint = "" + } else { + outputs.EnabledOutputs = append(outputs.EnabledOutputs, "OTLPMetrics") + shutDownFuncs = append(shutDownFuncs, otlpMetricsClient.ShutDownFunc) } } diff --git a/outputs/otlp_metrics.go b/outputs/otlp_metrics.go new file mode 100644 index 000000000..0378209ff --- /dev/null +++ b/outputs/otlp_metrics.go @@ -0,0 +1,237 @@ +package outputs + +import ( + "context" + "fmt" + "github.com/DataDog/datadog-go/statsd" + "github.com/falcosecurity/falcosidekick/types" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/metric" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + sdkresource "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.23.1" + "log" + "os" + "strings" +) + +// TODO: move logging logic out of this context + +// NewOTLPMetricsClient creates a new OTLP Metrics Client. +func NewOTLPMetricsClient(ctx context.Context, config *types.Configuration, stats *types.Statistics, + promStats *types.PromStatistics, statsdClient, dogstatsdClient *statsd.Client) (*Client, error) { + initClientArgs := &types.InitClientArgs{ + Config: config, + Stats: stats, + DogstatsdClient: dogstatsdClient, + PromStats: promStats, + StatsdClient: statsdClient, + } + + cfg := &config.OTLP.Metrics + otlpClient, err := NewClient("OTLPMetrics", cfg.Endpoint, types.CommonConfig{}, *initClientArgs) + if err != nil { + return nil, fmt.Errorf("failed to create output client: %v", err) + } + + restoreEnvironment, err := initEnvironment(cfg) + if err != nil { + return nil, fmt.Errorf("failed to init environemt: %v", err) + } + defer restoreEnvironment() + + shutdownFunc, err := initMeterProvider(ctx, cfg) + if err != nil { + return nil, fmt.Errorf("failed to create meter provider: %v", err) + } + + otlpClient.ShutDownFunc = func() { + if err := shutdownFunc(ctx); err != nil { + log.Printf("[ERROR] : OTLP Metrics - Error: %v\n", err) + } + } + return otlpClient, nil +} + +// initEnvironment initializes the proper environment variables to the corresponding config values. If an environment +// variable is already set, it's value is left uncharged. It returns a function to restore the previous environment +// context. +func initEnvironment(config *types.OTLPMetrics) (cleanupFunc func(), err error) { + cleanupFuncs := make([]func(), 0, 5) + defer func() { + if err != nil { + for _, fn := range cleanupFuncs { + fn() + } + } + }() + + var unsetEnv func() + // As OTLPMetrics fields may have been set by our own config (e.g. YAML), We need to set SDK environment variables + // accordingly. + if endpoint := os.Getenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"); endpoint == "" { + unsetEnv, err = setEnv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", strings.TrimSpace(config.Endpoint)) + if err != nil { + return nil, err + } + cleanupFuncs = append(cleanupFuncs, unsetEnv) + } + + if protocol := os.Getenv("OTEL_EXPORTER_OTLP_METRICS_PROTOCOL"); protocol == "" { + unsetEnv, err = setEnv("OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", strings.TrimSpace(config.Protocol)) + if err != nil { + return nil, err + } + cleanupFuncs = append(cleanupFuncs, unsetEnv) + } + + if headers := os.Getenv("OTEL_EXPORTER_OTLP_METRICS_HEADERS"); headers == "" { + unsetEnv, err = setEnv("OTEL_EXPORTER_OTLP_METRICS_HEADERS", strings.TrimSpace(config.Headers)) + if err != nil { + return nil, err + } + cleanupFuncs = append(cleanupFuncs, unsetEnv) + } + + if timeout := os.Getenv("OTEL_EXPORTER_OTLP_METRICS_TIMEOUT"); timeout == "" { + unsetEnv, err = setEnv("OTEL_EXPORTER_OTLP_METRICS_TIMEOUT", fmt.Sprintf("%d", config.Timeout)) + if err != nil { + return nil, err + } + cleanupFuncs = append(cleanupFuncs, unsetEnv) + } + + for envVar, value := range config.ExtraEnvVars { + if configValue := os.Getenv(envVar); configValue != "" { + continue + } + unsetEnv, err = setEnv(envVar, value) + if err != nil { + return nil, err + } + cleanupFuncs = append(cleanupFuncs, unsetEnv) + } + + return func() { + for _, fn := range cleanupFuncs { + fn() + } + }, nil +} + +func setEnv(envVar, value string) (func(), error) { + if err := os.Setenv(envVar, value); err != nil { + return nil, fmt.Errorf("failed to set %v to %v: %v", envVar, value, err) + } + return func() { + if err := os.Setenv(envVar, ""); err != nil { + log.Printf("[ERROR] : OTLP Metrics - Error unsetting env variable %q: %v\n", envVar, err) + } + }, nil +} + +const ( + meterName = "falcosecurity.falco.otlp.meter" + serviceName = "falco" + serviceVersion = "0.1.0" +) + +// initMeterProvider initializes an OTEL meter provider (and the corresponding exporter). It returns a function to shut +// down the meter provider. +func initMeterProvider(ctx context.Context, config *types.OTLPMetrics) (func(context.Context) error, error) { + var err error + var metricExporter sdkmetric.Exporter + switch protocol := config.Protocol; protocol { + case "grpc": + metricExporter, err = createGRPCExporter(ctx, config) + if err != nil { + return nil, fmt.Errorf("failed to create gRPC exporter: %v", err) + } + case "http/protobuf": + metricExporter, err = createHTTPExporter(ctx, config) + if err != nil { + return nil, fmt.Errorf("failed to create HTTP exporter: %v", err) + } + default: + return nil, fmt.Errorf("unsupported OTLP transport protocol: %s", protocol) + } + + res, err := sdkresource.New(ctx, + sdkresource.WithSchemaURL(semconv.SchemaURL), + sdkresource.WithAttributes( + semconv.ServiceName(serviceName), + semconv.ServiceVersion(serviceVersion), + ), + ) + if err != nil { + return nil, fmt.Errorf("failed to create resource: %v", err) + } + + meterProvider := sdkmetric.NewMeterProvider( + sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExporter)), + sdkmetric.WithResource(res), + ) + + otel.SetMeterProvider(meterProvider) + return meterProvider.Shutdown, nil +} + +func createGRPCExporter(ctx context.Context, config *types.OTLPMetrics) (sdkmetric.Exporter, error) { + var options []otlpmetricgrpc.Option + if config.CheckCert == false { + options = append(options, otlpmetricgrpc.WithInsecure()) + } + + return otlpmetricgrpc.New(ctx, options...) +} + +func createHTTPExporter(ctx context.Context, config *types.OTLPMetrics) (sdkmetric.Exporter, error) { + var options []otlpmetrichttp.Option + if config.CheckCert == false { + options = append(options, otlpmetrichttp.WithInsecure()) + } + return otlpmetrichttp.New(ctx, options...) +} + +const ( + metricName = "falcosecurity_falco_rules_matches_total" + metricDescription = "Number of times rules match" + metricAttributeUUIDKey = "uuid" + metricAttributeSourceKey = "source" + metricAttributePriorityKey = "priority" + metricAttributeRuleKey = "rule" + metricAttributeHostnameKey = "hostname" + metricAttributeTagsKey = "tags" +) + +// OTLPMetricsPost generates a new OTEL metric data point for the provided falco payload. +func (c *Client) OTLPMetricsPost(falcoPayload types.FalcoPayload) { + c.Stats.OTLPMetrics.Add(Total, 1) + + meter := otel.Meter(meterName) + ruleCounter, err := meter.Int64Counter(metricName, metric.WithDescription(metricDescription)) + if err != nil { + go c.CountMetric(Outputs, 1, []string{"output:otlpmetrics", "status:error"}) + c.Stats.OTLPMetrics.Add(Error, 1) + c.PromStats.Outputs.With(map[string]string{"destination": "otlpmetrics", "status": Error}).Inc() + log.Printf("[ERROR] : OTLP Metrics - Error generating metric: %v\n", err) + return + } + + attrs := []attribute.KeyValue{ + attribute.String(metricAttributeUUIDKey, falcoPayload.UUID), + attribute.String(metricAttributeSourceKey, falcoPayload.Source), + attribute.String(metricAttributePriorityKey, falcoPayload.Priority.String()), + attribute.String(metricAttributeRuleKey, falcoPayload.Rule), + attribute.String(metricAttributeHostnameKey, falcoPayload.Hostname), + attribute.StringSlice(metricAttributeTagsKey, falcoPayload.Tags), + } + ruleCounter.Add(context.Background(), 1, metric.WithAttributes(attrs...)) + go c.CountMetric(Outputs, 1, []string{"output:otlpmetrics", "status:ok"}) + c.Stats.OTLPMetrics.Add(OK, 1) + c.PromStats.Outputs.With(map[string]string{"destination": "otlpmetrics", "status": OK}).Inc() + log.Println("[INFO] : OTLP Metrics - OK") +} diff --git a/outputs/otlp.go b/outputs/otlp_traces.go similarity index 100% rename from outputs/otlp.go rename to outputs/otlp_traces.go diff --git a/outputs/otlp_init.go b/outputs/otlp_traces_init.go similarity index 100% rename from outputs/otlp_init.go rename to outputs/otlp_traces_init.go diff --git a/outputs/otlp_test.go b/outputs/otlp_traces_test.go similarity index 100% rename from outputs/otlp_test.go rename to outputs/otlp_traces_test.go diff --git a/stats.go b/stats.go index 1b39f62dc..4c97392f6 100644 --- a/stats.go +++ b/stats.go @@ -87,6 +87,7 @@ func getInitStats() *types.Statistics { OpenObserve: getOutputNewMap("openobserve"), Dynatrace: getOutputNewMap("dynatrace"), OTLPTraces: getOutputNewMap("otlptraces"), + OTLPMetrics: getOutputNewMap("otlpmetrics"), Talon: getOutputNewMap("talon"), } stats.Falco.Add(outputs.Emergency, 0) diff --git a/types/types.go b/types/types.go index df138f215..cff1e8cdd 100644 --- a/types/types.go +++ b/types/types.go @@ -796,9 +796,21 @@ type OTLPTraces struct { MinimumPriority string } +// OTLPMetrics represents config parameters for OTLP Metrics +type OTLPMetrics struct { + Endpoint string + Protocol string + Timeout int64 + Headers string + ExtraEnvVars map[string]string + CheckCert bool + MinimumPriority string +} + // OTLPOutputConfig represents config parameters for OTLP type OTLPOutputConfig struct { - Traces OTLPTraces + Traces OTLPTraces + Metrics OTLPMetrics } // TalonOutputConfig represents parameters for Talon @@ -877,6 +889,7 @@ type Statistics struct { OpenObserve *expvar.Map Dynatrace *expvar.Map OTLPTraces *expvar.Map + OTLPMetrics *expvar.Map Talon *expvar.Map }