diff --git a/.chloggen/entity-event.yaml b/.chloggen/entity-event.yaml new file mode 100644 index 000000000000..ede42ca98660 --- /dev/null +++ b/.chloggen/entity-event.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: hostmetricsreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "allow configuring log pipeline to send host EntityState event" + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33927] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/connector/datadogconnector/go.mod b/connector/datadogconnector/go.mod index a47976342033..a45838b9ab04 100644 --- a/connector/datadogconnector/go.mod +++ b/connector/datadogconnector/go.mod @@ -345,3 +345,5 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/tran replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling => ../../pkg/sampling replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil => ../../internal/pdatautil + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/experimentalmetricmetadata => ../../pkg/experimentalmetricmetadata diff --git a/exporter/datadogexporter/go.mod b/exporter/datadogexporter/go.mod index aa5c9bf0ffdd..4f76a941804b 100644 --- a/exporter/datadogexporter/go.mod +++ b/exporter/datadogexporter/go.mod @@ -257,6 +257,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/internal/docker v0.105.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.105.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil v0.105.0 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/experimentalmetricmetadata v0.0.0-00010101000000-000000000000 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl v0.105.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.105.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling v0.105.0 // indirect @@ -445,3 +446,5 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/tran replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling => ../../pkg/sampling replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil => ../../internal/pdatautil + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/experimentalmetricmetadata => ../../pkg/experimentalmetricmetadata diff --git a/exporter/datadogexporter/integrationtest/go.mod b/exporter/datadogexporter/integrationtest/go.mod index 27304bb5e2ab..82bf59a1198e 100644 --- a/exporter/datadogexporter/integrationtest/go.mod +++ b/exporter/datadogexporter/integrationtest/go.mod @@ -355,3 +355,5 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/processor/tran replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/sampling => ../../../pkg/sampling replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/pdatautil => ../../../internal/pdatautil + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/experimentalmetricmetadata => ../../../pkg/experimentalmetricmetadata diff --git a/receiver/hostmetricsreceiver/README.md b/receiver/hostmetricsreceiver/README.md index 855f4b2be4f7..9e6d2df91dc8 100644 --- a/receiver/hostmetricsreceiver/README.md +++ b/receiver/hostmetricsreceiver/README.md @@ -3,19 +3,21 @@ | Status | | | ------------- |-----------| -| Stability | [beta]: metrics | +| Stability | [development]: logs | +| | [beta]: metrics | | Distributions | [core], [contrib] | | Issues | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Areceiver%2Fhostmetrics%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Areceiver%2Fhostmetrics) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Areceiver%2Fhostmetrics%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Areceiver%2Fhostmetrics) | | [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner) | [@dmitryax](https://www.github.com/dmitryax), [@braydonk](https://www.github.com/braydonk) | +[development]: https://github.com/open-telemetry/opentelemetry-collector#development [beta]: https://github.com/open-telemetry/opentelemetry-collector#beta [core]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol [contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib The Host Metrics receiver generates metrics about the host system scraped -from various sources. This is intended to be used when the collector is -deployed as an agent. +from various sources and host entity event as log. This is intended to be +used when the collector is deployed as an agent. ## Getting Started @@ -188,6 +190,9 @@ Currently, the hostmetrics receiver does not set any Resource attributes on the ``` export OTEL_RESOURCE_ATTRIBUTES="service.name=,service.namespace=,service.instance.id=" ``` +## Entity Events + +**Entity Events as logs are experimental** and might eventually be replaced by the result of [the OTEP](https://github.com/open-telemetry/oteps/blob/main/text/entities/0256-entities-data-model.md#entity-events). For now, the hostmetrics receiver can send the host entity event as a log records. By default, the hostmetrics receiver sends periodic EntityState events every 5 minutes. You can change that by setting `metadata_collection_interval`. Entity Events as logs are experimental. The result of the OTEP might eventually replace that. ## Feature Gates diff --git a/receiver/hostmetricsreceiver/config.go b/receiver/hostmetricsreceiver/config.go index cc20564cd0e9..aae68f79e713 100644 --- a/receiver/hostmetricsreceiver/config.go +++ b/receiver/hostmetricsreceiver/config.go @@ -6,6 +6,7 @@ package hostmetricsreceiver // import "github.com/open-telemetry/opentelemetry-c import ( "errors" "fmt" + "time" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/confmap" @@ -25,6 +26,13 @@ type Config struct { Scrapers map[string]internal.Config `mapstructure:"-"` // RootPath is the host's root directory (linux only). RootPath string `mapstructure:"root_path"` + + // Collection interval for metadata. + // Metadata of the particular entity is collected when the entity changes. + // In addition metadata of all entities is collected periodically even if no changes happen. + // Setting the duration to 0 will disable periodic collection (however will not impact + // metadata collection on changes). + MetadataCollectionInterval time.Duration `mapstructure:"metadata_collection_interval"` } var _ component.Config = (*Config)(nil) diff --git a/receiver/hostmetricsreceiver/config_test.go b/receiver/hostmetricsreceiver/config_test.go index 059896f4b07f..f5247a19651d 100644 --- a/receiver/hostmetricsreceiver/config_test.go +++ b/receiver/hostmetricsreceiver/config_test.go @@ -58,6 +58,7 @@ func TestLoadConfig(t *testing.T) { r1 := cfg.Receivers[component.NewIDWithName(metadata.Type, "customname")].(*Config) expectedConfig := &Config{ + MetadataCollectionInterval: 5 * time.Minute, ControllerConfig: scraperhelper.ControllerConfig{ CollectionInterval: 30 * time.Second, InitialDelay: time.Second, diff --git a/receiver/hostmetricsreceiver/factory.go b/receiver/hostmetricsreceiver/factory.go index 91514da163bd..81cfbd572bed 100644 --- a/receiver/hostmetricsreceiver/factory.go +++ b/receiver/hostmetricsreceiver/factory.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "os" + "time" "github.com/shirou/gopsutil/v4/host" "github.com/shirou/gopsutil/v4/process" @@ -28,6 +29,10 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver/internal/scraper/processscraper" ) +const ( + defaultMetadataCollectionInterval = 5 * time.Minute +) + // This file implements Factory for HostMetrics receiver. var ( scraperFactories = map[string]internal.ScraperFactory{ @@ -48,7 +53,8 @@ func NewFactory() receiver.Factory { return receiver.NewFactory( metadata.Type, createDefaultConfig, - receiver.WithMetrics(createMetricsReceiver, metadata.MetricsStability)) + receiver.WithMetrics(createMetricsReceiver, metadata.MetricsStability), + receiver.WithLogs(createLogsReceiver, metadata.LogsStability)) } func getScraperFactory(key string) (internal.ScraperFactory, bool) { @@ -61,7 +67,10 @@ func getScraperFactory(key string) (internal.ScraperFactory, bool) { // createDefaultConfig creates the default configuration for receiver. func createDefaultConfig() component.Config { - return &Config{ControllerConfig: scraperhelper.NewDefaultControllerConfig()} + return &Config{ + ControllerConfig: scraperhelper.NewDefaultControllerConfig(), + MetadataCollectionInterval: defaultMetadataCollectionInterval, + } } // createMetricsReceiver creates a metrics receiver based on provided config. @@ -89,6 +98,16 @@ func createMetricsReceiver( ) } +func createLogsReceiver( + _ context.Context, set receiver.Settings, cfg component.Config, consumer consumer.Logs, +) (receiver.Logs, error) { + return &hostEntitiesReceiver{ + cfg: cfg.(*Config), + nextLogs: consumer, + settings: &set, + }, nil +} + func createAddScraperOptions( ctx context.Context, set receiver.Settings, diff --git a/receiver/hostmetricsreceiver/factory_test.go b/receiver/hostmetricsreceiver/factory_test.go index a5b854b5969d..542139951c14 100644 --- a/receiver/hostmetricsreceiver/factory_test.go +++ b/receiver/hostmetricsreceiver/factory_test.go @@ -39,8 +39,8 @@ func TestCreateReceiver(t *testing.T) { assert.NotNil(t, mReceiver) tLogs, err := factory.CreateLogsReceiver(context.Background(), creationSet, cfg, consumertest.NewNop()) - assert.Equal(t, err, component.ErrDataTypeIsNotSupported) - assert.Nil(t, tLogs) + assert.NoError(t, err) + assert.NotNil(t, tLogs) } func TestCreateReceiver_ScraperKeyConfigError(t *testing.T) { diff --git a/receiver/hostmetricsreceiver/generated_component_test.go b/receiver/hostmetricsreceiver/generated_component_test.go index 6061f8c59257..35f5556be36f 100644 --- a/receiver/hostmetricsreceiver/generated_component_test.go +++ b/receiver/hostmetricsreceiver/generated_component_test.go @@ -31,6 +31,13 @@ func TestComponentLifecycle(t *testing.T) { createFn func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) }{ + { + name: "logs", + createFn: func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) { + return factory.CreateLogsReceiver(ctx, set, cfg, consumertest.NewNop()) + }, + }, + { name: "metrics", createFn: func(ctx context.Context, set receiver.Settings, cfg component.Config) (component.Component, error) { diff --git a/receiver/hostmetricsreceiver/go.mod b/receiver/hostmetricsreceiver/go.mod index 97ff37183e1c..0c8a4972d8fc 100644 --- a/receiver/hostmetricsreceiver/go.mod +++ b/receiver/hostmetricsreceiver/go.mod @@ -6,6 +6,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.105.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter v0.105.0 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/experimentalmetricmetadata v0.0.0-00010101000000-000000000000 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.105.0 github.com/prometheus-community/windows_exporter v0.25.1 github.com/prometheus/procfs v0.15.1 @@ -21,8 +22,6 @@ require ( go.opentelemetry.io/collector/pdata v1.12.1-0.20240716231837-5753a58f712b go.opentelemetry.io/collector/receiver v0.105.1-0.20240717163034-43ed6184f9fe go.opentelemetry.io/collector/semconv v0.105.1-0.20240717163034-43ed6184f9fe - go.opentelemetry.io/otel/metric v1.28.0 - go.opentelemetry.io/otel/trace v1.28.0 go.uber.org/goleak v1.3.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 @@ -128,9 +127,11 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.28.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.28.0 // indirect go.opentelemetry.io/otel/log v0.4.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/otel/sdk v1.28.0 // indirect go.opentelemetry.io/otel/sdk/log v0.4.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/crypto v0.25.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect @@ -164,3 +165,5 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest => ../../pkg/pdatatest replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden => ../../pkg/golden + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/experimentalmetricmetadata => ../../pkg/experimentalmetricmetadata diff --git a/receiver/hostmetricsreceiver/internal/metadata/generated_status.go b/receiver/hostmetricsreceiver/internal/metadata/generated_status.go index 6273c2b11f4c..01f8ae409aae 100644 --- a/receiver/hostmetricsreceiver/internal/metadata/generated_status.go +++ b/receiver/hostmetricsreceiver/internal/metadata/generated_status.go @@ -11,5 +11,6 @@ var ( ) const ( + LogsStability = component.StabilityLevelDevelopment MetricsStability = component.StabilityLevelBeta ) diff --git a/receiver/hostmetricsreceiver/internal/metadata/generated_telemetry.go b/receiver/hostmetricsreceiver/internal/metadata/generated_telemetry.go deleted file mode 100644 index 51f58af06440..000000000000 --- a/receiver/hostmetricsreceiver/internal/metadata/generated_telemetry.go +++ /dev/null @@ -1,17 +0,0 @@ -// Code generated by mdatagen. DO NOT EDIT. - -package metadata - -import ( - "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/trace" -) - -func Meter(settings component.TelemetrySettings) metric.Meter { - return settings.MeterProvider.Meter("otelcol/hostmetricsreceiver") -} - -func Tracer(settings component.TelemetrySettings) trace.Tracer { - return settings.TracerProvider.Tracer("otelcol/hostmetricsreceiver") -} diff --git a/receiver/hostmetricsreceiver/internal/metadata/generated_telemetry_test.go b/receiver/hostmetricsreceiver/internal/metadata/generated_telemetry_test.go deleted file mode 100644 index 1ee1227f5bb1..000000000000 --- a/receiver/hostmetricsreceiver/internal/metadata/generated_telemetry_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// Code generated by mdatagen. DO NOT EDIT. - -package metadata - -import ( - "testing" - - "github.com/stretchr/testify/require" - "go.opentelemetry.io/otel/metric" - embeddedmetric "go.opentelemetry.io/otel/metric/embedded" - noopmetric "go.opentelemetry.io/otel/metric/noop" - "go.opentelemetry.io/otel/trace" - embeddedtrace "go.opentelemetry.io/otel/trace/embedded" - nooptrace "go.opentelemetry.io/otel/trace/noop" - - "go.opentelemetry.io/collector/component" -) - -type mockMeter struct { - noopmetric.Meter - name string -} -type mockMeterProvider struct { - embeddedmetric.MeterProvider -} - -func (m mockMeterProvider) Meter(name string, opts ...metric.MeterOption) metric.Meter { - return mockMeter{name: name} -} - -type mockTracer struct { - nooptrace.Tracer - name string -} - -type mockTracerProvider struct { - embeddedtrace.TracerProvider -} - -func (m mockTracerProvider) Tracer(name string, opts ...trace.TracerOption) trace.Tracer { - return mockTracer{name: name} -} - -func TestProviders(t *testing.T) { - set := component.TelemetrySettings{ - MeterProvider: mockMeterProvider{}, - TracerProvider: mockTracerProvider{}, - } - - meter := Meter(set) - if m, ok := meter.(mockMeter); ok { - require.Equal(t, "otelcol/hostmetricsreceiver", m.name) - } else { - require.Fail(t, "returned Meter not mockMeter") - } - - tracer := Tracer(set) - if m, ok := tracer.(mockTracer); ok { - require.Equal(t, "otelcol/hostmetricsreceiver", m.name) - } else { - require.Fail(t, "returned Meter not mockTracer") - } -} diff --git a/receiver/hostmetricsreceiver/metadata.yaml b/receiver/hostmetricsreceiver/metadata.yaml index 65c27977d1a9..4652d7b7a511 100644 --- a/receiver/hostmetricsreceiver/metadata.yaml +++ b/receiver/hostmetricsreceiver/metadata.yaml @@ -5,6 +5,7 @@ status: class: receiver stability: beta: [metrics] + development: [logs] distributions: [core, contrib] codeowners: active: [dmitryax, braydonk] diff --git a/receiver/hostmetricsreceiver/receiver.go b/receiver/hostmetricsreceiver/receiver.go new file mode 100644 index 000000000000..598e93bb65d1 --- /dev/null +++ b/receiver/hostmetricsreceiver/receiver.go @@ -0,0 +1,83 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package hostmetricsreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver" + +import ( + "context" + "time" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/receiver" + "go.uber.org/zap" + + metadataPkg "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/experimentalmetricmetadata" +) + +const entityType = "host" + +type hostEntitiesReceiver struct { + cfg *Config + + nextLogs consumer.Logs + cancel context.CancelFunc + + settings *receiver.Settings +} + +func (hmr *hostEntitiesReceiver) Start(ctx context.Context, _ component.Host) error { + ctx, hmr.cancel = context.WithCancel(ctx) + if hmr.nextLogs != nil { + hmr.sendEntityEvent(ctx) + if hmr.cfg.MetadataCollectionInterval != 0 { + ticker := time.NewTicker(hmr.cfg.MetadataCollectionInterval) + go func() { + for { + select { + case <-ticker.C: + hmr.sendEntityEvent(ctx) + case <-ctx.Done(): + ticker.Stop() + return + } + } + }() + } + } + + return nil +} + +func (hmr *hostEntitiesReceiver) Shutdown(_ context.Context) error { + if hmr.cancel != nil { + hmr.cancel() + } + return nil +} + +func (hmr *hostEntitiesReceiver) sendEntityEvent(ctx context.Context) { + timestamp := pcommon.NewTimestampFromTime(time.Now()) + + out := metadataPkg.NewEntityEventsSlice() + entityEvent := out.AppendEmpty() + entityEvent.SetTimestamp(timestamp) + state := entityEvent.SetEntityState() + state.SetEntityType(entityType) + + logs := out.ConvertAndMoveToLogs() + + err := hmr.nextLogs.ConsumeLogs(ctx, logs) + if err != nil { + hmr.settings.Logger.Error("Error sending entity event to the consumer", zap.Error(err)) + } + + // Note: receiver contract says that we need to retry sending if the + // returned error is not Permanent. However, we are not doing it here. + // Instead, we rely on the fact the metadata is collected periodically + // and the entity events will be delivered on the next cycle. This is + // fine because we deliver cumulative entity state. + // This allows us to avoid stressing the Collector or its destination + // unnecessarily (typically non-Permanent errors happen in stressed conditions). +} diff --git a/receiver/hostmetricsreceiver/receiver_test.go b/receiver/hostmetricsreceiver/receiver_test.go new file mode 100644 index 000000000000..291e5a1cbb0d --- /dev/null +++ b/receiver/hostmetricsreceiver/receiver_test.go @@ -0,0 +1,30 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package hostmetricsreceiver + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/receiver/receivertest" +) + +func TestReceiver(t *testing.T) { + factory := NewFactory() + cfg := factory.CreateDefaultConfig() + + settings := receivertest.NewNopSettings() + sink := new(consumertest.LogsSink) + logs, err := factory.CreateLogsReceiver(context.Background(), settings, cfg, sink) + require.NoError(t, err) + assert.NoError(t, logs.Start(context.Background(), componenttest.NewNopHost())) + assert.NoError(t, logs.Shutdown(context.Background())) + + allLogs := sink.AllLogs() + require.Len(t, allLogs, 1) +}