From 7da5cddf485647cdc2a1e2bcaabd056012fb6eb8 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Thu, 19 Sep 2024 09:11:47 +0200 Subject: [PATCH 01/35] Add conversion webhook config to dev CRD variant --- .../crd/patches/cainjection_in_logpipelines.yaml | 7 ------- .../cainjection_in_operator_telemetries.yaml | 7 ------- .../cainjection_in_telemetry_logparsers.yaml | 7 ------- ...cainjection_in_telemetry_metricpipelines.yaml | 7 ------- .../cainjection_in_telemetry_tracepipelines.yaml | 7 ------- .../patches/webhook_in_telemetry_logparsers.yaml | 2 +- .../webhook_in_telemetry_logpipelines.yaml | 16 ++++++++++++++++ .../webhook_in_telemetry_metricpipelines.yaml | 2 +- .../webhook_in_telemetry_tracepipelines.yaml | 2 +- config/development/crd/kustomization.yaml | 3 +++ .../webhook_in_telemetry_logpipelines.yaml} | 2 +- 11 files changed, 23 insertions(+), 39 deletions(-) delete mode 100644 config/crd/patches/cainjection_in_logpipelines.yaml delete mode 100644 config/crd/patches/cainjection_in_operator_telemetries.yaml delete mode 100644 config/crd/patches/cainjection_in_telemetry_logparsers.yaml delete mode 100644 config/crd/patches/cainjection_in_telemetry_metricpipelines.yaml delete mode 100644 config/crd/patches/cainjection_in_telemetry_tracepipelines.yaml create mode 100644 config/crd/patches/webhook_in_telemetry_logpipelines.yaml rename config/{crd/patches/webhook_in_logpipelines.yaml => development/crd/patches/webhook_in_telemetry_logpipelines.yaml} (92%) diff --git a/config/crd/patches/cainjection_in_logpipelines.yaml b/config/crd/patches/cainjection_in_logpipelines.yaml deleted file mode 100644 index a50360943..000000000 --- a/config/crd/patches/cainjection_in_logpipelines.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: logpipelines.telemetry.kyma-project.io diff --git a/config/crd/patches/cainjection_in_operator_telemetries.yaml b/config/crd/patches/cainjection_in_operator_telemetries.yaml deleted file mode 100644 index aad22cd70..000000000 --- a/config/crd/patches/cainjection_in_operator_telemetries.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: telemetries.operator.kyma-project.io diff --git a/config/crd/patches/cainjection_in_telemetry_logparsers.yaml b/config/crd/patches/cainjection_in_telemetry_logparsers.yaml deleted file mode 100644 index cf124b551..000000000 --- a/config/crd/patches/cainjection_in_telemetry_logparsers.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: logparsers.telemetry.kyma-project.io diff --git a/config/crd/patches/cainjection_in_telemetry_metricpipelines.yaml b/config/crd/patches/cainjection_in_telemetry_metricpipelines.yaml deleted file mode 100644 index 81bf50ab9..000000000 --- a/config/crd/patches/cainjection_in_telemetry_metricpipelines.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: metricpipelines.telemetry.kyma-project.io diff --git a/config/crd/patches/cainjection_in_telemetry_tracepipelines.yaml b/config/crd/patches/cainjection_in_telemetry_tracepipelines.yaml deleted file mode 100644 index 28d8550e3..000000000 --- a/config/crd/patches/cainjection_in_telemetry_tracepipelines.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# The following patch adds a directive for certmanager to inject CA into the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) - name: tracepipelines.telemetry.kyma-project.io diff --git a/config/crd/patches/webhook_in_telemetry_logparsers.yaml b/config/crd/patches/webhook_in_telemetry_logparsers.yaml index 47f48cca3..5366726d9 100644 --- a/config/crd/patches/webhook_in_telemetry_logparsers.yaml +++ b/config/crd/patches/webhook_in_telemetry_logparsers.yaml @@ -10,7 +10,7 @@ spec: clientConfig: service: namespace: system - name: webhook-service + name: manager-webhook path: /convert conversionReviewVersions: - v1 diff --git a/config/crd/patches/webhook_in_telemetry_logpipelines.yaml b/config/crd/patches/webhook_in_telemetry_logpipelines.yaml new file mode 100644 index 000000000..baae9696e --- /dev/null +++ b/config/crd/patches/webhook_in_telemetry_logpipelines.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: logpipelines.telemetry.kyma-project.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: manager-webhook + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/crd/patches/webhook_in_telemetry_metricpipelines.yaml b/config/crd/patches/webhook_in_telemetry_metricpipelines.yaml index ed5aac757..9e3922b96 100644 --- a/config/crd/patches/webhook_in_telemetry_metricpipelines.yaml +++ b/config/crd/patches/webhook_in_telemetry_metricpipelines.yaml @@ -10,7 +10,7 @@ spec: clientConfig: service: namespace: system - name: webhook-service + name: manager-webhook path: /convert conversionReviewVersions: - v1 diff --git a/config/crd/patches/webhook_in_telemetry_tracepipelines.yaml b/config/crd/patches/webhook_in_telemetry_tracepipelines.yaml index 64ad1b6f6..82a9e5cb7 100644 --- a/config/crd/patches/webhook_in_telemetry_tracepipelines.yaml +++ b/config/crd/patches/webhook_in_telemetry_tracepipelines.yaml @@ -10,7 +10,7 @@ spec: clientConfig: service: namespace: system - name: webhook-service + name: manager-webhook path: /convert conversionReviewVersions: - v1 diff --git a/config/development/crd/kustomization.yaml b/config/development/crd/kustomization.yaml index 339fa6bae..ec26b82a6 100644 --- a/config/development/crd/kustomization.yaml +++ b/config/development/crd/kustomization.yaml @@ -5,6 +5,9 @@ resources: - bases/operator.kyma-project.io_telemetries.yaml - bases/telemetry.kyma-project.io_metricpipelines.yaml +patchesStrategicMerge: +- patches/webhook_in_telemetry_logpipelines.yaml + # the following config is for teaching kustomize how to do kustomization for CRDs. configurations: - kustomizeconfig.yaml diff --git a/config/crd/patches/webhook_in_logpipelines.yaml b/config/development/crd/patches/webhook_in_telemetry_logpipelines.yaml similarity index 92% rename from config/crd/patches/webhook_in_logpipelines.yaml rename to config/development/crd/patches/webhook_in_telemetry_logpipelines.yaml index 50289f385..0425cc8d7 100644 --- a/config/crd/patches/webhook_in_logpipelines.yaml +++ b/config/development/crd/patches/webhook_in_telemetry_logpipelines.yaml @@ -10,7 +10,7 @@ spec: clientConfig: service: namespace: system - name: webhook-service + name: manager-webhook path: /convert conversionReviewVersions: - v1 From 7f2a84b232c3d405625b2417afe87d3fe1cd8cd9 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Thu, 19 Sep 2024 09:23:36 +0200 Subject: [PATCH 02/35] Rename v1beta1 types --- apis/telemetry/v1beta1/logpipeline_types.go | 64 +++++++------- .../v1beta1/logpipeline_types_test.go | 12 +-- .../v1beta1/logpipeline_validation.go | 8 +- .../v1beta1/logpipeline_validation_test.go | 86 +++++++++---------- apis/telemetry/v1beta1/secret_refs_test.go | 6 +- .../v1beta1/zz_generated.deepcopy.go | 72 ++++++++-------- ...elemetry.kyma-project.io_logpipelines.yaml | 2 +- test/e2e/logs_basic_v1beta1_test.go | 6 +- 8 files changed, 128 insertions(+), 128 deletions(-) diff --git a/apis/telemetry/v1beta1/logpipeline_types.go b/apis/telemetry/v1beta1/logpipeline_types.go index 6c1e5b51c..5879edec9 100644 --- a/apis/telemetry/v1beta1/logpipeline_types.go +++ b/apis/telemetry/v1beta1/logpipeline_types.go @@ -47,27 +47,27 @@ type LogPipelineSpec struct { // Important: Run "make" to regenerate code after modifying this file // Defines where to collect logs, including selector mechanisms. - Input Input `json:"input,omitempty"` - Filters []Filter `json:"filters,omitempty"` + Input LogPipelineInput `json:"input,omitempty"` + Filters []LogPipelineFilter `json:"filters,omitempty"` // [Fluent Bit output](https://docs.fluentbit.io/manual/pipeline/outputs) where you want to push the logs. Only one output can be specified. - Output Output `json:"output,omitempty"` - Files []FileMount `json:"files,omitempty"` + Output LogPipelineOutput `json:"output,omitempty"` + Files []LogPipelineFileMount `json:"files,omitempty"` // A list of mappings from Kubernetes Secret keys to environment variables. Mapped keys are mounted as environment variables, so that they are available as [Variables](https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/variables) in the sections. - Variables []VariableRef `json:"variables,omitempty"` + Variables []LogPipelineVariableRef `json:"variables,omitempty"` } -// Input describes a log input for a LogPipeline. -type Input struct { +// LogPipelineInput describes a log input for a LogPipeline. +type LogPipelineInput struct { // Configures in more detail from which containers application logs are enabled as input. - Application ApplicationInput `json:"application,omitempty"` + Runtime *LogPipelineRuntimeInput `json:"runtime,omitempty"` } -// ApplicationInput specifies the default type of Input that handles application logs from runtime containers. It configures in more detail from which containers logs are selected as input. -type ApplicationInput struct { +// LogPipelineRuntimeInput specifies the default type of Input that handles application logs from runtime containers. It configures in more detail from which containers logs are selected as input. +type LogPipelineRuntimeInput struct { // Describes whether application logs from specific Namespaces are selected. The options are mutually exclusive. System Namespaces are excluded by default from the collection. - Namespaces InputNamespaces `json:"namespaces,omitempty"` + Namespaces LogPipelineInputNamespaces `json:"namespaces,omitempty"` // Describes whether application logs from specific containers are selected. The options are mutually exclusive. - Containers InputContainers `json:"containers,omitempty"` + Containers LogPipelineInputContainers `json:"containers,omitempty"` // Defines whether to keep all Kubernetes annotations. The default is `false`. KeepAnnotations bool `json:"keepAnnotations,omitempty"` // Defines whether to drop all Kubernetes labels. The default is `false`. @@ -78,8 +78,8 @@ type ApplicationInput struct { KeepOriginalBody *bool `json:"keepOriginalBody,omitempty"` } -// InputNamespaces describes whether application logs from specific Namespaces are selected. The options are mutually exclusive. System Namespaces are excluded by default from the collection. -type InputNamespaces struct { +// LogPipelineInputNamespaces describes whether application logs from specific Namespaces are selected. The options are mutually exclusive. System Namespaces are excluded by default from the collection. +type LogPipelineInputNamespaces struct { // Include only the container logs of the specified Namespace names. Include []string `json:"include,omitempty"` // Exclude the container logs of the specified Namespace names. @@ -88,8 +88,8 @@ type InputNamespaces struct { System bool `json:"system,omitempty"` } -// InputContainers describes whether application logs from specific containers are selected. The options are mutually exclusive. -type InputContainers struct { +// LogPipelineInputContainers describes whether application logs from specific containers are selected. The options are mutually exclusive. +type LogPipelineInputContainers struct { // Specifies to include only the container logs with the specified container names. Include []string `json:"include,omitempty"` // Specifies to exclude only the container logs with the specified container names. @@ -97,21 +97,21 @@ type InputContainers struct { } // Describes a filtering option on the logs of the pipeline. -type Filter struct { +type LogPipelineFilter struct { // Custom filter definition in the Fluent Bit syntax. Note: If you use a `custom` filter, you put the LogPipeline in unsupported mode. Custom string `json:"custom,omitempty"` } -// Output describes a Fluent Bit output configuration section. -type Output struct { +// LogPipelineOutput describes a Fluent Bit output configuration section. +type LogPipelineOutput struct { // Defines a custom output in the Fluent Bit syntax. Note: If you use a `custom` output, you put the LogPipeline in unsupported mode. Custom string `json:"custom,omitempty"` // Configures an HTTP-based output compatible with the Fluent Bit HTTP output plugin. - HTTP *HTTPOutput `json:"http,omitempty"` + HTTP *LogPipelineHTTPOutput `json:"http,omitempty"` } -// HTTPOutput configures an HTTP-based output compatible with the Fluent Bit HTTP output plugin. -type HTTPOutput struct { +// LogPipelineHTTPOutput configures an HTTP-based output compatible with the Fluent Bit HTTP output plugin. +type LogPipelineHTTPOutput struct { // Defines the host of the HTTP receiver. Host ValueType `json:"host,omitempty"` // Defines the basic auth user. @@ -127,13 +127,13 @@ type HTTPOutput struct { // Data format to be used in the HTTP request body. Default is `json`. Format string `json:"format,omitempty"` // Configures TLS for the HTTP target server. - TLSConfig TLSConfig `json:"tls,omitempty"` + TLSConfig LogPipelineHTTPOutputTLS `json:"tls,omitempty"` // Enables de-dotting of Kubernetes labels and annotations for compatibility with ElasticSearch based backends. Dots (.) will be replaced by underscores (_). Default is `false`. Dedot bool `json:"dedot,omitempty"` } // +kubebuilder:validation:XValidation:rule="has(self.cert) == has(self.key)", message="Can define either both 'cert' and 'key', or neither" -type TLSConfig struct { +type LogPipelineHTTPOutputTLS struct { // Indicates if TLS is disabled or enabled. Default is `false`. Disabled bool `json:"disabled,omitempty"` // If `true`, the validation of certificates is skipped. Default is `false`. @@ -147,13 +147,13 @@ type TLSConfig struct { } // Provides file content to be consumed by a LogPipeline configuration -type FileMount struct { +type LogPipelineFileMount struct { Name string `json:"name,omitempty"` Content string `json:"content,omitempty"` } // References a Kubernetes secret that should be provided as environment variable to Fluent Bit -type VariableRef struct { +type LogPipelineVariableRef struct { // Name of the variable to map. Name string `json:"name,omitempty"` ValueFrom ValueFromSource `json:"valueFrom,omitempty"` @@ -180,27 +180,27 @@ func init() { SchemeBuilder.Register(&LogPipeline{}, &LogPipelineList{}) } -func (i *Input) IsDefined() bool { +func (i *LogPipelineInput) IsDefined() bool { return i != nil } -func (o *Output) IsCustomDefined() bool { +func (o *LogPipelineOutput) IsCustomDefined() bool { return o.Custom != "" } -func (o *Output) IsHTTPDefined() bool { +func (o *LogPipelineOutput) IsHTTPDefined() bool { return o.HTTP != nil && o.HTTP.Host.IsDefined() } -func (o *Output) IsAnyDefined() bool { +func (o *LogPipelineOutput) IsAnyDefined() bool { return o.pluginCount() > 0 } -func (o *Output) IsSingleDefined() bool { +func (o *LogPipelineOutput) IsSingleDefined() bool { return o.pluginCount() == 1 } -func (o *Output) pluginCount() int { +func (o *LogPipelineOutput) pluginCount() int { plugins := 0 if o.IsCustomDefined() { plugins++ diff --git a/apis/telemetry/v1beta1/logpipeline_types_test.go b/apis/telemetry/v1beta1/logpipeline_types_test.go index fff2a2940..9b008dc9a 100644 --- a/apis/telemetry/v1beta1/logpipeline_types_test.go +++ b/apis/telemetry/v1beta1/logpipeline_types_test.go @@ -9,7 +9,7 @@ import ( func TestLogPipelineOutput(t *testing.T) { tests := []struct { name string - given Output + given LogPipelineOutput expectedCustom bool expectedHTTP bool expectedLoki bool @@ -18,21 +18,21 @@ func TestLogPipelineOutput(t *testing.T) { }{ { name: "custom", - given: Output{Custom: "name: null"}, + given: LogPipelineOutput{Custom: "name: null"}, expectedCustom: true, expectedAny: true, expectedSingle: true, }, { name: "http", - given: Output{HTTP: &HTTPOutput{Host: ValueType{Value: "localhost"}}}, + given: LogPipelineOutput{HTTP: &LogPipelineHTTPOutput{Host: ValueType{Value: "localhost"}}}, expectedHTTP: true, expectedAny: true, expectedSingle: true, }, { name: "invalid: none defined", - given: Output{}, + given: LogPipelineOutput{}, expectedAny: false, expectedSingle: false, }, @@ -50,7 +50,7 @@ func TestLogPipelineOutput(t *testing.T) { func TestLogPipelineContainsCustomPluginWithCustomFilter(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Filters: []Filter{ + Filters: []LogPipelineFilter{ {Custom: ` Name some-filter`, }, @@ -65,7 +65,7 @@ func TestLogPipelineContainsCustomPluginWithCustomFilter(t *testing.T) { func TestLogPipelineContainsCustomPluginWithCustomOutput(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Output: Output{ + Output: LogPipelineOutput{ Custom: ` Name some-output`, }, diff --git a/apis/telemetry/v1beta1/logpipeline_validation.go b/apis/telemetry/v1beta1/logpipeline_validation.go index bef63fcb9..387896c0f 100644 --- a/apis/telemetry/v1beta1/logpipeline_validation.go +++ b/apis/telemetry/v1beta1/logpipeline_validation.go @@ -33,7 +33,7 @@ func (lp *LogPipeline) validateOutput(deniedOutputPlugins []string) error { return validateCustomOutput(deniedOutputPlugins, output.Custom) } -func checkSingleOutputPlugin(output Output) error { +func checkSingleOutputPlugin(output LogPipelineOutput) error { if !output.IsAnyDefined() { return fmt.Errorf("no output plugin is defined, you must define one output plugin") } @@ -43,7 +43,7 @@ func checkSingleOutputPlugin(output Output) error { return nil } -func validateHTTPOutput(httpOutput *HTTPOutput) error { +func validateHTTPOutput(httpOutput *LogPipelineHTTPOutput) error { isValidHostname, err := validHostname(httpOutput.Host.Value) if err != nil { @@ -155,12 +155,12 @@ func (lp *LogPipeline) validateInput() error { return nil } - var containers = input.Application.Containers + var containers = input.Runtime.Containers if len(containers.Include) > 0 && len(containers.Exclude) > 0 { return fmt.Errorf("invalid log pipeline definition: Cannot define both 'input.application.containers.include' and 'input.application.containers.exclude'") } - var namespaces = input.Application.Namespaces + var namespaces = input.Runtime.Namespaces if (len(namespaces.Include) > 0 && len(namespaces.Exclude) > 0) || (len(namespaces.Include) > 0 && namespaces.System) || (len(namespaces.Exclude) > 0 && namespaces.System) { diff --git a/apis/telemetry/v1beta1/logpipeline_validation_test.go b/apis/telemetry/v1beta1/logpipeline_validation_test.go index d288ce1c6..40b32a312 100644 --- a/apis/telemetry/v1beta1/logpipeline_validation_test.go +++ b/apis/telemetry/v1beta1/logpipeline_validation_test.go @@ -10,7 +10,7 @@ import ( func TestContainsNoOutputPlugins(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Output: Output{}, + Output: LogPipelineOutput{}, }} vc := getLogPipelineValidationConfig() @@ -23,9 +23,9 @@ func TestContainsNoOutputPlugins(t *testing.T) { func TestContainsMultipleOutputPlugins(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Output: Output{ + Output: LogPipelineOutput{ Custom: `Name http`, - HTTP: &HTTPOutput{ + HTTP: &LogPipelineHTTPOutput{ Host: ValueType{ Value: "localhost", }, @@ -43,7 +43,7 @@ func TestDeniedOutputPlugins(t *testing.T) { logPipeline := &LogPipeline{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: LogPipelineSpec{ - Output: Output{ + Output: LogPipelineOutput{ Custom: ` Name lua`, }, @@ -61,7 +61,7 @@ func TestValidateCustomOutput(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Output: Output{ + Output: LogPipelineOutput{ Custom: ` name http`, }, @@ -77,7 +77,7 @@ func TestValidateCustomHasForbiddenParameter(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Output: Output{ + Output: LogPipelineOutput{ Custom: ` name http storage.total_limit_size 10G`, @@ -93,7 +93,7 @@ func TestValidateCustomHasForbiddenParameter(t *testing.T) { func TestValidateCustomOutputsContainsNoName(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Output: Output{ + Output: LogPipelineOutput{ Custom: ` Regex .*`, }, @@ -110,8 +110,8 @@ func TestValidateCustomOutputsContainsNoName(t *testing.T) { func TestBothValueAndValueFromPresent(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Output: Output{ - HTTP: &HTTPOutput{ + Output: LogPipelineOutput{ + HTTP: &LogPipelineHTTPOutput{ Host: ValueType{ Value: "localhost", ValueFrom: &ValueFromSource{ @@ -134,8 +134,8 @@ func TestBothValueAndValueFromPresent(t *testing.T) { func TestValueFromSecretKeyRef(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Output: Output{ - HTTP: &HTTPOutput{ + Output: LogPipelineOutput{ + HTTP: &LogPipelineHTTPOutput{ Host: ValueType{ ValueFrom: &ValueFromSource{ SecretKeyRef: &SecretKeyRef{ @@ -161,7 +161,7 @@ func TestValidateCustomFilter(t *testing.T) { logPipeline := &LogPipeline{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: LogPipelineSpec{ - Output: Output{ + Output: LogPipelineOutput{ Custom: ` Name http`, }, @@ -176,7 +176,7 @@ func TestValidateCustomFilter(t *testing.T) { func TestValidateCustomFiltersContainsNoName(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Filters: []Filter{ + Filters: []LogPipelineFilter{ {Custom: ` Match *`, }, @@ -193,7 +193,7 @@ func TestValidateCustomFiltersContainsNoName(t *testing.T) { func TestValidateCustomFiltersContainsMatch(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Filters: []Filter{ + Filters: []LogPipelineFilter{ {Custom: ` Name grep Match *`, @@ -213,7 +213,7 @@ func TestDeniedFilterPlugins(t *testing.T) { logPipeline := &LogPipeline{ ObjectMeta: metav1.ObjectMeta{Name: "foo"}, Spec: LogPipelineSpec{ - Filters: []Filter{ + Filters: []LogPipelineFilter{ {Custom: ` Name lua`, }, @@ -231,12 +231,12 @@ func TestDeniedFilterPlugins(t *testing.T) { func TestValidateWithValidInputIncludes(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Input: Input{ - Application: ApplicationInput{ - Namespaces: InputNamespaces{ + Input: LogPipelineInput{ + Runtime: LogPipelineRuntimeInput{ + Namespaces: LogPipelineInputNamespaces{ Include: []string{"namespace-1", "namespace-2"}, }, - Containers: InputContainers{ + Containers: LogPipelineInputContainers{ Include: []string{"container-1"}, }, }, @@ -250,12 +250,12 @@ func TestValidateWithValidInputIncludes(t *testing.T) { func TestValidateWithValidInputExcludes(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Input: Input{ - Application: ApplicationInput{ - Namespaces: InputNamespaces{ + Input: LogPipelineInput{ + Runtime: LogPipelineRuntimeInput{ + Namespaces: LogPipelineInputNamespaces{ Exclude: []string{"namespace-1", "namespace-2"}, }, - Containers: InputContainers{ + Containers: LogPipelineInputContainers{ Exclude: []string{"container-1"}, }, }, @@ -270,12 +270,12 @@ func TestValidateWithValidInputExcludes(t *testing.T) { func TestValidateWithValidInputIncludeContainersSystemFlag(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Input: Input{ - Application: ApplicationInput{ - Namespaces: InputNamespaces{ + Input: LogPipelineInput{ + Runtime: LogPipelineRuntimeInput{ + Namespaces: LogPipelineInputNamespaces{ System: true, }, - Containers: InputContainers{ + Containers: LogPipelineInputContainers{ Include: []string{"container-1"}, }, }, @@ -290,12 +290,12 @@ func TestValidateWithValidInputIncludeContainersSystemFlag(t *testing.T) { func TestValidateWithValidInputExcludeContainersSystemFlag(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Input: Input{ - Application: ApplicationInput{ - Namespaces: InputNamespaces{ + Input: LogPipelineInput{ + Runtime: LogPipelineRuntimeInput{ + Namespaces: LogPipelineInputNamespaces{ System: true, }, - Containers: InputContainers{ + Containers: LogPipelineInputContainers{ Exclude: []string{"container-1"}, }, }, @@ -310,9 +310,9 @@ func TestValidateWithValidInputExcludeContainersSystemFlag(t *testing.T) { func TestValidateWithInvalidNamespaceSelectors(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Input: Input{ - Application: ApplicationInput{ - Namespaces: InputNamespaces{ + Input: LogPipelineInput{ + Runtime: LogPipelineRuntimeInput{ + Namespaces: LogPipelineInputNamespaces{ Include: []string{"namespace-1", "namespace-2"}, Exclude: []string{"namespace-3"}, }, @@ -328,9 +328,9 @@ func TestValidateWithInvalidNamespaceSelectors(t *testing.T) { func TestValidateWithInvalidIncludeSystemFlag(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Input: Input{ - Application: ApplicationInput{ - Namespaces: InputNamespaces{ + Input: LogPipelineInput{ + Runtime: LogPipelineRuntimeInput{ + Namespaces: LogPipelineInputNamespaces{ Include: []string{"namespace-1", "namespace-2"}, System: true, }, @@ -346,9 +346,9 @@ func TestValidateWithInvalidIncludeSystemFlag(t *testing.T) { func TestValidateWithInvalidExcludeSystemFlag(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Input: Input{ - Application: ApplicationInput{ - Namespaces: InputNamespaces{ + Input: LogPipelineInput{ + Runtime: LogPipelineRuntimeInput{ + Namespaces: LogPipelineInputNamespaces{ Exclude: []string{"namespace-3"}, System: true, }, @@ -364,9 +364,9 @@ func TestValidateWithInvalidExcludeSystemFlag(t *testing.T) { func TestValidateWithInvalidContainerSelectors(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ - Input: Input{ - Application: ApplicationInput{ - Containers: InputContainers{ + Input: LogPipelineInput{ + Runtime: LogPipelineRuntimeInput{ + Containers: LogPipelineInputContainers{ Include: []string{"container-1", "container-2"}, Exclude: []string{"container-3"}, }, diff --git a/apis/telemetry/v1beta1/secret_refs_test.go b/apis/telemetry/v1beta1/secret_refs_test.go index 3228f628b..8c3a8d7d0 100644 --- a/apis/telemetry/v1beta1/secret_refs_test.go +++ b/apis/telemetry/v1beta1/secret_refs_test.go @@ -17,7 +17,7 @@ func TestLogPipeline_GetSecretRefs(t *testing.T) { name: "only variables", given: LogPipeline{ Spec: LogPipelineSpec{ - Variables: []VariableRef{ + Variables: []LogPipelineVariableRef{ { Name: "password-1", ValueFrom: ValueFromSource{ @@ -46,8 +46,8 @@ func TestLogPipeline_GetSecretRefs(t *testing.T) { Name: "cls", }, Spec: LogPipelineSpec{ - Output: Output{ - HTTP: &HTTPOutput{ + Output: LogPipelineOutput{ + HTTP: &LogPipelineHTTPOutput{ Host: ValueType{ ValueFrom: &ValueFromSource{ SecretKeyRef: &SecretKeyRef{ diff --git a/apis/telemetry/v1beta1/zz_generated.deepcopy.go b/apis/telemetry/v1beta1/zz_generated.deepcopy.go index 3407f84db..83859e042 100644 --- a/apis/telemetry/v1beta1/zz_generated.deepcopy.go +++ b/apis/telemetry/v1beta1/zz_generated.deepcopy.go @@ -1,4 +1,4 @@ -//go:build !ignore_autogenerated +// s / */go:build !ignore_autogenerated /* Copyright 2021. @@ -26,7 +26,7 @@ import ( ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ApplicationInput) DeepCopyInto(out *ApplicationInput) { +func (in *LogPipelineRuntimeInput) DeepCopyInto(out *LogPipelineRuntimeInput) { *out = *in in.Namespaces.DeepCopyInto(&out.Namespaces) in.Containers.DeepCopyInto(&out.Containers) @@ -38,11 +38,11 @@ func (in *ApplicationInput) DeepCopyInto(out *ApplicationInput) { } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApplicationInput. -func (in *ApplicationInput) DeepCopy() *ApplicationInput { +func (in *LogPipelineRuntimeInput) DeepCopy() *LogPipelineRuntimeInput { if in == nil { return nil } - out := new(ApplicationInput) + out := new(LogPipelineRuntimeInput) in.DeepCopyInto(out) return out } @@ -100,37 +100,37 @@ func (in *DiagnosticMetrics) DeepCopy() *DiagnosticMetrics { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *FileMount) DeepCopyInto(out *FileMount) { +func (in *LogPipelineFileMount) DeepCopyInto(out *LogPipelineFileMount) { *out = *in } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileMount. -func (in *FileMount) DeepCopy() *FileMount { +func (in *LogPipelineFileMount) DeepCopy() *LogPipelineFileMount { if in == nil { return nil } - out := new(FileMount) + out := new(LogPipelineFileMount) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Filter) DeepCopyInto(out *Filter) { +func (in *LogPipelineFilter) DeepCopyInto(out *LogPipelineFilter) { *out = *in } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Filter. -func (in *Filter) DeepCopy() *Filter { +func (in *LogPipelineFilter) DeepCopy() *LogPipelineFilter { if in == nil { return nil } - out := new(Filter) + out := new(LogPipelineFilter) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *HTTPOutput) DeepCopyInto(out *HTTPOutput) { +func (in *LogPipelineHTTPOutput) DeepCopyInto(out *LogPipelineHTTPOutput) { *out = *in in.Host.DeepCopyInto(&out.Host) in.User.DeepCopyInto(&out.User) @@ -139,11 +139,11 @@ func (in *HTTPOutput) DeepCopyInto(out *HTTPOutput) { } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPOutput. -func (in *HTTPOutput) DeepCopy() *HTTPOutput { +func (in *LogPipelineHTTPOutput) DeepCopy() *LogPipelineHTTPOutput { if in == nil { return nil } - out := new(HTTPOutput) + out := new(LogPipelineHTTPOutput) in.DeepCopyInto(out) return out } @@ -165,23 +165,23 @@ func (in *Header) DeepCopy() *Header { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Input) DeepCopyInto(out *Input) { +func (in *LogPipelineInput) DeepCopyInto(out *LogPipelineInput) { *out = *in - in.Application.DeepCopyInto(&out.Application) + in.Runtime.DeepCopyInto(&out.Runtime) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Input. -func (in *Input) DeepCopy() *Input { +func (in *LogPipelineInput) DeepCopy() *LogPipelineInput { if in == nil { return nil } - out := new(Input) + out := new(LogPipelineInput) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *InputContainers) DeepCopyInto(out *InputContainers) { +func (in *LogPipelineInputContainers) DeepCopyInto(out *LogPipelineInputContainers) { *out = *in if in.Include != nil { in, out := &in.Include, &out.Include @@ -196,17 +196,17 @@ func (in *InputContainers) DeepCopyInto(out *InputContainers) { } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InputContainers. -func (in *InputContainers) DeepCopy() *InputContainers { +func (in *LogPipelineInputContainers) DeepCopy() *LogPipelineInputContainers { if in == nil { return nil } - out := new(InputContainers) + out := new(LogPipelineInputContainers) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *InputNamespaces) DeepCopyInto(out *InputNamespaces) { +func (in *LogPipelineInputNamespaces) DeepCopyInto(out *LogPipelineInputNamespaces) { *out = *in if in.Include != nil { in, out := &in.Include, &out.Include @@ -221,11 +221,11 @@ func (in *InputNamespaces) DeepCopyInto(out *InputNamespaces) { } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InputNamespaces. -func (in *InputNamespaces) DeepCopy() *InputNamespaces { +func (in *LogPipelineInputNamespaces) DeepCopy() *LogPipelineInputNamespaces { if in == nil { return nil } - out := new(InputNamespaces) + out := new(LogPipelineInputNamespaces) in.DeepCopyInto(out) return out } @@ -295,18 +295,18 @@ func (in *LogPipelineSpec) DeepCopyInto(out *LogPipelineSpec) { in.Input.DeepCopyInto(&out.Input) if in.Filters != nil { in, out := &in.Filters, &out.Filters - *out = make([]Filter, len(*in)) + *out = make([]LogPipelineFilter, len(*in)) copy(*out, *in) } in.Output.DeepCopyInto(&out.Output) if in.Files != nil { in, out := &in.Files, &out.Files - *out = make([]FileMount, len(*in)) + *out = make([]LogPipelineFileMount, len(*in)) copy(*out, *in) } if in.Variables != nil { in, out := &in.Variables, &out.Variables - *out = make([]VariableRef, len(*in)) + *out = make([]LogPipelineVariableRef, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -757,21 +757,21 @@ func (in *OTLPTLS) DeepCopy() *OTLPTLS { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Output) DeepCopyInto(out *Output) { +func (in *LogPipelineOutput) DeepCopyInto(out *LogPipelineOutput) { *out = *in if in.HTTP != nil { in, out := &in.HTTP, &out.HTTP - *out = new(HTTPOutput) + *out = new(LogPipelineHTTPOutput) (*in).DeepCopyInto(*out) } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Output. -func (in *Output) DeepCopy() *Output { +func (in *LogPipelineOutput) DeepCopy() *LogPipelineOutput { if in == nil { return nil } - out := new(Output) + out := new(LogPipelineOutput) in.DeepCopyInto(out) return out } @@ -792,7 +792,7 @@ func (in *SecretKeyRef) DeepCopy() *SecretKeyRef { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { +func (in *LogPipelineHTTPOutputTLS) DeepCopyInto(out *LogPipelineHTTPOutputTLS) { *out = *in if in.CA != nil { in, out := &in.CA, &out.CA @@ -812,11 +812,11 @@ func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig. -func (in *TLSConfig) DeepCopy() *TLSConfig { +func (in *LogPipelineHTTPOutputTLS) DeepCopy() *LogPipelineHTTPOutputTLS { if in == nil { return nil } - out := new(TLSConfig) + out := new(LogPipelineHTTPOutputTLS) in.DeepCopyInto(out) return out } @@ -979,17 +979,17 @@ func (in *ValueType) DeepCopy() *ValueType { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *VariableRef) DeepCopyInto(out *VariableRef) { +func (in *LogPipelineVariableRef) DeepCopyInto(out *LogPipelineVariableRef) { *out = *in in.ValueFrom.DeepCopyInto(&out.ValueFrom) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VariableRef. -func (in *VariableRef) DeepCopy() *VariableRef { +func (in *LogPipelineVariableRef) DeepCopy() *LogPipelineVariableRef { if in == nil { return nil } - out := new(VariableRef) + out := new(LogPipelineVariableRef) in.DeepCopyInto(out) return out } diff --git a/config/development/crd/bases/telemetry.kyma-project.io_logpipelines.yaml b/config/development/crd/bases/telemetry.kyma-project.io_logpipelines.yaml index c93b5d72b..bbf77f006 100644 --- a/config/development/crd/bases/telemetry.kyma-project.io_logpipelines.yaml +++ b/config/development/crd/bases/telemetry.kyma-project.io_logpipelines.yaml @@ -547,7 +547,7 @@ spec: input: description: Defines where to collect logs, including selector mechanisms. properties: - application: + runtime: description: Configures in more detail from which containers application logs are enabled as input. properties: diff --git a/test/e2e/logs_basic_v1beta1_test.go b/test/e2e/logs_basic_v1beta1_test.go index d47f57fd2..208634423 100644 --- a/test/e2e/logs_basic_v1beta1_test.go +++ b/test/e2e/logs_basic_v1beta1_test.go @@ -43,14 +43,14 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs, suite.LabelExperimental), Or Name: pipelineName, }, Spec: telemetryv1beta1.LogPipelineSpec{ - Output: telemetryv1beta1.Output{ - HTTP: &telemetryv1beta1.HTTPOutput{ + Output: telemetryv1beta1.LogPipelineOutput{ + HTTP: &telemetryv1beta1.LogPipelineHTTPOutput{ Host: telemetryv1beta1.ValueType{ Value: backend.Host(), }, Port: strconv.Itoa(int(backend.Port())), URI: "/", - TLSConfig: telemetryv1beta1.TLSConfig{ + TLSConfig: telemetryv1beta1.LogPipelineHTTPOutputTLS{ Disabled: true, SkipCertificateValidation: true, }, From 5c34b46b1d0daffb41b3275053dd9241ff63c6b4 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Thu, 19 Sep 2024 11:31:51 +0200 Subject: [PATCH 03/35] Implemented ConvertTo --- .../v1alpha1/logpipeline_conversion.go | 86 ++++++ .../v1beta1/lgpipeline_conversion.go | 5 + .../v1beta1/logpipeline_validation_test.go | 16 +- .../v1beta1/zz_generated.deepcopy.go | 256 +++++++++--------- 4 files changed, 229 insertions(+), 134 deletions(-) create mode 100644 apis/telemetry/v1alpha1/logpipeline_conversion.go create mode 100644 apis/telemetry/v1beta1/lgpipeline_conversion.go diff --git a/apis/telemetry/v1alpha1/logpipeline_conversion.go b/apis/telemetry/v1alpha1/logpipeline_conversion.go new file mode 100644 index 000000000..cd5b29509 --- /dev/null +++ b/apis/telemetry/v1alpha1/logpipeline_conversion.go @@ -0,0 +1,86 @@ +package v1alpha1 + +import ( + "sigs.k8s.io/controller-runtime/pkg/conversion" + + telemetryv1beta1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1beta1" +) + +// ConvertTo converts this CronJob to the Hub version (v1). +func (src *LogPipeline) ConvertTo(dstRaw conversion.Hub) error { + dst := dstRaw.(*telemetryv1beta1.LogPipeline) + + srcAppInput := src.Spec.Input.Application + dst.Spec.Input = telemetryv1beta1.LogPipelineInput{ + Runtime: &telemetryv1beta1.LogPipelineRuntimeInput{ + Namespaces: telemetryv1beta1.LogPipelineInputNamespaces(srcAppInput.Namespaces), + Containers: telemetryv1beta1.LogPipelineInputContainers(srcAppInput.Containers), + KeepAnnotations: srcAppInput.KeepAnnotations, + DropLabels: srcAppInput.DropLabels, + KeepOriginalBody: srcAppInput.KeepOriginalBody, + }, + } + + for _, f := range src.Spec.Files { + dst.Spec.Files = append(dst.Spec.Files, telemetryv1beta1.LogPipelineFileMount(f)) + } + + for _, f := range src.Spec.Filters { + dst.Spec.Filters = append(dst.Spec.Filters, telemetryv1beta1.LogPipelineFilter(f)) + } + + srcHTTPOutput := src.Spec.Output.HTTP + if srcHTTPOutput != nil { + dst.Spec.Output.HTTP = &telemetryv1beta1.LogPipelineHTTPOutput{ + Host: v1Alpha1ValueTypeToV1Beta1(srcHTTPOutput.Host), + User: v1Alpha1ValueTypeToV1Beta1(srcHTTPOutput.User), + Password: v1Alpha1ValueTypeToV1Beta1(srcHTTPOutput.Password), + URI: srcHTTPOutput.URI, + Port: srcHTTPOutput.Port, + Compress: srcHTTPOutput.Compress, + Format: srcHTTPOutput.Format, + TLSConfig: v1Alpha1TLSToV1Beta1(srcHTTPOutput.TLSConfig), + Dedot: srcHTTPOutput.Dedot, + } + } + + srcCustomOutput := src.Spec.Output.Custom + if srcCustomOutput != "" { + dst.Spec.Output.Custom = srcCustomOutput + } + + return nil +} + +func v1Alpha1ValueTypeToV1Beta1(src ValueType) telemetryv1beta1.ValueType { + if src.ValueFrom != nil && src.ValueFrom.SecretKeyRef != nil { + return telemetryv1beta1.ValueType{ + ValueFrom: &telemetryv1beta1.ValueFromSource{ + SecretKeyRef: (*telemetryv1beta1.SecretKeyRef)(src.ValueFrom.SecretKeyRef), + }, + } + } + return telemetryv1beta1.ValueType{ + Value: src.Value, + } +} + +func v1Alpha1TLSToV1Beta1(src TLSConfig) telemetryv1beta1.LogPipelineHTTPOutputTLS { + var dst telemetryv1beta1.LogPipelineHTTPOutputTLS + if src.CA != nil { + ca := v1Alpha1ValueTypeToV1Beta1(*src.CA) + dst.CA = &ca + } + if src.Cert != nil { + cert := v1Alpha1ValueTypeToV1Beta1(*src.Cert) + dst.Cert = &cert + } + if src.Key != nil { + key := v1Alpha1ValueTypeToV1Beta1(*src.Key) + dst.Key = &key + } + dst.Disabled = src.Disabled + dst.SkipCertificateValidation = src.SkipCertificateValidation + + return dst +} diff --git a/apis/telemetry/v1beta1/lgpipeline_conversion.go b/apis/telemetry/v1beta1/lgpipeline_conversion.go new file mode 100644 index 000000000..ffb8e4163 --- /dev/null +++ b/apis/telemetry/v1beta1/lgpipeline_conversion.go @@ -0,0 +1,5 @@ +package v1beta1 + +// Hub marks this type as a conversion hub. +func (lp *LogPipeline) Hub() { +} diff --git a/apis/telemetry/v1beta1/logpipeline_validation_test.go b/apis/telemetry/v1beta1/logpipeline_validation_test.go index 40b32a312..c52bcdba3 100644 --- a/apis/telemetry/v1beta1/logpipeline_validation_test.go +++ b/apis/telemetry/v1beta1/logpipeline_validation_test.go @@ -232,7 +232,7 @@ func TestValidateWithValidInputIncludes(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: LogPipelineRuntimeInput{ + Runtime: &LogPipelineRuntimeInput{ Namespaces: LogPipelineInputNamespaces{ Include: []string{"namespace-1", "namespace-2"}, }, @@ -251,7 +251,7 @@ func TestValidateWithValidInputExcludes(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: LogPipelineRuntimeInput{ + Runtime: &LogPipelineRuntimeInput{ Namespaces: LogPipelineInputNamespaces{ Exclude: []string{"namespace-1", "namespace-2"}, }, @@ -271,7 +271,7 @@ func TestValidateWithValidInputIncludeContainersSystemFlag(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: LogPipelineRuntimeInput{ + Runtime: &LogPipelineRuntimeInput{ Namespaces: LogPipelineInputNamespaces{ System: true, }, @@ -291,7 +291,7 @@ func TestValidateWithValidInputExcludeContainersSystemFlag(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: LogPipelineRuntimeInput{ + Runtime: &LogPipelineRuntimeInput{ Namespaces: LogPipelineInputNamespaces{ System: true, }, @@ -311,7 +311,7 @@ func TestValidateWithInvalidNamespaceSelectors(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: LogPipelineRuntimeInput{ + Runtime: &LogPipelineRuntimeInput{ Namespaces: LogPipelineInputNamespaces{ Include: []string{"namespace-1", "namespace-2"}, Exclude: []string{"namespace-3"}, @@ -329,7 +329,7 @@ func TestValidateWithInvalidIncludeSystemFlag(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: LogPipelineRuntimeInput{ + Runtime: &LogPipelineRuntimeInput{ Namespaces: LogPipelineInputNamespaces{ Include: []string{"namespace-1", "namespace-2"}, System: true, @@ -347,7 +347,7 @@ func TestValidateWithInvalidExcludeSystemFlag(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: LogPipelineRuntimeInput{ + Runtime: &LogPipelineRuntimeInput{ Namespaces: LogPipelineInputNamespaces{ Exclude: []string{"namespace-3"}, System: true, @@ -365,7 +365,7 @@ func TestValidateWithInvalidContainerSelectors(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: LogPipelineRuntimeInput{ + Runtime: &LogPipelineRuntimeInput{ Containers: LogPipelineInputContainers{ Include: []string{"container-1", "container-2"}, Exclude: []string{"container-3"}, diff --git a/apis/telemetry/v1beta1/zz_generated.deepcopy.go b/apis/telemetry/v1beta1/zz_generated.deepcopy.go index 83859e042..7722b6992 100644 --- a/apis/telemetry/v1beta1/zz_generated.deepcopy.go +++ b/apis/telemetry/v1beta1/zz_generated.deepcopy.go @@ -1,4 +1,4 @@ -// s / */go:build !ignore_autogenerated +//go:build !ignore_autogenerated /* Copyright 2021. @@ -25,28 +25,6 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LogPipelineRuntimeInput) DeepCopyInto(out *LogPipelineRuntimeInput) { - *out = *in - in.Namespaces.DeepCopyInto(&out.Namespaces) - in.Containers.DeepCopyInto(&out.Containers) - if in.KeepOriginalBody != nil { - in, out := &in.KeepOriginalBody, &out.KeepOriginalBody - *out = new(bool) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApplicationInput. -func (in *LogPipelineRuntimeInput) DeepCopy() *LogPipelineRuntimeInput { - if in == nil { - return nil - } - out := new(LogPipelineRuntimeInput) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AuthenticationOptions) DeepCopyInto(out *AuthenticationOptions) { *out = *in @@ -99,12 +77,55 @@ func (in *DiagnosticMetrics) DeepCopy() *DiagnosticMetrics { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Header) DeepCopyInto(out *Header) { + *out = *in + in.ValueType.DeepCopyInto(&out.ValueType) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Header. +func (in *Header) DeepCopy() *Header { + if in == nil { + return nil + } + out := new(Header) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LogPipeline) DeepCopyInto(out *LogPipeline) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogPipeline. +func (in *LogPipeline) DeepCopy() *LogPipeline { + if in == nil { + return nil + } + out := new(LogPipeline) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *LogPipeline) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LogPipelineFileMount) DeepCopyInto(out *LogPipelineFileMount) { *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FileMount. +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogPipelineFileMount. func (in *LogPipelineFileMount) DeepCopy() *LogPipelineFileMount { if in == nil { return nil @@ -119,7 +140,7 @@ func (in *LogPipelineFilter) DeepCopyInto(out *LogPipelineFilter) { *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Filter. +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogPipelineFilter. func (in *LogPipelineFilter) DeepCopy() *LogPipelineFilter { if in == nil { return nil @@ -138,7 +159,7 @@ func (in *LogPipelineHTTPOutput) DeepCopyInto(out *LogPipelineHTTPOutput) { in.TLSConfig.DeepCopyInto(&out.TLSConfig) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPOutput. +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogPipelineHTTPOutput. func (in *LogPipelineHTTPOutput) DeepCopy() *LogPipelineHTTPOutput { if in == nil { return nil @@ -149,17 +170,31 @@ func (in *LogPipelineHTTPOutput) DeepCopy() *LogPipelineHTTPOutput { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Header) DeepCopyInto(out *Header) { +func (in *LogPipelineHTTPOutputTLS) DeepCopyInto(out *LogPipelineHTTPOutputTLS) { *out = *in - in.ValueType.DeepCopyInto(&out.ValueType) + if in.CA != nil { + in, out := &in.CA, &out.CA + *out = new(ValueType) + (*in).DeepCopyInto(*out) + } + if in.Cert != nil { + in, out := &in.Cert, &out.Cert + *out = new(ValueType) + (*in).DeepCopyInto(*out) + } + if in.Key != nil { + in, out := &in.Key, &out.Key + *out = new(ValueType) + (*in).DeepCopyInto(*out) + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Header. -func (in *Header) DeepCopy() *Header { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogPipelineHTTPOutputTLS. +func (in *LogPipelineHTTPOutputTLS) DeepCopy() *LogPipelineHTTPOutputTLS { if in == nil { return nil } - out := new(Header) + out := new(LogPipelineHTTPOutputTLS) in.DeepCopyInto(out) return out } @@ -167,10 +202,14 @@ func (in *Header) DeepCopy() *Header { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LogPipelineInput) DeepCopyInto(out *LogPipelineInput) { *out = *in - in.Runtime.DeepCopyInto(&out.Runtime) + if in.Runtime != nil { + in, out := &in.Runtime, &out.Runtime + *out = new(LogPipelineRuntimeInput) + (*in).DeepCopyInto(*out) + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Input. +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogPipelineInput. func (in *LogPipelineInput) DeepCopy() *LogPipelineInput { if in == nil { return nil @@ -195,7 +234,7 @@ func (in *LogPipelineInputContainers) DeepCopyInto(out *LogPipelineInputContaine } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InputContainers. +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogPipelineInputContainers. func (in *LogPipelineInputContainers) DeepCopy() *LogPipelineInputContainers { if in == nil { return nil @@ -220,7 +259,7 @@ func (in *LogPipelineInputNamespaces) DeepCopyInto(out *LogPipelineInputNamespac } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InputNamespaces. +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogPipelineInputNamespaces. func (in *LogPipelineInputNamespaces) DeepCopy() *LogPipelineInputNamespaces { if in == nil { return nil @@ -231,26 +270,31 @@ func (in *LogPipelineInputNamespaces) DeepCopy() *LogPipelineInputNamespaces { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LogPipeline) DeepCopyInto(out *LogPipeline) { +func (in *LogPipelineList) DeepCopyInto(out *LogPipelineList) { *out = *in out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]LogPipeline, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogPipeline. -func (in *LogPipeline) DeepCopy() *LogPipeline { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogPipelineList. +func (in *LogPipelineList) DeepCopy() *LogPipelineList { if in == nil { return nil } - out := new(LogPipeline) + out := new(LogPipelineList) in.DeepCopyInto(out) return out } // DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *LogPipeline) DeepCopyObject() runtime.Object { +func (in *LogPipelineList) DeepCopyObject() runtime.Object { if c := in.DeepCopy(); c != nil { return c } @@ -258,35 +302,45 @@ func (in *LogPipeline) DeepCopyObject() runtime.Object { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LogPipelineList) DeepCopyInto(out *LogPipelineList) { +func (in *LogPipelineOutput) DeepCopyInto(out *LogPipelineOutput) { *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]LogPipeline, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } + if in.HTTP != nil { + in, out := &in.HTTP, &out.HTTP + *out = new(LogPipelineHTTPOutput) + (*in).DeepCopyInto(*out) } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogPipelineList. -func (in *LogPipelineList) DeepCopy() *LogPipelineList { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogPipelineOutput. +func (in *LogPipelineOutput) DeepCopy() *LogPipelineOutput { if in == nil { return nil } - out := new(LogPipelineList) + out := new(LogPipelineOutput) in.DeepCopyInto(out) return out } -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *LogPipelineList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LogPipelineRuntimeInput) DeepCopyInto(out *LogPipelineRuntimeInput) { + *out = *in + in.Namespaces.DeepCopyInto(&out.Namespaces) + in.Containers.DeepCopyInto(&out.Containers) + if in.KeepOriginalBody != nil { + in, out := &in.KeepOriginalBody, &out.KeepOriginalBody + *out = new(bool) + **out = **in } - return nil +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogPipelineRuntimeInput. +func (in *LogPipelineRuntimeInput) DeepCopy() *LogPipelineRuntimeInput { + if in == nil { + return nil + } + out := new(LogPipelineRuntimeInput) + in.DeepCopyInto(out) + return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -375,6 +429,22 @@ func (in *LogPipelineValidationConfig) DeepCopy() *LogPipelineValidationConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LogPipelineVariableRef) DeepCopyInto(out *LogPipelineVariableRef) { + *out = *in + in.ValueFrom.DeepCopyInto(&out.ValueFrom) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogPipelineVariableRef. +func (in *LogPipelineVariableRef) DeepCopy() *LogPipelineVariableRef { + if in == nil { + return nil + } + out := new(LogPipelineVariableRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MetricPipeline) DeepCopyInto(out *MetricPipeline) { *out = *in @@ -756,26 +826,6 @@ func (in *OTLPTLS) DeepCopy() *OTLPTLS { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LogPipelineOutput) DeepCopyInto(out *LogPipelineOutput) { - *out = *in - if in.HTTP != nil { - in, out := &in.HTTP, &out.HTTP - *out = new(LogPipelineHTTPOutput) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Output. -func (in *LogPipelineOutput) DeepCopy() *LogPipelineOutput { - if in == nil { - return nil - } - out := new(LogPipelineOutput) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SecretKeyRef) DeepCopyInto(out *SecretKeyRef) { *out = *in @@ -791,36 +841,6 @@ func (in *SecretKeyRef) DeepCopy() *SecretKeyRef { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LogPipelineHTTPOutputTLS) DeepCopyInto(out *LogPipelineHTTPOutputTLS) { - *out = *in - if in.CA != nil { - in, out := &in.CA, &out.CA - *out = new(ValueType) - (*in).DeepCopyInto(*out) - } - if in.Cert != nil { - in, out := &in.Cert, &out.Cert - *out = new(ValueType) - (*in).DeepCopyInto(*out) - } - if in.Key != nil { - in, out := &in.Key, &out.Key - *out = new(ValueType) - (*in).DeepCopyInto(*out) - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig. -func (in *LogPipelineHTTPOutputTLS) DeepCopy() *LogPipelineHTTPOutputTLS { - if in == nil { - return nil - } - out := new(LogPipelineHTTPOutputTLS) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TracePipeline) DeepCopyInto(out *TracePipeline) { *out = *in @@ -977,19 +997,3 @@ func (in *ValueType) DeepCopy() *ValueType { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *LogPipelineVariableRef) DeepCopyInto(out *LogPipelineVariableRef) { - *out = *in - in.ValueFrom.DeepCopyInto(&out.ValueFrom) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VariableRef. -func (in *LogPipelineVariableRef) DeepCopy() *LogPipelineVariableRef { - if in == nil { - return nil - } - out := new(LogPipelineVariableRef) - in.DeepCopyInto(out) - return out -} From bb557a969465af0c10039e5a6a7a245adf009fc2 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Thu, 19 Sep 2024 11:59:08 +0200 Subject: [PATCH 04/35] Implemented ConvertFrom --- .../v1alpha1/logpipeline_conversion.go | 88 +++++++++++++++++-- 1 file changed, 83 insertions(+), 5 deletions(-) diff --git a/apis/telemetry/v1alpha1/logpipeline_conversion.go b/apis/telemetry/v1alpha1/logpipeline_conversion.go index cd5b29509..68b04500c 100644 --- a/apis/telemetry/v1alpha1/logpipeline_conversion.go +++ b/apis/telemetry/v1alpha1/logpipeline_conversion.go @@ -6,7 +6,7 @@ import ( telemetryv1beta1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1beta1" ) -// ConvertTo converts this CronJob to the Hub version (v1). +// ConvertTo converts this LogPipeline to the Hub version (v1beta1). func (src *LogPipeline) ConvertTo(dstRaw conversion.Hub) error { dst := dstRaw.(*telemetryv1beta1.LogPipeline) @@ -29,8 +29,7 @@ func (src *LogPipeline) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.Filters = append(dst.Spec.Filters, telemetryv1beta1.LogPipelineFilter(f)) } - srcHTTPOutput := src.Spec.Output.HTTP - if srcHTTPOutput != nil { + if srcHTTPOutput := src.Spec.Output.HTTP; srcHTTPOutput != nil { dst.Spec.Output.HTTP = &telemetryv1beta1.LogPipelineHTTPOutput{ Host: v1Alpha1ValueTypeToV1Beta1(srcHTTPOutput.Host), User: v1Alpha1ValueTypeToV1Beta1(srcHTTPOutput.User), @@ -44,8 +43,7 @@ func (src *LogPipeline) ConvertTo(dstRaw conversion.Hub) error { } } - srcCustomOutput := src.Spec.Output.Custom - if srcCustomOutput != "" { + if srcCustomOutput := src.Spec.Output.Custom; srcCustomOutput != "" { dst.Spec.Output.Custom = srcCustomOutput } @@ -60,6 +58,7 @@ func v1Alpha1ValueTypeToV1Beta1(src ValueType) telemetryv1beta1.ValueType { }, } } + return telemetryv1beta1.ValueType{ Value: src.Value, } @@ -67,20 +66,99 @@ func v1Alpha1ValueTypeToV1Beta1(src ValueType) telemetryv1beta1.ValueType { func v1Alpha1TLSToV1Beta1(src TLSConfig) telemetryv1beta1.LogPipelineHTTPOutputTLS { var dst telemetryv1beta1.LogPipelineHTTPOutputTLS + if src.CA != nil { ca := v1Alpha1ValueTypeToV1Beta1(*src.CA) dst.CA = &ca } + if src.Cert != nil { cert := v1Alpha1ValueTypeToV1Beta1(*src.Cert) dst.Cert = &cert } + if src.Key != nil { key := v1Alpha1ValueTypeToV1Beta1(*src.Key) dst.Key = &key } + dst.Disabled = src.Disabled dst.SkipCertificateValidation = src.SkipCertificateValidation return dst } + +// ConvertFrom converts from the Hub version (v1beta1) to this version. +func (dst *LogPipeline) ConvertFrom(srcRaw conversion.Hub) error { + src := srcRaw.(*telemetryv1beta1.LogPipeline) + + if srcAppInput := src.Spec.Input.Runtime; srcAppInput != nil { + dst.Spec.Input.Application = ApplicationInput{ + Namespaces: InputNamespaces(srcAppInput.Namespaces), + Containers: InputContainers(srcAppInput.Containers), + KeepAnnotations: srcAppInput.KeepAnnotations, + DropLabels: srcAppInput.DropLabels, + KeepOriginalBody: srcAppInput.KeepOriginalBody, + } + } + + for _, f := range src.Spec.Files { + dst.Spec.Files = append(dst.Spec.Files, FileMount(f)) + } + + for _, f := range src.Spec.Filters { + dst.Spec.Filters = append(dst.Spec.Filters, Filter(f)) + } + + if srcHTTPOutput := src.Spec.Output.HTTP; srcHTTPOutput != nil { + dst.Spec.Output.HTTP = &HTTPOutput{ + Host: v1Beta1ValueTypeToV1Alpha1(srcHTTPOutput.Host), + User: v1Beta1ValueTypeToV1Alpha1(srcHTTPOutput.User), + Password: v1Beta1ValueTypeToV1Alpha1(srcHTTPOutput.Password), + URI: srcHTTPOutput.URI, + Port: srcHTTPOutput.Port, + Compress: srcHTTPOutput.Compress, + Format: srcHTTPOutput.Format, + TLSConfig: v1Beta1TLSToV1Alpha1(srcHTTPOutput.TLSConfig), + Dedot: srcHTTPOutput.Dedot, + } + } + + if srcCustomOutput := src.Spec.Output.Custom; srcCustomOutput != "" { + dst.Spec.Output.Custom = srcCustomOutput + } + + return nil +} + +func v1Beta1TLSToV1Alpha1(src telemetryv1beta1.LogPipelineHTTPOutputTLS) TLSConfig { + var dst TLSConfig + if src.CA != nil { + ca := v1Beta1ValueTypeToV1Alpha1(*src.CA) + dst.CA = &ca + } + if src.Cert != nil { + cert := v1Beta1ValueTypeToV1Alpha1(*src.Cert) + dst.Cert = &cert + } + if src.Key != nil { + key := v1Beta1ValueTypeToV1Alpha1(*src.Key) + dst.Key = &key + } + dst.Disabled = src.Disabled + dst.SkipCertificateValidation = src.SkipCertificateValidation + return dst +} + +func v1Beta1ValueTypeToV1Alpha1(src telemetryv1beta1.ValueType) ValueType { + if src.ValueFrom != nil && src.ValueFrom.SecretKeyRef != nil { + return ValueType{ + ValueFrom: &ValueFromSource{ + SecretKeyRef: (*SecretKeyRef)(src.ValueFrom.SecretKeyRef), + }, + } + } + return ValueType{ + Value: src.Value, + } +} From b6c079436617263c5a2df5b704ee2904431d41d2 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Thu, 19 Sep 2024 12:16:53 +0200 Subject: [PATCH 05/35] Setup webhook --- main.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/main.go b/main.go index c28ffa8a8..3ed5784e5 100644 --- a/main.go +++ b/main.go @@ -49,6 +49,7 @@ import ( operatorv1alpha1 "github.com/kyma-project/telemetry-manager/apis/operator/v1alpha1" telemetryv1alpha1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1alpha1" + telemetryv1beta1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1beta1" "github.com/kyma-project/telemetry-manager/controllers/operator" telemetrycontrollers "github.com/kyma-project/telemetry-manager/controllers/telemetry" "github.com/kyma-project/telemetry-manager/internal/fluentbit/config/builder" @@ -426,6 +427,20 @@ func enableLoggingController(mgr manager.Manager, reconcileTriggerChan <-chan ev mgr.GetWebhookServer().Register("/validate-logpipeline", &webhook.Admission{Handler: createLogPipelineValidator(mgr.GetClient())}) mgr.GetWebhookServer().Register("/validate-logparser", &webhook.Admission{Handler: createLogParserValidator(mgr.GetClient())}) + if err := ctrl.NewWebhookManagedBy(mgr). + For(&telemetryv1alpha1.LogPipeline{}). + Complete(); err != nil { + setupLog.Error(err, "Failed to create v1alpha1 conversion webhook", "webhook", "LogPipeline") + os.Exit(1) + } + + if err := ctrl.NewWebhookManagedBy(mgr). + For(&telemetryv1beta1.LogPipeline{}). + Complete(); err != nil { + setupLog.Error(err, "Failed to create v1beta1 conversion webhook", "webhook", "LogPipeline") + os.Exit(1) + } + logPipelineController, err := telemetrycontrollers.NewLogPipelineController( mgr.GetClient(), reconcileTriggerChan, From cd463336000ea4df830539efd5edfa895b6d994f Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Thu, 19 Sep 2024 13:04:29 +0200 Subject: [PATCH 06/35] Fix --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 3ed5784e5..973e59bc8 100644 --- a/main.go +++ b/main.go @@ -146,6 +146,7 @@ func init() { utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) utilruntime.Must(telemetryv1alpha1.AddToScheme(scheme)) + utilruntime.Must(telemetryv1beta1.AddToScheme(scheme)) utilruntime.Must(operatorv1alpha1.AddToScheme(scheme)) utilruntime.Must(istiosecurityclientv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme From d8f252a9a12c8e92bd686629feb6ed77ce61a689 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Fri, 20 Sep 2024 09:24:56 +0200 Subject: [PATCH 07/35] Adapt rbac --- config/rbac/role.yaml | 2 ++ main.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index e8ba28baf..f7b588294 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -59,6 +59,8 @@ rules: verbs: - get - list + - patch + - update - watch - apiGroups: - apps diff --git a/main.go b/main.go index 973e59bc8..6f74c09e7 100644 --- a/main.go +++ b/main.go @@ -199,7 +199,7 @@ func getEnvOrDefault(envVar string, defaultValue string) string { //+kubebuilder:rbac:urls=/metrics,verbs=get //+kubebuilder:rbac:urls=/metrics/cadvisor,verbs=get -//+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch +//+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch;update;patch //+kubebuilder:rbac:groups=apps,namespace=system,resources=deployments,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=apps,namespace=system,resources=daemonsets,verbs=get;list;watch;create;update;patch;delete From ff473c37b625fcf66214848c86c865f2e06eeb61 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Fri, 20 Sep 2024 09:41:50 +0200 Subject: [PATCH 08/35] Move validating webhook config to a separate file --- .../log_pipeline_webhook_configs.go | 133 ++++++++++++++++++ internal/webhookcert/webhook_cert.go | 96 +------------ 2 files changed, 134 insertions(+), 95 deletions(-) create mode 100644 internal/webhookcert/log_pipeline_webhook_configs.go diff --git a/internal/webhookcert/log_pipeline_webhook_configs.go b/internal/webhookcert/log_pipeline_webhook_configs.go new file mode 100644 index 000000000..c0c7590ce --- /dev/null +++ b/internal/webhookcert/log_pipeline_webhook_configs.go @@ -0,0 +1,133 @@ +package webhookcert + +import ( + "context" + "fmt" + + "github.com/kyma-project/telemetry-manager/internal/k8sutils" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ensureLogPipelineWebhookConfigs creates or updates the ValidatingWebhookConfiguration for the LogPipeline resources. +// additionally it patches the conversion webhook configuration with the CA bundle. +func ensureLogPipelineWebhookConfigs(ctx context.Context, c client.Client, caBundle []byte, config Config) error { + validatingWebhookConfig := makeValidatingWebhookConfig(caBundle, config) + if err := k8sutils.CreateOrUpdateValidatingWebhookConfiguration(ctx, c, &validatingWebhookConfig); err != nil { + return fmt.Errorf("failed to create or update validating webhook configuration: %w", err) + } + + if err := patchConversionWebhookConfig(ctx, c, caBundle, config); err != nil { + return fmt.Errorf("failed to patch conversion webhook configuration: %w", err) + } + + return nil +} + +func makeValidatingWebhookConfig(certificate []byte, config Config) admissionregistrationv1.ValidatingWebhookConfiguration { + logPipelinePath := "/validate-logpipeline" + logParserPath := "/validate-logparser" + failurePolicy := admissionregistrationv1.Fail + matchPolicy := admissionregistrationv1.Exact + sideEffects := admissionregistrationv1.SideEffectClassNone + operations := []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + admissionregistrationv1.Update, + } + apiGroups := []string{"telemetry.kyma-project.io"} + apiVersions := []string{"v1alpha1"} + scope := admissionregistrationv1.AllScopes + servicePort := int32(443) + timeout := int32(15) + labels := map[string]string{ + "control-plane": "telemetry-manager", + "app.kubernetes.io/instance": "telemetry", + "app.kubernetes.io/name": "manager", + "kyma-project.io/component": "controller", + } + + return admissionregistrationv1.ValidatingWebhookConfiguration{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: config.WebhookName.Name, + Labels: labels, + }, + Webhooks: []admissionregistrationv1.ValidatingWebhook{ + { + AdmissionReviewVersions: []string{"v1beta1", "v1"}, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Name: config.ServiceName.Name, + Namespace: config.ServiceName.Namespace, + Port: &servicePort, + Path: &logPipelinePath, + }, + CABundle: certificate, + }, + FailurePolicy: &failurePolicy, + MatchPolicy: &matchPolicy, + Name: "validation.logpipelines.telemetry.kyma-project.io", + SideEffects: &sideEffects, + TimeoutSeconds: &timeout, + Rules: []admissionregistrationv1.RuleWithOperations{ + { + Operations: operations, + Rule: admissionregistrationv1.Rule{ + APIGroups: apiGroups, + APIVersions: apiVersions, + Scope: &scope, + Resources: []string{"logpipelines"}, + }, + }, + }, + }, + { + AdmissionReviewVersions: []string{"v1beta1", "v1"}, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Name: config.ServiceName.Name, + Namespace: config.ServiceName.Namespace, + Port: &servicePort, + Path: &logParserPath, + }, + CABundle: certificate, + }, + FailurePolicy: &failurePolicy, + MatchPolicy: &matchPolicy, + Name: "validation.logparsers.telemetry.kyma-project.io", + SideEffects: &sideEffects, + TimeoutSeconds: &timeout, + Rules: []admissionregistrationv1.RuleWithOperations{ + { + Operations: operations, + Rule: admissionregistrationv1.Rule{ + APIGroups: apiGroups, + APIVersions: apiVersions, + Scope: &scope, + Resources: []string{"logparsers"}, + }, + }, + }, + }, + }, + } +} + +func patchConversionWebhookConfig(ctx context.Context, c client.Client, caBundle []byte, config Config) error { + var logPipelineCRD apiextensionsv1.CustomResourceDefinition + if err := c.Get(ctx, types.NamespacedName{Name: "logpipelines.telemetry.kyma-project.io"}, &logPipelineCRD); err != nil { + return fmt.Errorf("failed to get logpipelines CRD: %w", err) + } + + patch := client.MergeFrom(logPipelineCRD.DeepCopy()) + + conversion := logPipelineCRD.Spec.Conversion + if conversion != nil && conversion.Webhook != nil && conversion.Webhook.ClientConfig != nil { + conversion.Webhook.ClientConfig.CABundle = caBundle + } + + return c.Patch(ctx, &logPipelineCRD, patch) +} diff --git a/internal/webhookcert/webhook_cert.go b/internal/webhookcert/webhook_cert.go index 4ab1347de..bfeb554ee 100644 --- a/internal/webhookcert/webhook_cert.go +++ b/internal/webhookcert/webhook_cert.go @@ -4,12 +4,8 @@ import ( "context" "fmt" - admissionregistrationv1 "k8s.io/api/admissionregistration/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/kyma-project/telemetry-manager/internal/k8sutils" ) type Config struct { @@ -36,8 +32,7 @@ func EnsureCertificate(ctx context.Context, client client.Client, config Config) return fmt.Errorf("failed to provider server cert/key: %w", err) } - validatingWebhookConfig := makeValidatingWebhookConfig(caCertPEM, config) - return k8sutils.CreateOrUpdateValidatingWebhookConfiguration(ctx, client, &validatingWebhookConfig) + return ensureLogPipelineWebhookConfigs(ctx, client, caCertPEM, config) } func dnsNames(webhookService types.NamespacedName) (host string, alternativeDNSNames []string) { @@ -49,92 +44,3 @@ func dnsNames(webhookService types.NamespacedName) (host string, alternativeDNSN } return } - -func makeValidatingWebhookConfig(certificate []byte, config Config) admissionregistrationv1.ValidatingWebhookConfiguration { - logPipelinePath := "/validate-logpipeline" - logParserPath := "/validate-logparser" - failurePolicy := admissionregistrationv1.Fail - matchPolicy := admissionregistrationv1.Exact - sideEffects := admissionregistrationv1.SideEffectClassNone - operations := []admissionregistrationv1.OperationType{ - admissionregistrationv1.Create, - admissionregistrationv1.Update, - } - apiGroups := []string{"telemetry.kyma-project.io"} - apiVersions := []string{"v1alpha1"} - scope := admissionregistrationv1.AllScopes - servicePort := int32(443) - timeout := int32(15) - labels := map[string]string{ - "control-plane": "telemetry-manager", - "app.kubernetes.io/instance": "telemetry", - "app.kubernetes.io/name": "manager", - "kyma-project.io/component": "controller", - } - - return admissionregistrationv1.ValidatingWebhookConfiguration{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: config.WebhookName.Name, - Labels: labels, - }, - Webhooks: []admissionregistrationv1.ValidatingWebhook{ - { - AdmissionReviewVersions: []string{"v1beta1", "v1"}, - ClientConfig: admissionregistrationv1.WebhookClientConfig{ - Service: &admissionregistrationv1.ServiceReference{ - Name: config.ServiceName.Name, - Namespace: config.ServiceName.Namespace, - Port: &servicePort, - Path: &logPipelinePath, - }, - CABundle: certificate, - }, - FailurePolicy: &failurePolicy, - MatchPolicy: &matchPolicy, - Name: "validation.logpipelines.telemetry.kyma-project.io", - SideEffects: &sideEffects, - TimeoutSeconds: &timeout, - Rules: []admissionregistrationv1.RuleWithOperations{ - { - Operations: operations, - Rule: admissionregistrationv1.Rule{ - APIGroups: apiGroups, - APIVersions: apiVersions, - Scope: &scope, - Resources: []string{"logpipelines"}, - }, - }, - }, - }, - { - AdmissionReviewVersions: []string{"v1beta1", "v1"}, - ClientConfig: admissionregistrationv1.WebhookClientConfig{ - Service: &admissionregistrationv1.ServiceReference{ - Name: config.ServiceName.Name, - Namespace: config.ServiceName.Namespace, - Port: &servicePort, - Path: &logParserPath, - }, - CABundle: certificate, - }, - FailurePolicy: &failurePolicy, - MatchPolicy: &matchPolicy, - Name: "validation.logparsers.telemetry.kyma-project.io", - SideEffects: &sideEffects, - TimeoutSeconds: &timeout, - Rules: []admissionregistrationv1.RuleWithOperations{ - { - Operations: operations, - Rule: admissionregistrationv1.Rule{ - APIGroups: apiGroups, - APIVersions: apiVersions, - Scope: &scope, - Resources: []string{"logparsers"}, - }, - }, - }, - }, - }, - } -} From 93d8226a65a4e35e47c12b41ae36a78daed9bfcc Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Fri, 20 Sep 2024 11:13:30 +0200 Subject: [PATCH 09/35] Fix conversion --- apis/telemetry/v1alpha1/logpipeline_conversion.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apis/telemetry/v1alpha1/logpipeline_conversion.go b/apis/telemetry/v1alpha1/logpipeline_conversion.go index 68b04500c..f466cedec 100644 --- a/apis/telemetry/v1alpha1/logpipeline_conversion.go +++ b/apis/telemetry/v1alpha1/logpipeline_conversion.go @@ -9,6 +9,7 @@ import ( // ConvertTo converts this LogPipeline to the Hub version (v1beta1). func (src *LogPipeline) ConvertTo(dstRaw conversion.Hub) error { dst := dstRaw.(*telemetryv1beta1.LogPipeline) + dst.ObjectMeta = src.ObjectMeta srcAppInput := src.Spec.Input.Application dst.Spec.Input = telemetryv1beta1.LogPipelineInput{ @@ -47,6 +48,8 @@ func (src *LogPipeline) ConvertTo(dstRaw conversion.Hub) error { dst.Spec.Output.Custom = srcCustomOutput } + dst.Status = telemetryv1beta1.LogPipelineStatus(src.Status) + return nil } @@ -91,6 +94,7 @@ func v1Alpha1TLSToV1Beta1(src TLSConfig) telemetryv1beta1.LogPipelineHTTPOutputT // ConvertFrom converts from the Hub version (v1beta1) to this version. func (dst *LogPipeline) ConvertFrom(srcRaw conversion.Hub) error { src := srcRaw.(*telemetryv1beta1.LogPipeline) + dst.ObjectMeta = src.ObjectMeta if srcAppInput := src.Spec.Input.Runtime; srcAppInput != nil { dst.Spec.Input.Application = ApplicationInput{ @@ -128,6 +132,8 @@ func (dst *LogPipeline) ConvertFrom(srcRaw conversion.Hub) error { dst.Spec.Output.Custom = srcCustomOutput } + dst.Status = LogPipelineStatus(src.Status) + return nil } From f18d675eea721242fc75492182961a9fb4256305 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Sun, 22 Sep 2024 10:11:02 +0200 Subject: [PATCH 10/35] Add feature flag --- config/development/kustomization.yaml | 3 +++ main.go | 35 +++++++++++++++++---------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/config/development/kustomization.yaml b/config/development/kustomization.yaml index a3fcc1aa7..a6d2a3520 100644 --- a/config/development/kustomization.yaml +++ b/config/development/kustomization.yaml @@ -24,6 +24,9 @@ patches: - op: add path: /spec/template/spec/containers/0/args/- value: --kyma-input-allowed=true + - op: add + path: /spec/template/spec/containers/0/args/- + value: --enable-v1beta1-logpipelines=true target: kind: Deployment name: manager diff --git a/main.go b/main.go index 6f74c09e7..b65396a75 100644 --- a/main.go +++ b/main.go @@ -122,7 +122,8 @@ var ( selfMonitorImage string selfMonitorPriorityClass string - kymaInputAllowed bool + kymaInputAllowed bool + enableV1Beta1LogPipelines bool version = "main" ) @@ -146,7 +147,9 @@ func init() { utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) utilruntime.Must(telemetryv1alpha1.AddToScheme(scheme)) - utilruntime.Must(telemetryv1beta1.AddToScheme(scheme)) + if enableV1Beta1LogPipelines { + utilruntime.Must(telemetryv1beta1.AddToScheme(scheme)) + } utilruntime.Must(operatorv1alpha1.AddToScheme(scheme)) utilruntime.Must(istiosecurityclientv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme @@ -280,6 +283,7 @@ func main() { flag.StringVar(&selfMonitorPriorityClass, "self-monitor-priority-class", "", "Priority class name for self-monitor") flag.BoolVar(&kymaInputAllowed, "kyma-input-allowed", false, "Allow collecting status metrics for Kyma Telemetry module") + flag.BoolVar(&enableV1Beta1LogPipelines, "enable-v1beta1-log-pipelines", false, "Enable v1beta1 log pipelines CRD") flag.Parse() if err := validateFlags(); err != nil { @@ -428,18 +432,23 @@ func enableLoggingController(mgr manager.Manager, reconcileTriggerChan <-chan ev mgr.GetWebhookServer().Register("/validate-logpipeline", &webhook.Admission{Handler: createLogPipelineValidator(mgr.GetClient())}) mgr.GetWebhookServer().Register("/validate-logparser", &webhook.Admission{Handler: createLogParserValidator(mgr.GetClient())}) - if err := ctrl.NewWebhookManagedBy(mgr). - For(&telemetryv1alpha1.LogPipeline{}). - Complete(); err != nil { - setupLog.Error(err, "Failed to create v1alpha1 conversion webhook", "webhook", "LogPipeline") - os.Exit(1) - } + if enableV1Beta1LogPipelines { + setupLog.Info("Registering conversion webhooks for LogPipelines") - if err := ctrl.NewWebhookManagedBy(mgr). - For(&telemetryv1beta1.LogPipeline{}). - Complete(); err != nil { - setupLog.Error(err, "Failed to create v1beta1 conversion webhook", "webhook", "LogPipeline") - os.Exit(1) + // Register conversion webhooks for LogPipelines + if err := ctrl.NewWebhookManagedBy(mgr). + For(&telemetryv1alpha1.LogPipeline{}). + Complete(); err != nil { + setupLog.Error(err, "Failed to create v1alpha1 conversion webhook", "webhook", "LogPipeline") + os.Exit(1) + } + + if err := ctrl.NewWebhookManagedBy(mgr). + For(&telemetryv1beta1.LogPipeline{}). + Complete(); err != nil { + setupLog.Error(err, "Failed to create v1beta1 conversion webhook", "webhook", "LogPipeline") + os.Exit(1) + } } logPipelineController, err := telemetrycontrollers.NewLogPipelineController( From 23e3bc63b7f7e5150b109816cab30f3324e7977f Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Sun, 22 Sep 2024 10:26:59 +0200 Subject: [PATCH 11/35] Fix --- config/development/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/development/kustomization.yaml b/config/development/kustomization.yaml index a6d2a3520..60f97b146 100644 --- a/config/development/kustomization.yaml +++ b/config/development/kustomization.yaml @@ -26,7 +26,7 @@ patches: value: --kyma-input-allowed=true - op: add path: /spec/template/spec/containers/0/args/- - value: --enable-v1beta1-logpipelines=true + value: --enable-v1beta1-log-pipelines=true target: kind: Deployment name: manager From 34abeb45c3481d41402e7a6ac7ec33b287f2085f Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Sun, 22 Sep 2024 10:33:04 +0200 Subject: [PATCH 12/35] Fix --- main.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/main.go b/main.go index b65396a75..e9c67a025 100644 --- a/main.go +++ b/main.go @@ -147,9 +147,6 @@ func init() { utilruntime.Must(apiextensionsv1.AddToScheme(scheme)) utilruntime.Must(telemetryv1alpha1.AddToScheme(scheme)) - if enableV1Beta1LogPipelines { - utilruntime.Must(telemetryv1beta1.AddToScheme(scheme)) - } utilruntime.Must(operatorv1alpha1.AddToScheme(scheme)) utilruntime.Must(istiosecurityclientv1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme @@ -434,7 +431,7 @@ func enableLoggingController(mgr manager.Manager, reconcileTriggerChan <-chan ev if enableV1Beta1LogPipelines { setupLog.Info("Registering conversion webhooks for LogPipelines") - + utilruntime.Must(telemetryv1beta1.AddToScheme(scheme)) // Register conversion webhooks for LogPipelines if err := ctrl.NewWebhookManagedBy(mgr). For(&telemetryv1alpha1.LogPipeline{}). From 7e178bcf886cc4c17725769298a26bda760383f3 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Sun, 22 Sep 2024 11:16:24 +0200 Subject: [PATCH 13/35] Add basic unit test --- .../v1alpha1/logpipeline_conversion_test.go | 154 ++++++++++++++++++ ...onversion.go => logpipeline_conversion.go} | 0 2 files changed, 154 insertions(+) create mode 100644 apis/telemetry/v1alpha1/logpipeline_conversion_test.go rename apis/telemetry/v1beta1/{lgpipeline_conversion.go => logpipeline_conversion.go} (100%) diff --git a/apis/telemetry/v1alpha1/logpipeline_conversion_test.go b/apis/telemetry/v1alpha1/logpipeline_conversion_test.go new file mode 100644 index 000000000..d1507f5b4 --- /dev/null +++ b/apis/telemetry/v1alpha1/logpipeline_conversion_test.go @@ -0,0 +1,154 @@ +package v1alpha1 + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kyma-project/telemetry-manager/apis/telemetry/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" +) + +// TestConvertTo tests the ConvertTo method of the LogPipeline +func TestConvertTo(t *testing.T) { + src := &LogPipeline{ + ObjectMeta: metav1.ObjectMeta{ + Name: "log-pipeline-test", + }, + Spec: LogPipelineSpec{ + Input: Input{ + Application: ApplicationInput{ + Namespaces: InputNamespaces{ + Include: []string{"default", "kube-system"}, + Exclude: []string{"kube-public"}, + System: true, + }, + Containers: InputContainers{ + Include: []string{"nginx", "app"}, + Exclude: []string{"sidecar"}, + }, + KeepAnnotations: true, + DropLabels: true, + KeepOriginalBody: ptr.To(true), + }, + }, + Output: Output{ + Custom: "custom-output", + }, + }, + Status: LogPipelineStatus{ + Conditions: []metav1.Condition{ + { + Type: "LogAgentHealthy", + Status: "True", + Reason: "FluentBitReady", + Message: "FluentBit is and collecting logs", + }, + }, + UnsupportedMode: ptr.To(true), + }, + } + + dst := &v1beta1.LogPipeline{} + + err := src.ConvertTo(dst) + require.NoError(t, err, "expected no error during ConvertTo") + + require.Equal(t, src.ObjectMeta, dst.ObjectMeta) + + srcAppInput := src.Spec.Input.Application + dstRuntimeInput := dst.Spec.Input.Runtime + + require.Equal(t, srcAppInput.Namespaces.Include, dstRuntimeInput.Namespaces.Include, "included namespaces mismatch") + require.Equal(t, srcAppInput.Namespaces.Exclude, dstRuntimeInput.Namespaces.Exclude, "excluded namespaces mismatch") + require.Equal(t, srcAppInput.Namespaces.System, dstRuntimeInput.Namespaces.System, "system namespaces mismatch") + require.Equal(t, srcAppInput.Containers.Include, dstRuntimeInput.Containers.Include, "included containers mismatch") + require.Equal(t, srcAppInput.Containers.Exclude, dstRuntimeInput.Containers.Exclude, "excluded containers mismatch") + require.Equal(t, srcAppInput.KeepAnnotations, dstRuntimeInput.KeepAnnotations, "keep annotations mismatch") + require.Equal(t, srcAppInput.DropLabels, dstRuntimeInput.DropLabels, "drop labels mismatch") + require.Equal(t, srcAppInput.KeepOriginalBody, dstRuntimeInput.KeepOriginalBody, "keep original body mismatch") + + require.Equal(t, src.Spec.Output.Custom, dst.Spec.Output.Custom, "custom output mismatch") + require.Equal(t, src.Status.UnsupportedMode, dst.Status.UnsupportedMode, "status unsupported mode mismatch") +} + +// // TestConvertFrom tests the ConvertFrom method of the LogPipeline +// func TestConvertFrom(t *testing.T) { +// src := &v1beta1.LogPipeline{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "log-pipeline-test", +// }, +// Spec: v1beta1.LogPipelineSpec{ +// Input: v1beta1.LogPipelineInput{ +// Runtime: &v1beta1.LogPipelineRuntimeInput{ +// Namespaces: []string{"default", "kube-system"}, +// Containers: []string{"nginx", "app"}, +// }, +// }, +// Output: v1beta1.LogPipelineOutput{ +// Custom: "custom-output", +// }, +// }, +// Status: v1beta1.LogPipelineStatus("Running"), +// } +// +// dst := &LogPipeline{} +// +// err := dst.ConvertFrom(src) +// require.NoError(t, err, "expected no error during ConvertFrom") +// +// require.Equal(t, src.ObjectMeta, dst.ObjectMeta, "metadata mismatch") +// require.Equal(t, src.Spec.Input.Runtime.Namespaces, dst.Spec.Input.Application.Namespaces, "input namespaces mismatch") +// require.Equal(t, src.Spec.Output.Custom, dst.Spec.Output.Custom, "custom output mismatch") +// require.Equal(t, string(src.Status), string(dst.Status), "status mismatch") +// } +// +// // TestRoundTripConversion tests round-trip conversion between and v1beta1 +// func TestRoundTripConversion(t *testing.T) { +// original := &LogPipeline{ +// ObjectMeta: metav1.ObjectMeta{ +// Name: "log-pipeline-test", +// }, +// Spec: LogPipelineSpec{ +// Input: LogPipelineInputSpec{ +// Application: ApplicationInput{ +// Namespaces: []string{"default", "kube-system"}, +// Containers: []string{"nginx", "app"}, +// KeepAnnotations: true, +// DropLabels: true, +// KeepOriginalBody: true, +// }, +// }, +// Files: []FileMount{"file1", "file2"}, +// Filters: []Filter{"filter1", "filter2"}, +// Output: LogPipelineOutputSpec{ +// Custom: "custom-output", +// HTTP: &HTTPOutput{ +// Host: ValueType{Value: "http://localhost"}, +// Port: 8080, +// Compress: true, +// }, +// }, +// }, +// Status: LogPipelineStatus("Running"), +// } +// +// v1beta := &v1beta1.LogPipeline{} +// v1alpha := &LogPipeline{} +// +// // Convert original -> v1beta1 +// err := original.ConvertTo(v1beta) +// require.NoError(t, err, "expected no error during ConvertTo") +// +// // Convert back v1beta1 -> v1alpha1 +// err = v1alpha.ConvertFrom(v1beta) +// require.NoError(t, err, "expected no error during ConvertFrom") +// +// // Assert that the original and the round-tripped version are the same +// require.Equal(t, original.ObjectMeta, v1alpha.ObjectMeta, "metadata mismatch after round-trip conversion") +// require.Equal(t, original.Spec.Input.Application.Namespaces, v1alpha.Spec.Input.Application.Namespaces, "input namespaces mismatch after round-trip conversion") +// require.Equal(t, original.Spec.Output.Custom, v1alpha.Spec.Output.Custom, "custom output mismatch after round-trip conversion") +// require.Equal(t, original.Spec.Output.HTTP.Host.Value, v1alpha.Spec.Output.HTTP.Host.Value, "HTTP host mismatch after round-trip conversion") +// require.Equal(t, string(original.Status), string(v1alpha.Status), "status mismatch after round-trip conversion") +// } diff --git a/apis/telemetry/v1beta1/lgpipeline_conversion.go b/apis/telemetry/v1beta1/logpipeline_conversion.go similarity index 100% rename from apis/telemetry/v1beta1/lgpipeline_conversion.go rename to apis/telemetry/v1beta1/logpipeline_conversion.go From da6debebca66d5c7b2d1c0cc795747f209b6b38d Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Sun, 22 Sep 2024 11:24:19 +0200 Subject: [PATCH 14/35] Add more assertions --- .../v1alpha1/logpipeline_conversion_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/apis/telemetry/v1alpha1/logpipeline_conversion_test.go b/apis/telemetry/v1alpha1/logpipeline_conversion_test.go index d1507f5b4..ac454ae3f 100644 --- a/apis/telemetry/v1alpha1/logpipeline_conversion_test.go +++ b/apis/telemetry/v1alpha1/logpipeline_conversion_test.go @@ -33,6 +33,12 @@ func TestConvertTo(t *testing.T) { KeepOriginalBody: ptr.To(true), }, }, + Files: []FileMount{ + {Name: "file1", Content: "file1-content"}, + }, + Filters: []Filter{ + {Custom: "name stdout"}, + }, Output: Output{ Custom: "custom-output", }, @@ -69,8 +75,15 @@ func TestConvertTo(t *testing.T) { require.Equal(t, srcAppInput.DropLabels, dstRuntimeInput.DropLabels, "drop labels mismatch") require.Equal(t, srcAppInput.KeepOriginalBody, dstRuntimeInput.KeepOriginalBody, "keep original body mismatch") + require.Len(t, dst.Spec.Files, 1, "expected one file") + require.Equal(t, src.Spec.Files[0].Name, dst.Spec.Files[0].Name, "file name mismatch") + + require.Len(t, dst.Spec.Filters, 1, "expected one filter") + require.Equal(t, src.Spec.Filters[0].Custom, dst.Spec.Filters[0].Custom, "custom filter mismatch") + require.Equal(t, src.Spec.Output.Custom, dst.Spec.Output.Custom, "custom output mismatch") require.Equal(t, src.Status.UnsupportedMode, dst.Status.UnsupportedMode, "status unsupported mode mismatch") + require.ElementsMatch(t, src.Status.Conditions, dst.Status.Conditions, "status conditions mismatch") } // // TestConvertFrom tests the ConvertFrom method of the LogPipeline From dc5585a5f61513a01237a8591f468c22c07c7b7e Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Sun, 22 Sep 2024 12:52:58 +0200 Subject: [PATCH 15/35] Add even more assertions --- .../v1alpha1/logpipeline_conversion_test.go | 52 ++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/apis/telemetry/v1alpha1/logpipeline_conversion_test.go b/apis/telemetry/v1alpha1/logpipeline_conversion_test.go index ac454ae3f..2c157121b 100644 --- a/apis/telemetry/v1alpha1/logpipeline_conversion_test.go +++ b/apis/telemetry/v1alpha1/logpipeline_conversion_test.go @@ -41,6 +41,40 @@ func TestConvertTo(t *testing.T) { }, Output: Output{ Custom: "custom-output", + HTTP: &HTTPOutput{ + Host: ValueType{ + Value: "http://localhost", + }, + User: ValueType{ + Value: "user", + }, + Password: ValueType{ + ValueFrom: &ValueFromSource{ + SecretKeyRef: &SecretKeyRef{ + Name: "secret-name", + Namespace: "secret-namespace", + Key: "secret-key", + }, + }, + }, + URI: "/ingest/v1beta1/logs", + Port: "8080", + Compress: "on", + Format: "json", + TLSConfig: TLSConfig{ + SkipCertificateValidation: true, + CA: &ValueType{ + Value: "ca", + }, + Cert: &ValueType{ + Value: "cert", + }, + Key: &ValueType{ + Value: "key", + }, + }, + Dedot: true, + }, }, }, Status: LogPipelineStatus{ @@ -65,7 +99,6 @@ func TestConvertTo(t *testing.T) { srcAppInput := src.Spec.Input.Application dstRuntimeInput := dst.Spec.Input.Runtime - require.Equal(t, srcAppInput.Namespaces.Include, dstRuntimeInput.Namespaces.Include, "included namespaces mismatch") require.Equal(t, srcAppInput.Namespaces.Exclude, dstRuntimeInput.Namespaces.Exclude, "excluded namespaces mismatch") require.Equal(t, srcAppInput.Namespaces.System, dstRuntimeInput.Namespaces.System, "system namespaces mismatch") @@ -82,6 +115,23 @@ func TestConvertTo(t *testing.T) { require.Equal(t, src.Spec.Filters[0].Custom, dst.Spec.Filters[0].Custom, "custom filter mismatch") require.Equal(t, src.Spec.Output.Custom, dst.Spec.Output.Custom, "custom output mismatch") + + srcHTTP := src.Spec.Output.HTTP + dstHTTP := dst.Spec.Output.HTTP + require.Equal(t, srcHTTP.Host.Value, dstHTTP.Host.Value, "HTTP host mismatch") + require.Equal(t, srcHTTP.User.Value, dstHTTP.User.Value, "HTTP user mismatch") + require.Equal(t, srcHTTP.Password.ValueFrom.SecretKeyRef.Name, dstHTTP.Password.ValueFrom.SecretKeyRef.Name, "HTTP password secret name mismatch") + require.Equal(t, srcHTTP.Password.ValueFrom.SecretKeyRef.Namespace, dstHTTP.Password.ValueFrom.SecretKeyRef.Namespace, "HTTP password secret namespace mismatch") + require.Equal(t, srcHTTP.Password.ValueFrom.SecretKeyRef.Key, dstHTTP.Password.ValueFrom.SecretKeyRef.Key, "HTTP password secret key mismatch") + require.Equal(t, srcHTTP.URI, dstHTTP.URI, "HTTP URI mismatch") + require.Equal(t, srcHTTP.Port, dstHTTP.Port, "HTTP port mismatch") + require.Equal(t, srcHTTP.Compress, dstHTTP.Compress, "HTTP compress mismatch") + require.Equal(t, srcHTTP.Format, dstHTTP.Format, "HTTP format mismatch") + require.Equal(t, srcHTTP.TLSConfig.SkipCertificateValidation, dstHTTP.TLSConfig.SkipCertificateValidation, "HTTP TLS skip certificate validation mismatch") + require.Equal(t, srcHTTP.TLSConfig.CA.Value, dstHTTP.TLSConfig.CA.Value, "HTTP TLS CA mismatch") + require.Equal(t, srcHTTP.TLSConfig.Cert.Value, dstHTTP.TLSConfig.Cert.Value, "HTTP TLS cert mismatch") + require.Equal(t, srcHTTP.TLSConfig.Key.Value, dstHTTP.TLSConfig.Key.Value, "HTTP TLS key mismatch") + require.Equal(t, src.Status.UnsupportedMode, dst.Status.UnsupportedMode, "status unsupported mode mismatch") require.ElementsMatch(t, src.Status.Conditions, dst.Status.Conditions, "status conditions mismatch") } From 402c74c04e4fd0ed956d4400ada08bd479b28921 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Sun, 22 Sep 2024 13:12:49 +0200 Subject: [PATCH 16/35] Implement ConvertFrom --- .../v1alpha1/logpipeline_conversion_test.go | 207 +++++++++++------- 1 file changed, 133 insertions(+), 74 deletions(-) diff --git a/apis/telemetry/v1alpha1/logpipeline_conversion_test.go b/apis/telemetry/v1alpha1/logpipeline_conversion_test.go index 2c157121b..20b582a2d 100644 --- a/apis/telemetry/v1alpha1/logpipeline_conversion_test.go +++ b/apis/telemetry/v1alpha1/logpipeline_conversion_test.go @@ -4,13 +4,12 @@ import ( "testing" "github.com/stretchr/testify/require" - - "github.com/kyma-project/telemetry-manager/apis/telemetry/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" + + telemetryv1beta1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1beta1" ) -// TestConvertTo tests the ConvertTo method of the LogPipeline func TestConvertTo(t *testing.T) { src := &LogPipeline{ ObjectMeta: metav1.ObjectMeta{ @@ -90,83 +89,143 @@ func TestConvertTo(t *testing.T) { }, } - dst := &v1beta1.LogPipeline{} + dst := &telemetryv1beta1.LogPipeline{} err := src.ConvertTo(dst) require.NoError(t, err, "expected no error during ConvertTo") - require.Equal(t, src.ObjectMeta, dst.ObjectMeta) - - srcAppInput := src.Spec.Input.Application - dstRuntimeInput := dst.Spec.Input.Runtime - require.Equal(t, srcAppInput.Namespaces.Include, dstRuntimeInput.Namespaces.Include, "included namespaces mismatch") - require.Equal(t, srcAppInput.Namespaces.Exclude, dstRuntimeInput.Namespaces.Exclude, "excluded namespaces mismatch") - require.Equal(t, srcAppInput.Namespaces.System, dstRuntimeInput.Namespaces.System, "system namespaces mismatch") - require.Equal(t, srcAppInput.Containers.Include, dstRuntimeInput.Containers.Include, "included containers mismatch") - require.Equal(t, srcAppInput.Containers.Exclude, dstRuntimeInput.Containers.Exclude, "excluded containers mismatch") - require.Equal(t, srcAppInput.KeepAnnotations, dstRuntimeInput.KeepAnnotations, "keep annotations mismatch") - require.Equal(t, srcAppInput.DropLabels, dstRuntimeInput.DropLabels, "drop labels mismatch") - require.Equal(t, srcAppInput.KeepOriginalBody, dstRuntimeInput.KeepOriginalBody, "keep original body mismatch") - - require.Len(t, dst.Spec.Files, 1, "expected one file") - require.Equal(t, src.Spec.Files[0].Name, dst.Spec.Files[0].Name, "file name mismatch") - - require.Len(t, dst.Spec.Filters, 1, "expected one filter") - require.Equal(t, src.Spec.Filters[0].Custom, dst.Spec.Filters[0].Custom, "custom filter mismatch") - - require.Equal(t, src.Spec.Output.Custom, dst.Spec.Output.Custom, "custom output mismatch") - - srcHTTP := src.Spec.Output.HTTP - dstHTTP := dst.Spec.Output.HTTP - require.Equal(t, srcHTTP.Host.Value, dstHTTP.Host.Value, "HTTP host mismatch") - require.Equal(t, srcHTTP.User.Value, dstHTTP.User.Value, "HTTP user mismatch") - require.Equal(t, srcHTTP.Password.ValueFrom.SecretKeyRef.Name, dstHTTP.Password.ValueFrom.SecretKeyRef.Name, "HTTP password secret name mismatch") - require.Equal(t, srcHTTP.Password.ValueFrom.SecretKeyRef.Namespace, dstHTTP.Password.ValueFrom.SecretKeyRef.Namespace, "HTTP password secret namespace mismatch") - require.Equal(t, srcHTTP.Password.ValueFrom.SecretKeyRef.Key, dstHTTP.Password.ValueFrom.SecretKeyRef.Key, "HTTP password secret key mismatch") - require.Equal(t, srcHTTP.URI, dstHTTP.URI, "HTTP URI mismatch") - require.Equal(t, srcHTTP.Port, dstHTTP.Port, "HTTP port mismatch") - require.Equal(t, srcHTTP.Compress, dstHTTP.Compress, "HTTP compress mismatch") - require.Equal(t, srcHTTP.Format, dstHTTP.Format, "HTTP format mismatch") - require.Equal(t, srcHTTP.TLSConfig.SkipCertificateValidation, dstHTTP.TLSConfig.SkipCertificateValidation, "HTTP TLS skip certificate validation mismatch") - require.Equal(t, srcHTTP.TLSConfig.CA.Value, dstHTTP.TLSConfig.CA.Value, "HTTP TLS CA mismatch") - require.Equal(t, srcHTTP.TLSConfig.Cert.Value, dstHTTP.TLSConfig.Cert.Value, "HTTP TLS cert mismatch") - require.Equal(t, srcHTTP.TLSConfig.Key.Value, dstHTTP.TLSConfig.Key.Value, "HTTP TLS key mismatch") - - require.Equal(t, src.Status.UnsupportedMode, dst.Status.UnsupportedMode, "status unsupported mode mismatch") - require.ElementsMatch(t, src.Status.Conditions, dst.Status.Conditions, "status conditions mismatch") + requireLogPipelinesEquivalent(t, src, dst) +} + +func TestConvertFrom(t *testing.T) { + src := &telemetryv1beta1.LogPipeline{ + ObjectMeta: metav1.ObjectMeta{ + Name: "log-pipeline-test", + }, + Spec: telemetryv1beta1.LogPipelineSpec{ + Input: telemetryv1beta1.LogPipelineInput{ + Runtime: &telemetryv1beta1.LogPipelineRuntimeInput{ + Namespaces: telemetryv1beta1.LogPipelineInputNamespaces{ + Include: []string{"default", "kube-system"}, + Exclude: []string{"kube-public"}, + System: true, + }, + Containers: telemetryv1beta1.LogPipelineInputContainers{ + Include: []string{"nginx", "app"}, + Exclude: []string{"sidecar"}, + }, + KeepAnnotations: true, + DropLabels: true, + KeepOriginalBody: ptr.To(true), + }, + }, + Files: []telemetryv1beta1.LogPipelineFileMount{ + {Name: "file1", Content: "file1-content"}, + }, + Filters: []telemetryv1beta1.LogPipelineFilter{ + {Custom: "name stdout"}, + }, + Output: telemetryv1beta1.LogPipelineOutput{ + Custom: "custom-output", + HTTP: &telemetryv1beta1.LogPipelineHTTPOutput{ + Host: telemetryv1beta1.ValueType{ + Value: "http://localhost", + }, + User: telemetryv1beta1.ValueType{ + Value: "user", + }, + Password: telemetryv1beta1.ValueType{ + ValueFrom: &telemetryv1beta1.ValueFromSource{ + SecretKeyRef: &telemetryv1beta1.SecretKeyRef{ + Name: "secret-name", + Namespace: "secret-namespace", + Key: "secret-key", + }, + }, + }, + URI: "/ingest/v1beta1/logs", + Port: "8080", + Compress: "on", + Format: "json", + TLSConfig: telemetryv1beta1.LogPipelineHTTPOutputTLS{ + SkipCertificateValidation: true, + CA: &telemetryv1beta1.ValueType{ + Value: "ca", + }, + Cert: &telemetryv1beta1.ValueType{ + Value: "cert", + }, + Key: &telemetryv1beta1.ValueType{ + Value: "key", + }, + }, + Dedot: true, + }, + }, + }, + Status: telemetryv1beta1.LogPipelineStatus{ + Conditions: []metav1.Condition{ + { + Type: "LogAgentHealthy", + Status: "True", + Reason: "FluentBitReady", + Message: "FluentBit is and collecting logs", + }, + }, + UnsupportedMode: ptr.To(true), + }, + } + + dst := &LogPipeline{} + + err := dst.ConvertFrom(src) + require.NoError(t, err, "expected no error during ConvertTo") + + requireLogPipelinesEquivalent(t, dst, src) +} + +func requireLogPipelinesEquivalent(t *testing.T, x *LogPipeline, y *telemetryv1beta1.LogPipeline) { + require.Equal(t, x.ObjectMeta, y.ObjectMeta) + + xAppInput := x.Spec.Input.Application + yRuntimeInput := y.Spec.Input.Runtime + require.Equal(t, xAppInput.Namespaces.Include, yRuntimeInput.Namespaces.Include, "included namespaces mismatch") + require.Equal(t, xAppInput.Namespaces.Exclude, yRuntimeInput.Namespaces.Exclude, "excluded namespaces mismatch") + require.Equal(t, xAppInput.Namespaces.System, yRuntimeInput.Namespaces.System, "system namespaces mismatch") + require.Equal(t, xAppInput.Containers.Include, yRuntimeInput.Containers.Include, "included containers mismatch") + require.Equal(t, xAppInput.Containers.Exclude, yRuntimeInput.Containers.Exclude, "excluded containers mismatch") + require.Equal(t, xAppInput.KeepAnnotations, yRuntimeInput.KeepAnnotations, "keep annotations mismatch") + require.Equal(t, xAppInput.DropLabels, yRuntimeInput.DropLabels, "drop labels mismatch") + require.Equal(t, xAppInput.KeepOriginalBody, yRuntimeInput.KeepOriginalBody, "keep original body mismatch") + + require.Len(t, y.Spec.Files, 1, "expected one file") + require.Equal(t, x.Spec.Files[0].Name, y.Spec.Files[0].Name, "file name mismatch") + + require.Len(t, y.Spec.Filters, 1, "expected one filter") + require.Equal(t, x.Spec.Filters[0].Custom, y.Spec.Filters[0].Custom, "custom filter mismatch") + + require.Equal(t, x.Spec.Output.Custom, y.Spec.Output.Custom, "custom output mismatch") + + xHTTP := x.Spec.Output.HTTP + yHTTP := y.Spec.Output.HTTP + require.Equal(t, xHTTP.Host.Value, yHTTP.Host.Value, "HTTP host mismatch") + require.Equal(t, xHTTP.User.Value, yHTTP.User.Value, "HTTP user mismatch") + require.Equal(t, xHTTP.Password.ValueFrom.SecretKeyRef.Name, yHTTP.Password.ValueFrom.SecretKeyRef.Name, "HTTP password secret name mismatch") + require.Equal(t, xHTTP.Password.ValueFrom.SecretKeyRef.Namespace, yHTTP.Password.ValueFrom.SecretKeyRef.Namespace, "HTTP password secret namespace mismatch") + require.Equal(t, xHTTP.Password.ValueFrom.SecretKeyRef.Key, yHTTP.Password.ValueFrom.SecretKeyRef.Key, "HTTP password secret key mismatch") + require.Equal(t, xHTTP.URI, yHTTP.URI, "HTTP URI mismatch") + require.Equal(t, xHTTP.Port, yHTTP.Port, "HTTP port mismatch") + require.Equal(t, xHTTP.Compress, yHTTP.Compress, "HTTP compress mismatch") + require.Equal(t, xHTTP.Format, yHTTP.Format, "HTTP format mismatch") + require.Equal(t, xHTTP.TLSConfig.SkipCertificateValidation, yHTTP.TLSConfig.SkipCertificateValidation, "HTTP TLS skip certificate validation mismatch") + require.Equal(t, xHTTP.TLSConfig.CA.Value, yHTTP.TLSConfig.CA.Value, "HTTP TLS CA mismatch") + require.Equal(t, xHTTP.TLSConfig.Cert.Value, yHTTP.TLSConfig.Cert.Value, "HTTP TLS cert mismatch") + require.Equal(t, xHTTP.TLSConfig.Key.Value, yHTTP.TLSConfig.Key.Value, "HTTP TLS key mismatch") + + require.Equal(t, x.Status.UnsupportedMode, y.Status.UnsupportedMode, "status unsupported mode mismatch") + require.ElementsMatch(t, x.Status.Conditions, y.Status.Conditions, "status conditions mismatch") } -// // TestConvertFrom tests the ConvertFrom method of the LogPipeline -// func TestConvertFrom(t *testing.T) { -// src := &v1beta1.LogPipeline{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: "log-pipeline-test", -// }, -// Spec: v1beta1.LogPipelineSpec{ -// Input: v1beta1.LogPipelineInput{ -// Runtime: &v1beta1.LogPipelineRuntimeInput{ -// Namespaces: []string{"default", "kube-system"}, -// Containers: []string{"nginx", "app"}, -// }, -// }, -// Output: v1beta1.LogPipelineOutput{ -// Custom: "custom-output", -// }, -// }, -// Status: v1beta1.LogPipelineStatus("Running"), -// } -// -// dst := &LogPipeline{} -// -// err := dst.ConvertFrom(src) -// require.NoError(t, err, "expected no error during ConvertFrom") -// -// require.Equal(t, src.ObjectMeta, dst.ObjectMeta, "metadata mismatch") -// require.Equal(t, src.Spec.Input.Runtime.Namespaces, dst.Spec.Input.Application.Namespaces, "input namespaces mismatch") -// require.Equal(t, src.Spec.Output.Custom, dst.Spec.Output.Custom, "custom output mismatch") -// require.Equal(t, string(src.Status), string(dst.Status), "status mismatch") -// } -// // // TestRoundTripConversion tests round-trip conversion between and v1beta1 // func TestRoundTripConversion(t *testing.T) { // original := &LogPipeline{ From 9887371798d4cd7345a5adbdf2ecdbaec703aa90 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Sun, 22 Sep 2024 14:16:31 +0200 Subject: [PATCH 17/35] Add round-trip tests --- .../v1alpha1/logpipeline_conversion_test.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/apis/telemetry/v1alpha1/logpipeline_conversion_test.go b/apis/telemetry/v1alpha1/logpipeline_conversion_test.go index 20b582a2d..52bd6024b 100644 --- a/apis/telemetry/v1alpha1/logpipeline_conversion_test.go +++ b/apis/telemetry/v1alpha1/logpipeline_conversion_test.go @@ -1,6 +1,7 @@ package v1alpha1 import ( + "reflect" "testing" "github.com/stretchr/testify/require" @@ -92,9 +93,15 @@ func TestConvertTo(t *testing.T) { dst := &telemetryv1beta1.LogPipeline{} err := src.ConvertTo(dst) - require.NoError(t, err, "expected no error during ConvertTo") + require.NoError(t, err) requireLogPipelinesEquivalent(t, src, dst) + + srcAfterRoundTrip := &LogPipeline{} + err = srcAfterRoundTrip.ConvertFrom(dst) + require.NoError(t, err) + + require.True(t, reflect.DeepEqual(src, srcAfterRoundTrip), "expected source and source after round-trip to be equal") } func TestConvertFrom(t *testing.T) { @@ -182,6 +189,12 @@ func TestConvertFrom(t *testing.T) { require.NoError(t, err, "expected no error during ConvertTo") requireLogPipelinesEquivalent(t, dst, src) + + srcAfterRoundTrip := &telemetryv1beta1.LogPipeline{} + err = dst.ConvertTo(srcAfterRoundTrip) + require.NoError(t, err, "expected no error during ConvertFrom (round-trip)") + + require.True(t, reflect.DeepEqual(src, srcAfterRoundTrip), "expected source and source after round-trip to be equal") } func requireLogPipelinesEquivalent(t *testing.T, x *LogPipeline, y *telemetryv1beta1.LogPipeline) { From e430a86b050376a09d6f0471472261a63a11c44a Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Sun, 22 Sep 2024 19:44:32 +0200 Subject: [PATCH 18/35] Use go-cmp --- .../v1alpha1/logpipeline_conversion_test.go | 57 +------------------ go.mod | 2 +- 2 files changed, 4 insertions(+), 55 deletions(-) diff --git a/apis/telemetry/v1alpha1/logpipeline_conversion_test.go b/apis/telemetry/v1alpha1/logpipeline_conversion_test.go index 52bd6024b..ac48db0d1 100644 --- a/apis/telemetry/v1alpha1/logpipeline_conversion_test.go +++ b/apis/telemetry/v1alpha1/logpipeline_conversion_test.go @@ -1,9 +1,9 @@ package v1alpha1 import ( - "reflect" "testing" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/ptr" @@ -100,8 +100,7 @@ func TestConvertTo(t *testing.T) { srcAfterRoundTrip := &LogPipeline{} err = srcAfterRoundTrip.ConvertFrom(dst) require.NoError(t, err) - - require.True(t, reflect.DeepEqual(src, srcAfterRoundTrip), "expected source and source after round-trip to be equal") + require.Empty(t, cmp.Diff(src, srcAfterRoundTrip), "expected source be equal to itself after round-trip") } func TestConvertFrom(t *testing.T) { @@ -193,8 +192,7 @@ func TestConvertFrom(t *testing.T) { srcAfterRoundTrip := &telemetryv1beta1.LogPipeline{} err = dst.ConvertTo(srcAfterRoundTrip) require.NoError(t, err, "expected no error during ConvertFrom (round-trip)") - - require.True(t, reflect.DeepEqual(src, srcAfterRoundTrip), "expected source and source after round-trip to be equal") + require.Empty(t, cmp.Diff(src, srcAfterRoundTrip), "expected source be equal to itself after round-trip") } func requireLogPipelinesEquivalent(t *testing.T, x *LogPipeline, y *telemetryv1beta1.LogPipeline) { @@ -238,52 +236,3 @@ func requireLogPipelinesEquivalent(t *testing.T, x *LogPipeline, y *telemetryv1b require.Equal(t, x.Status.UnsupportedMode, y.Status.UnsupportedMode, "status unsupported mode mismatch") require.ElementsMatch(t, x.Status.Conditions, y.Status.Conditions, "status conditions mismatch") } - -// // TestRoundTripConversion tests round-trip conversion between and v1beta1 -// func TestRoundTripConversion(t *testing.T) { -// original := &LogPipeline{ -// ObjectMeta: metav1.ObjectMeta{ -// Name: "log-pipeline-test", -// }, -// Spec: LogPipelineSpec{ -// Input: LogPipelineInputSpec{ -// Application: ApplicationInput{ -// Namespaces: []string{"default", "kube-system"}, -// Containers: []string{"nginx", "app"}, -// KeepAnnotations: true, -// DropLabels: true, -// KeepOriginalBody: true, -// }, -// }, -// Files: []FileMount{"file1", "file2"}, -// Filters: []Filter{"filter1", "filter2"}, -// Output: LogPipelineOutputSpec{ -// Custom: "custom-output", -// HTTP: &HTTPOutput{ -// Host: ValueType{Value: "http://localhost"}, -// Port: 8080, -// Compress: true, -// }, -// }, -// }, -// Status: LogPipelineStatus("Running"), -// } -// -// v1beta := &v1beta1.LogPipeline{} -// v1alpha := &LogPipeline{} -// -// // Convert original -> v1beta1 -// err := original.ConvertTo(v1beta) -// require.NoError(t, err, "expected no error during ConvertTo") -// -// // Convert back v1beta1 -> v1alpha1 -// err = v1alpha.ConvertFrom(v1beta) -// require.NoError(t, err, "expected no error during ConvertFrom") -// -// // Assert that the original and the round-tripped version are the same -// require.Equal(t, original.ObjectMeta, v1alpha.ObjectMeta, "metadata mismatch after round-trip conversion") -// require.Equal(t, original.Spec.Input.Application.Namespaces, v1alpha.Spec.Input.Application.Namespaces, "input namespaces mismatch after round-trip conversion") -// require.Equal(t, original.Spec.Output.Custom, v1alpha.Spec.Output.Custom, "custom output mismatch after round-trip conversion") -// require.Equal(t, original.Spec.Output.HTTP.Host.Value, v1alpha.Spec.Output.HTTP.Host.Value, "HTTP host mismatch after round-trip conversion") -// require.Equal(t, string(original.Status), string(v1alpha.Status), "status mismatch after round-trip conversion") -// } diff --git a/go.mod b/go.mod index 1848ebc58..df213d271 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23.1 require ( github.com/go-logr/logr v1.4.2 github.com/go-logr/zapr v1.3.0 + github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.20.2 github.com/onsi/gomega v1.34.2 @@ -43,7 +44,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect github.com/imdario/mergo v0.3.16 // indirect From fc18509b32d2b66dbc8d20e2ed7cf0e61b086406 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Sun, 22 Sep 2024 20:07:18 +0200 Subject: [PATCH 19/35] Fix lint --- .../v1alpha1/logpipeline_conversion.go | 25 ++++++++++++++++--- .../log_pipeline_webhook_configs.go | 7 +++--- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/apis/telemetry/v1alpha1/logpipeline_conversion.go b/apis/telemetry/v1alpha1/logpipeline_conversion.go index f466cedec..b3424886d 100644 --- a/apis/telemetry/v1alpha1/logpipeline_conversion.go +++ b/apis/telemetry/v1alpha1/logpipeline_conversion.go @@ -1,14 +1,24 @@ package v1alpha1 import ( + "errors" + "sigs.k8s.io/controller-runtime/pkg/conversion" telemetryv1beta1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1beta1" ) +var errSrcTypeUnsupported = errors.New("source type is not LogPipeline v1alpha1") +var errDstTypeUnsupported = errors.New("destination type is not LogPipeline v1beta1") + // ConvertTo converts this LogPipeline to the Hub version (v1beta1). -func (src *LogPipeline) ConvertTo(dstRaw conversion.Hub) error { - dst := dstRaw.(*telemetryv1beta1.LogPipeline) +func (lp *LogPipeline) ConvertTo(dstRaw conversion.Hub) error { + src := lp + dst, ok := dstRaw.(*telemetryv1beta1.LogPipeline) + if !ok { + return errDstTypeUnsupported + } + dst.ObjectMeta = src.ObjectMeta srcAppInput := src.Spec.Input.Application @@ -92,8 +102,15 @@ func v1Alpha1TLSToV1Beta1(src TLSConfig) telemetryv1beta1.LogPipelineHTTPOutputT } // ConvertFrom converts from the Hub version (v1beta1) to this version. -func (dst *LogPipeline) ConvertFrom(srcRaw conversion.Hub) error { - src := srcRaw.(*telemetryv1beta1.LogPipeline) +// +//lint:ignore ST1016 This is a conversion function and "dst" makes sense here. +func (lp *LogPipeline) ConvertFrom(srcRaw conversion.Hub) error { + dst := lp + src, ok := srcRaw.(*telemetryv1beta1.LogPipeline) + if !ok { + return errSrcTypeUnsupported + } + dst.ObjectMeta = src.ObjectMeta if srcAppInput := src.Spec.Input.Runtime; srcAppInput != nil { diff --git a/internal/webhookcert/log_pipeline_webhook_configs.go b/internal/webhookcert/log_pipeline_webhook_configs.go index c0c7590ce..25319f17f 100644 --- a/internal/webhookcert/log_pipeline_webhook_configs.go +++ b/internal/webhookcert/log_pipeline_webhook_configs.go @@ -4,12 +4,13 @@ import ( "context" "fmt" - "github.com/kyma-project/telemetry-manager/internal/k8sutils" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kyma-project/telemetry-manager/internal/k8sutils" ) // ensureLogPipelineWebhookConfigs creates or updates the ValidatingWebhookConfiguration for the LogPipeline resources. @@ -20,7 +21,7 @@ func ensureLogPipelineWebhookConfigs(ctx context.Context, c client.Client, caBun return fmt.Errorf("failed to create or update validating webhook configuration: %w", err) } - if err := patchConversionWebhookConfig(ctx, c, caBundle, config); err != nil { + if err := patchConversionWebhookConfig(ctx, c, caBundle); err != nil { return fmt.Errorf("failed to patch conversion webhook configuration: %w", err) } @@ -116,7 +117,7 @@ func makeValidatingWebhookConfig(certificate []byte, config Config) admissionreg } } -func patchConversionWebhookConfig(ctx context.Context, c client.Client, caBundle []byte, config Config) error { +func patchConversionWebhookConfig(ctx context.Context, c client.Client, caBundle []byte) error { var logPipelineCRD apiextensionsv1.CustomResourceDefinition if err := c.Get(ctx, types.NamespacedName{Name: "logpipelines.telemetry.kyma-project.io"}, &logPipelineCRD); err != nil { return fmt.Errorf("failed to get logpipelines CRD: %w", err) From db23599836c482208a0592f043e2eb68afbbc005 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Mon, 23 Sep 2024 08:09:42 +0200 Subject: [PATCH 20/35] Fix tests --- internal/webhookcert/webhook_cert.go | 4 +-- internal/webhookcert/webhook_cert_test.go | 41 ++++++++++++++++++++--- 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/internal/webhookcert/webhook_cert.go b/internal/webhookcert/webhook_cert.go index bfeb554ee..183e4d246 100644 --- a/internal/webhookcert/webhook_cert.go +++ b/internal/webhookcert/webhook_cert.go @@ -18,7 +18,7 @@ type Config struct { func EnsureCertificate(ctx context.Context, client client.Client, config Config) error { caCertPEM, caKeyPEM, err := newCACertProvider(client).provideCert(ctx, config.CASecretName) if err != nil { - return fmt.Errorf("failed to provider ca cert/key: %w", err) + return fmt.Errorf("failed to provide ca cert/key: %w", err) } host, alternativeDNSNames := dnsNames(config.ServiceName) @@ -29,7 +29,7 @@ func EnsureCertificate(ctx context.Context, client client.Client, config Config) caKeyPEM: caKeyPEM, }) if err != nil { - return fmt.Errorf("failed to provider server cert/key: %w", err) + return fmt.Errorf("failed to provide server cert/key: %w", err) } return ensureLogPipelineWebhookConfigs(ctx, client, caCertPEM, config) diff --git a/internal/webhookcert/webhook_cert_test.go b/internal/webhookcert/webhook_cert_test.go index 7d19e1fdd..9e731a1f9 100644 --- a/internal/webhookcert/webhook_cert_test.go +++ b/internal/webhookcert/webhook_cert_test.go @@ -9,8 +9,11 @@ import ( "github.com/stretchr/testify/require" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -34,16 +37,34 @@ var ( "app.kubernetes.io/name": "manager", "kyma-project.io/component": "controller", } + logPipelinesCRD = apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: "logpipelines.telemetry.kyma-project.io", + }, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Conversion: &apiextensionsv1.CustomResourceConversion{ + Strategy: apiextensionsv1.WebhookConverter, + Webhook: &apiextensionsv1.WebhookConversion{ + ClientConfig: &apiextensionsv1.WebhookClientConfig{}, + }, + }, + }, + } ) func TestEnsureCertificate(t *testing.T) { - client := fake.NewClientBuilder().Build() + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, apiextensionsv1.AddToScheme(scheme)) + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&logPipelinesCRD).Build() + certDir, err := os.MkdirTemp("", "certificate") require.NoError(t, err) defer func(path string) { deleteErr := os.RemoveAll(path) require.NoError(t, deleteErr) }(certDir) + config := Config{ CertDir: certDir, ServiceName: webhookService, @@ -103,6 +124,10 @@ func TestEnsureCertificate(t *testing.T) { } func TestUpdateWebhookCertificate(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, apiextensionsv1.AddToScheme(scheme)) + logPipelinePath := "/validate-logpipeline" logParserPath := "/validate-logparser" failurePolicy := admissionregistrationv1.Fail @@ -119,7 +144,7 @@ func TestUpdateWebhookCertificate(t *testing.T) { timeout := int32(15) certificate := []byte("123") - initialValidatingWebhookConfiguration := &admissionregistrationv1.ValidatingWebhookConfiguration{ + initialValidatingWebhookConfiguration := admissionregistrationv1.ValidatingWebhookConfiguration{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -185,7 +210,7 @@ func TestUpdateWebhookCertificate(t *testing.T) { }, }, } - client := fake.NewClientBuilder().WithObjects(initialValidatingWebhookConfiguration).Build() + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&logPipelinesCRD, &initialValidatingWebhookConfiguration).Build() certDir, err := os.MkdirTemp("", "certificate") require.NoError(t, err) @@ -225,7 +250,10 @@ func TestUpdateWebhookCertificate(t *testing.T) { } func TestCreateSecret(t *testing.T) { - client := fake.NewClientBuilder().Build() + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, apiextensionsv1.AddToScheme(scheme)) + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&logPipelinesCRD).Build() certDir, err := os.MkdirTemp("", "certificate") require.NoError(t, err) @@ -252,7 +280,10 @@ func TestCreateSecret(t *testing.T) { } func TestReuseExistingCertificate(t *testing.T) { - client := fake.NewClientBuilder().Build() + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, apiextensionsv1.AddToScheme(scheme)) + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&logPipelinesCRD).Build() certDir, err := os.MkdirTemp("", "certificate") require.NoError(t, err) From b0e9595faf231104c8d30234d955479126c49c6f Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Mon, 23 Sep 2024 08:23:06 +0200 Subject: [PATCH 21/35] Add more assertions --- internal/webhookcert/webhook_cert_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/webhookcert/webhook_cert_test.go b/internal/webhookcert/webhook_cert_test.go index 9e731a1f9..27cb49040 100644 --- a/internal/webhookcert/webhook_cert_test.go +++ b/internal/webhookcert/webhook_cert_test.go @@ -121,6 +121,15 @@ func TestEnsureCertificate(t *testing.T) { require.Contains(t, validatingWebhookConfiguration.Webhooks[0].Rules[0].Resources, "logpipelines") require.Contains(t, validatingWebhookConfiguration.Webhooks[1].Rules[0].Resources, "logparsers") + var crd apiextensionsv1.CustomResourceDefinition + require.NoError(t, client.Get(context.Background(), types.NamespacedName{Name: "logpipelines.telemetry.kyma-project.io"}, &crd)) + + crdCABundle := crd.Spec.Conversion.Webhook.ClientConfig.CABundle + require.NotEmpty(t, crdCABundle, "ca bundle must be patched") + + certValid, err = chainChecker.checkRoot(context.Background(), serverCert, crdCABundle) + require.NoError(t, err) + require.True(t, certValid) } func TestUpdateWebhookCertificate(t *testing.T) { From d05404a475274a43761b3dec56be617fa86062cd Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Mon, 23 Sep 2024 08:24:01 +0200 Subject: [PATCH 22/35] Remove comment --- apis/telemetry/v1alpha1/logpipeline_conversion.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/apis/telemetry/v1alpha1/logpipeline_conversion.go b/apis/telemetry/v1alpha1/logpipeline_conversion.go index b3424886d..666cebe2a 100644 --- a/apis/telemetry/v1alpha1/logpipeline_conversion.go +++ b/apis/telemetry/v1alpha1/logpipeline_conversion.go @@ -102,8 +102,6 @@ func v1Alpha1TLSToV1Beta1(src TLSConfig) telemetryv1beta1.LogPipelineHTTPOutputT } // ConvertFrom converts from the Hub version (v1beta1) to this version. -// -//lint:ignore ST1016 This is a conversion function and "dst" makes sense here. func (lp *LogPipeline) ConvertFrom(srcRaw conversion.Hub) error { dst := lp src, ok := srcRaw.(*telemetryv1beta1.LogPipeline) From 56724cb97f23fa377308d1ed4b39d62ee9c01c66 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Wed, 25 Sep 2024 18:25:25 +0200 Subject: [PATCH 23/35] Fix rbac --- config/rbac/role.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f7b588294..34c541beb 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -57,11 +57,7 @@ rules: resources: - customresourcedefinitions verbs: - - get - - list - patch - - update - - watch - apiGroups: - apps resources: From 2d75a06f00da403c4030b3093d90c4548d04a97b Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Wed, 25 Sep 2024 19:57:57 +0200 Subject: [PATCH 24/35] Path the whole conversion session --- .../webhook_in_telemetry_logpipelines.yaml | 2 +- config/development/crd/kustomization.yaml | 3 -- .../webhook_in_telemetry_logpipelines.yaml | 16 ------- config/rbac/role.yaml | 1 + .../log_pipeline_webhook_configs.go | 44 ++++++++++++++----- main.go | 2 +- 6 files changed, 35 insertions(+), 33 deletions(-) delete mode 100644 config/development/crd/patches/webhook_in_telemetry_logpipelines.yaml diff --git a/config/crd/patches/webhook_in_telemetry_logpipelines.yaml b/config/crd/patches/webhook_in_telemetry_logpipelines.yaml index baae9696e..22627ae2e 100644 --- a/config/crd/patches/webhook_in_telemetry_logpipelines.yaml +++ b/config/crd/patches/webhook_in_telemetry_logpipelines.yaml @@ -1,4 +1,4 @@ -# The following patch enables a conversion webhook for the CRD +#=The following patch enables a conversion webhook for the CRD apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: diff --git a/config/development/crd/kustomization.yaml b/config/development/crd/kustomization.yaml index ec26b82a6..339fa6bae 100644 --- a/config/development/crd/kustomization.yaml +++ b/config/development/crd/kustomization.yaml @@ -5,9 +5,6 @@ resources: - bases/operator.kyma-project.io_telemetries.yaml - bases/telemetry.kyma-project.io_metricpipelines.yaml -patchesStrategicMerge: -- patches/webhook_in_telemetry_logpipelines.yaml - # the following config is for teaching kustomize how to do kustomization for CRDs. configurations: - kustomizeconfig.yaml diff --git a/config/development/crd/patches/webhook_in_telemetry_logpipelines.yaml b/config/development/crd/patches/webhook_in_telemetry_logpipelines.yaml deleted file mode 100644 index 0425cc8d7..000000000 --- a/config/development/crd/patches/webhook_in_telemetry_logpipelines.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# The following patch enables a conversion webhook for the CRD -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: logpipelines.telemetry.kyma-project.io -spec: - conversion: - strategy: Webhook - webhook: - clientConfig: - service: - namespace: system - name: manager-webhook - path: /convert - conversionReviewVersions: - - v1 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 34c541beb..6804150b9 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -57,6 +57,7 @@ rules: resources: - customresourcedefinitions verbs: + - get - patch - apiGroups: - apps diff --git a/internal/webhookcert/log_pipeline_webhook_configs.go b/internal/webhookcert/log_pipeline_webhook_configs.go index 25319f17f..b88b07d90 100644 --- a/internal/webhookcert/log_pipeline_webhook_configs.go +++ b/internal/webhookcert/log_pipeline_webhook_configs.go @@ -8,11 +8,16 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/kyma-project/telemetry-manager/internal/k8sutils" ) +const ( + webhookServicePort int32 = 443 +) + // ensureLogPipelineWebhookConfigs creates or updates the ValidatingWebhookConfiguration for the LogPipeline resources. // additionally it patches the conversion webhook configuration with the CA bundle. func ensureLogPipelineWebhookConfigs(ctx context.Context, c client.Client, caBundle []byte, config Config) error { @@ -21,14 +26,15 @@ func ensureLogPipelineWebhookConfigs(ctx context.Context, c client.Client, caBun return fmt.Errorf("failed to create or update validating webhook configuration: %w", err) } - if err := patchConversionWebhookConfig(ctx, c, caBundle); err != nil { + conversionWebhookConfig := makeConversionWebhookConfig(caBundle, config) + if err := patchConversionWebhookConfig(ctx, c, conversionWebhookConfig); err != nil { return fmt.Errorf("failed to patch conversion webhook configuration: %w", err) } return nil } -func makeValidatingWebhookConfig(certificate []byte, config Config) admissionregistrationv1.ValidatingWebhookConfiguration { +func makeValidatingWebhookConfig(caBundle []byte, config Config) admissionregistrationv1.ValidatingWebhookConfiguration { logPipelinePath := "/validate-logpipeline" logParserPath := "/validate-logparser" failurePolicy := admissionregistrationv1.Fail @@ -41,7 +47,6 @@ func makeValidatingWebhookConfig(certificate []byte, config Config) admissionreg apiGroups := []string{"telemetry.kyma-project.io"} apiVersions := []string{"v1alpha1"} scope := admissionregistrationv1.AllScopes - servicePort := int32(443) timeout := int32(15) labels := map[string]string{ "control-plane": "telemetry-manager", @@ -63,10 +68,10 @@ func makeValidatingWebhookConfig(certificate []byte, config Config) admissionreg Service: &admissionregistrationv1.ServiceReference{ Name: config.ServiceName.Name, Namespace: config.ServiceName.Namespace, - Port: &servicePort, + Port: ptr.To(webhookServicePort), Path: &logPipelinePath, }, - CABundle: certificate, + CABundle: caBundle, }, FailurePolicy: &failurePolicy, MatchPolicy: &matchPolicy, @@ -91,10 +96,10 @@ func makeValidatingWebhookConfig(certificate []byte, config Config) admissionreg Service: &admissionregistrationv1.ServiceReference{ Name: config.ServiceName.Name, Namespace: config.ServiceName.Namespace, - Port: &servicePort, + Port: ptr.To(webhookServicePort), Path: &logParserPath, }, - CABundle: certificate, + CABundle: caBundle, }, FailurePolicy: &failurePolicy, MatchPolicy: &matchPolicy, @@ -117,7 +122,25 @@ func makeValidatingWebhookConfig(certificate []byte, config Config) admissionreg } } -func patchConversionWebhookConfig(ctx context.Context, c client.Client, caBundle []byte) error { +func makeConversionWebhookConfig(caBundle []byte, config Config) apiextensionsv1.CustomResourceConversion { + return apiextensionsv1.CustomResourceConversion{ + Strategy: apiextensionsv1.WebhookConverter, + Webhook: &apiextensionsv1.WebhookConversion{ + ClientConfig: &apiextensionsv1.WebhookClientConfig{ + Service: &apiextensionsv1.ServiceReference{ + Namespace: config.ServiceName.Namespace, + Name: config.ServiceName.Name, + Path: ptr.To("/convert"), + Port: ptr.To(webhookServicePort), + }, + CABundle: caBundle, + }, + ConversionReviewVersions: []string{"v1"}, + }, + } +} + +func patchConversionWebhookConfig(ctx context.Context, c client.Client, conversion apiextensionsv1.CustomResourceConversion) error { var logPipelineCRD apiextensionsv1.CustomResourceDefinition if err := c.Get(ctx, types.NamespacedName{Name: "logpipelines.telemetry.kyma-project.io"}, &logPipelineCRD); err != nil { return fmt.Errorf("failed to get logpipelines CRD: %w", err) @@ -125,10 +148,7 @@ func patchConversionWebhookConfig(ctx context.Context, c client.Client, caBundle patch := client.MergeFrom(logPipelineCRD.DeepCopy()) - conversion := logPipelineCRD.Spec.Conversion - if conversion != nil && conversion.Webhook != nil && conversion.Webhook.ClientConfig != nil { - conversion.Webhook.ClientConfig.CABundle = caBundle - } + logPipelineCRD.Spec.Conversion = &conversion return c.Patch(ctx, &logPipelineCRD, patch) } diff --git a/main.go b/main.go index 0db7c9bc2..f4daeb514 100644 --- a/main.go +++ b/main.go @@ -198,7 +198,7 @@ func getEnvOrDefault(envVar string, defaultValue string) string { //+kubebuilder:rbac:urls=/metrics,verbs=get //+kubebuilder:rbac:urls=/metrics/cadvisor,verbs=get -//+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=patch +//+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;patch //+kubebuilder:rbac:groups=apps,namespace=system,resources=deployments,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=apps,namespace=system,resources=daemonsets,verbs=get;list;watch;create;update;patch;delete From e277b9f9e77f475fe62abeb133c89e6023d03ed6 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Wed, 25 Sep 2024 20:08:09 +0200 Subject: [PATCH 25/35] Improve unit tests --- internal/webhookcert/webhook_cert_test.go | 40 +++++++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/internal/webhookcert/webhook_cert_test.go b/internal/webhookcert/webhook_cert_test.go index 27cb49040..b0be001e8 100644 --- a/internal/webhookcert/webhook_cert_test.go +++ b/internal/webhookcert/webhook_cert_test.go @@ -52,7 +52,7 @@ var ( } ) -func TestEnsureCertificate(t *testing.T) { +func TestEnsureCertificate_CreatesValidatingWebhookConfig(t *testing.T) { scheme := runtime.NewScheme() require.NoError(t, clientgoscheme.AddToScheme(scheme)) require.NoError(t, apiextensionsv1.AddToScheme(scheme)) @@ -120,14 +120,48 @@ func TestEnsureCertificate(t *testing.T) { require.Contains(t, validatingWebhookConfiguration.Webhooks[0].Rules[0].Resources, "logpipelines") require.Contains(t, validatingWebhookConfiguration.Webhooks[1].Rules[0].Resources, "logparsers") +} + +func TestEnsureCertificate_PatchesConversionWebhookConfig(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, apiextensionsv1.AddToScheme(scheme)) + client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(&logPipelinesCRD).Build() + + certDir, err := os.MkdirTemp("", "certificate") + require.NoError(t, err) + defer func(path string) { + deleteErr := os.RemoveAll(path) + require.NoError(t, deleteErr) + }(certDir) + + config := Config{ + CertDir: certDir, + ServiceName: webhookService, + CASecretName: caBundleSecret, + WebhookName: webhookName, + } + + err = EnsureCertificate(context.TODO(), client, config) + require.NoError(t, err) + + serverCert, err := os.ReadFile(path.Join(certDir, "tls.crt")) + require.NoError(t, err) var crd apiextensionsv1.CustomResourceDefinition require.NoError(t, client.Get(context.Background(), types.NamespacedName{Name: "logpipelines.telemetry.kyma-project.io"}, &crd)) + require.Equal(t, apiextensionsv1.WebhookConverter, crd.Spec.Conversion.Strategy) + require.Equal(t, webhookService.Name, crd.Spec.Conversion.Webhook.ClientConfig.Service.Name) + require.Equal(t, webhookService.Namespace, crd.Spec.Conversion.Webhook.ClientConfig.Service.Namespace) + require.Equal(t, int32(443), *crd.Spec.Conversion.Webhook.ClientConfig.Service.Port) + require.Equal(t, "/convert", *crd.Spec.Conversion.Webhook.ClientConfig.Service.Path) + crdCABundle := crd.Spec.Conversion.Webhook.ClientConfig.CABundle - require.NotEmpty(t, crdCABundle, "ca bundle must be patched") + require.NotEmpty(t, crdCABundle) - certValid, err = chainChecker.checkRoot(context.Background(), serverCert, crdCABundle) + var chainChecker certChainCheckerImpl + certValid, err := chainChecker.checkRoot(context.Background(), serverCert, crdCABundle) require.NoError(t, err) require.True(t, certValid) } From 4a197da24c5037c57453ee6e0cb8e9a519e12297 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Wed, 25 Sep 2024 20:45:13 +0200 Subject: [PATCH 26/35] Add e2e test --- test/e2e/logs_version_conversion_test.go | 100 +++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 test/e2e/logs_version_conversion_test.go diff --git a/test/e2e/logs_version_conversion_test.go b/test/e2e/logs_version_conversion_test.go new file mode 100644 index 000000000..301067a4b --- /dev/null +++ b/test/e2e/logs_version_conversion_test.go @@ -0,0 +1,100 @@ +//go:build e2e + +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + telemetryv1alpha1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1alpha1" + telemetryv1beta1 "github.com/kyma-project/telemetry-manager/apis/telemetry/v1beta1" + "github.com/kyma-project/telemetry-manager/test/testkit/assert" + kitk8s "github.com/kyma-project/telemetry-manager/test/testkit/k8s" + "github.com/kyma-project/telemetry-manager/test/testkit/suite" +) + +var _ = Describe(suite.ID(), Label(suite.LabelLogs, suite.LabelExperimental), Ordered, func() { + var ( + v1Alpha1PipelineName = suite.IDWithSuffix("v1alpha1") + v1Beta1PipelineName = suite.IDWithSuffix("v1beta1") + ) + + makeResources := func() []client.Object { + v1Alpha1LogPipeline := telemetryv1alpha1.LogPipeline{ + ObjectMeta: metav1.ObjectMeta{ + Name: v1Alpha1PipelineName, + }, + Spec: telemetryv1alpha1.LogPipelineSpec{ + Output: telemetryv1alpha1.Output{ + HTTP: &telemetryv1alpha1.HTTPOutput{ + Host: telemetryv1alpha1.ValueType{ + Value: "localhost", + }, + Port: "443", + URI: "/", + TLSConfig: telemetryv1alpha1.TLSConfig{ + Disabled: true, + }, + }, + }, + }, + } + + v1Beta1LogPipeline := telemetryv1beta1.LogPipeline{ + ObjectMeta: metav1.ObjectMeta{ + Name: v1Beta1PipelineName, + }, + Spec: telemetryv1beta1.LogPipelineSpec{ + Output: telemetryv1beta1.LogPipelineOutput{ + HTTP: &telemetryv1beta1.LogPipelineHTTPOutput{ + Host: telemetryv1beta1.ValueType{ + Value: "localhost", + }, + Port: "443", + URI: "/", + TLSConfig: telemetryv1beta1.LogPipelineHTTPOutputTLS{ + Disabled: true, + }, + }, + }, + }, + } + + return []client.Object{&v1Alpha1LogPipeline, &v1Beta1LogPipeline} + } + + Context("Before deploying a logpipeline", func() { + It("Should have a healthy webhook", func() { + assert.WebhookHealthy(ctx, k8sClient) + }) + }) + + Context("When v1alpha1 and v1beta1 logpipelines exist", Ordered, func() { + BeforeAll(func() { + k8sObjects := makeResources() + DeferCleanup(func() { + Expect(kitk8s.DeleteObjects(ctx, k8sClient, k8sObjects...)).Should(Succeed()) + }) + Expect(kitk8s.CreateObjects(ctx, k8sClient, k8sObjects...)).Should(Succeed()) + }) + + It("Should convert v1alpha1 logpipeline to v1beta1 logpipeline", func() { + var v1Alpha1AsV1Beta1 telemetryv1beta1.LogPipeline + err := k8sClient.Get(ctx, types.NamespacedName{Name: v1Alpha1PipelineName}, &v1Alpha1AsV1Beta1) + Expect(err).ToNot(HaveOccurred()) + + Expect(v1Alpha1AsV1Beta1.Name).To(Equal(v1Alpha1PipelineName)) + }) + + It("Should convert v1beta1 logpipeline to v1alpha1 logpipeline", func() { + var v1Beta1AsV1Alpha1 telemetryv1alpha1.LogPipeline + err := k8sClient.Get(ctx, types.NamespacedName{Name: v1Beta1PipelineName}, &v1Beta1AsV1Alpha1) + Expect(err).ToNot(HaveOccurred()) + + Expect(v1Beta1AsV1Alpha1.Name).To(Equal(v1Beta1PipelineName)) + }) + }) +}) From a8c9558172a0a55141728e10545ec672870d7111 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Wed, 25 Sep 2024 20:53:41 +0200 Subject: [PATCH 27/35] Adapt v1beta1 log pipeline --- .../v1alpha1/logpipeline_conversion.go | 19 ++++++++++--------- .../v1alpha1/logpipeline_conversion_test.go | 4 +++- apis/telemetry/v1beta1/logpipeline_types.go | 6 +++++- .../v1beta1/logpipeline_validation_test.go | 16 ++++++++-------- .../v1beta1/zz_generated.deepcopy.go | 11 ++++++----- 5 files changed, 32 insertions(+), 24 deletions(-) diff --git a/apis/telemetry/v1alpha1/logpipeline_conversion.go b/apis/telemetry/v1alpha1/logpipeline_conversion.go index 666cebe2a..d006d4767 100644 --- a/apis/telemetry/v1alpha1/logpipeline_conversion.go +++ b/apis/telemetry/v1alpha1/logpipeline_conversion.go @@ -23,7 +23,8 @@ func (lp *LogPipeline) ConvertTo(dstRaw conversion.Hub) error { srcAppInput := src.Spec.Input.Application dst.Spec.Input = telemetryv1beta1.LogPipelineInput{ - Runtime: &telemetryv1beta1.LogPipelineRuntimeInput{ + Runtime: telemetryv1beta1.LogPipelineRuntimeInput{ + Enabled: srcAppInput.Enabled, Namespaces: telemetryv1beta1.LogPipelineInputNamespaces(srcAppInput.Namespaces), Containers: telemetryv1beta1.LogPipelineInputContainers(srcAppInput.Containers), KeepAnnotations: srcAppInput.KeepAnnotations, @@ -111,14 +112,14 @@ func (lp *LogPipeline) ConvertFrom(srcRaw conversion.Hub) error { dst.ObjectMeta = src.ObjectMeta - if srcAppInput := src.Spec.Input.Runtime; srcAppInput != nil { - dst.Spec.Input.Application = ApplicationInput{ - Namespaces: InputNamespaces(srcAppInput.Namespaces), - Containers: InputContainers(srcAppInput.Containers), - KeepAnnotations: srcAppInput.KeepAnnotations, - DropLabels: srcAppInput.DropLabels, - KeepOriginalBody: srcAppInput.KeepOriginalBody, - } + srcRuntimeInput := src.Spec.Input.Runtime + dst.Spec.Input.Application = ApplicationInput{ + Enabled: srcRuntimeInput.Enabled, + Namespaces: InputNamespaces(srcRuntimeInput.Namespaces), + Containers: InputContainers(srcRuntimeInput.Containers), + KeepAnnotations: srcRuntimeInput.KeepAnnotations, + DropLabels: srcRuntimeInput.DropLabels, + KeepOriginalBody: srcRuntimeInput.KeepOriginalBody, } for _, f := range src.Spec.Files { diff --git a/apis/telemetry/v1alpha1/logpipeline_conversion_test.go b/apis/telemetry/v1alpha1/logpipeline_conversion_test.go index ac48db0d1..d8e37ec63 100644 --- a/apis/telemetry/v1alpha1/logpipeline_conversion_test.go +++ b/apis/telemetry/v1alpha1/logpipeline_conversion_test.go @@ -19,6 +19,7 @@ func TestConvertTo(t *testing.T) { Spec: LogPipelineSpec{ Input: Input{ Application: ApplicationInput{ + Enabled: ptr.To(true), Namespaces: InputNamespaces{ Include: []string{"default", "kube-system"}, Exclude: []string{"kube-public"}, @@ -110,7 +111,8 @@ func TestConvertFrom(t *testing.T) { }, Spec: telemetryv1beta1.LogPipelineSpec{ Input: telemetryv1beta1.LogPipelineInput{ - Runtime: &telemetryv1beta1.LogPipelineRuntimeInput{ + Runtime: telemetryv1beta1.LogPipelineRuntimeInput{ + Enabled: ptr.To(true), Namespaces: telemetryv1beta1.LogPipelineInputNamespaces{ Include: []string{"default", "kube-system"}, Exclude: []string{"kube-public"}, diff --git a/apis/telemetry/v1beta1/logpipeline_types.go b/apis/telemetry/v1beta1/logpipeline_types.go index 5879edec9..687b62118 100644 --- a/apis/telemetry/v1beta1/logpipeline_types.go +++ b/apis/telemetry/v1beta1/logpipeline_types.go @@ -59,11 +59,15 @@ type LogPipelineSpec struct { // LogPipelineInput describes a log input for a LogPipeline. type LogPipelineInput struct { // Configures in more detail from which containers application logs are enabled as input. - Runtime *LogPipelineRuntimeInput `json:"runtime,omitempty"` + Runtime LogPipelineRuntimeInput `json:"runtime,omitempty"` } // LogPipelineRuntimeInput specifies the default type of Input that handles application logs from runtime containers. It configures in more detail from which containers logs are selected as input. type LogPipelineRuntimeInput struct { + // If enabled, application logs are collected. The default is `true`. + // +optional + // +kubebuilder:default=true + Enabled *bool `json:"enabled,omitempty"` // Describes whether application logs from specific Namespaces are selected. The options are mutually exclusive. System Namespaces are excluded by default from the collection. Namespaces LogPipelineInputNamespaces `json:"namespaces,omitempty"` // Describes whether application logs from specific containers are selected. The options are mutually exclusive. diff --git a/apis/telemetry/v1beta1/logpipeline_validation_test.go b/apis/telemetry/v1beta1/logpipeline_validation_test.go index c52bcdba3..40b32a312 100644 --- a/apis/telemetry/v1beta1/logpipeline_validation_test.go +++ b/apis/telemetry/v1beta1/logpipeline_validation_test.go @@ -232,7 +232,7 @@ func TestValidateWithValidInputIncludes(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: &LogPipelineRuntimeInput{ + Runtime: LogPipelineRuntimeInput{ Namespaces: LogPipelineInputNamespaces{ Include: []string{"namespace-1", "namespace-2"}, }, @@ -251,7 +251,7 @@ func TestValidateWithValidInputExcludes(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: &LogPipelineRuntimeInput{ + Runtime: LogPipelineRuntimeInput{ Namespaces: LogPipelineInputNamespaces{ Exclude: []string{"namespace-1", "namespace-2"}, }, @@ -271,7 +271,7 @@ func TestValidateWithValidInputIncludeContainersSystemFlag(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: &LogPipelineRuntimeInput{ + Runtime: LogPipelineRuntimeInput{ Namespaces: LogPipelineInputNamespaces{ System: true, }, @@ -291,7 +291,7 @@ func TestValidateWithValidInputExcludeContainersSystemFlag(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: &LogPipelineRuntimeInput{ + Runtime: LogPipelineRuntimeInput{ Namespaces: LogPipelineInputNamespaces{ System: true, }, @@ -311,7 +311,7 @@ func TestValidateWithInvalidNamespaceSelectors(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: &LogPipelineRuntimeInput{ + Runtime: LogPipelineRuntimeInput{ Namespaces: LogPipelineInputNamespaces{ Include: []string{"namespace-1", "namespace-2"}, Exclude: []string{"namespace-3"}, @@ -329,7 +329,7 @@ func TestValidateWithInvalidIncludeSystemFlag(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: &LogPipelineRuntimeInput{ + Runtime: LogPipelineRuntimeInput{ Namespaces: LogPipelineInputNamespaces{ Include: []string{"namespace-1", "namespace-2"}, System: true, @@ -347,7 +347,7 @@ func TestValidateWithInvalidExcludeSystemFlag(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: &LogPipelineRuntimeInput{ + Runtime: LogPipelineRuntimeInput{ Namespaces: LogPipelineInputNamespaces{ Exclude: []string{"namespace-3"}, System: true, @@ -365,7 +365,7 @@ func TestValidateWithInvalidContainerSelectors(t *testing.T) { logPipeline := &LogPipeline{ Spec: LogPipelineSpec{ Input: LogPipelineInput{ - Runtime: &LogPipelineRuntimeInput{ + Runtime: LogPipelineRuntimeInput{ Containers: LogPipelineInputContainers{ Include: []string{"container-1", "container-2"}, Exclude: []string{"container-3"}, diff --git a/apis/telemetry/v1beta1/zz_generated.deepcopy.go b/apis/telemetry/v1beta1/zz_generated.deepcopy.go index 7722b6992..2664022aa 100644 --- a/apis/telemetry/v1beta1/zz_generated.deepcopy.go +++ b/apis/telemetry/v1beta1/zz_generated.deepcopy.go @@ -202,11 +202,7 @@ func (in *LogPipelineHTTPOutputTLS) DeepCopy() *LogPipelineHTTPOutputTLS { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LogPipelineInput) DeepCopyInto(out *LogPipelineInput) { *out = *in - if in.Runtime != nil { - in, out := &in.Runtime, &out.Runtime - *out = new(LogPipelineRuntimeInput) - (*in).DeepCopyInto(*out) - } + in.Runtime.DeepCopyInto(&out.Runtime) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LogPipelineInput. @@ -324,6 +320,11 @@ func (in *LogPipelineOutput) DeepCopy() *LogPipelineOutput { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LogPipelineRuntimeInput) DeepCopyInto(out *LogPipelineRuntimeInput) { *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } in.Namespaces.DeepCopyInto(&out.Namespaces) in.Containers.DeepCopyInto(&out.Containers) if in.KeepOriginalBody != nil { From 569a3d86dc3564eaa5a425008c68f017d1e37ed4 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Wed, 25 Sep 2024 21:06:55 +0200 Subject: [PATCH 28/35] make manifests --- .../crd/bases/telemetry.kyma-project.io_logpipelines.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/development/crd/bases/telemetry.kyma-project.io_logpipelines.yaml b/config/development/crd/bases/telemetry.kyma-project.io_logpipelines.yaml index 9a65e5c8a..47ecf2ccc 100644 --- a/config/development/crd/bases/telemetry.kyma-project.io_logpipelines.yaml +++ b/config/development/crd/bases/telemetry.kyma-project.io_logpipelines.yaml @@ -577,6 +577,11 @@ spec: description: Defines whether to drop all Kubernetes labels. The default is `false`. type: boolean + enabled: + default: true + description: If enabled, application logs are collected. The + default is `true`. + type: boolean keepAnnotations: description: Defines whether to keep all Kubernetes annotations. The default is `false`. From 249024cbf5ad7031eb7fd0ff02ec70e7f930a559 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Thu, 26 Sep 2024 08:09:34 +0200 Subject: [PATCH 29/35] Refactor creating validation webhook: fix dupl --- .../log_pipeline_webhook_configs.go | 108 +++++++----------- 1 file changed, 40 insertions(+), 68 deletions(-) diff --git a/internal/webhookcert/log_pipeline_webhook_configs.go b/internal/webhookcert/log_pipeline_webhook_configs.go index b88b07d90..3edb87928 100644 --- a/internal/webhookcert/log_pipeline_webhook_configs.go +++ b/internal/webhookcert/log_pipeline_webhook_configs.go @@ -35,19 +35,9 @@ func ensureLogPipelineWebhookConfigs(ctx context.Context, c client.Client, caBun } func makeValidatingWebhookConfig(caBundle []byte, config Config) admissionregistrationv1.ValidatingWebhookConfiguration { - logPipelinePath := "/validate-logpipeline" - logParserPath := "/validate-logparser" - failurePolicy := admissionregistrationv1.Fail - matchPolicy := admissionregistrationv1.Exact - sideEffects := admissionregistrationv1.SideEffectClassNone - operations := []admissionregistrationv1.OperationType{ - admissionregistrationv1.Create, - admissionregistrationv1.Update, - } apiGroups := []string{"telemetry.kyma-project.io"} apiVersions := []string{"v1alpha1"} - scope := admissionregistrationv1.AllScopes - timeout := int32(15) + webhookTimeout := int32(15) labels := map[string]string{ "control-plane": "telemetry-manager", "app.kubernetes.io/instance": "telemetry", @@ -55,70 +45,52 @@ func makeValidatingWebhookConfig(caBundle []byte, config Config) admissionregist "kyma-project.io/component": "controller", } - return admissionregistrationv1.ValidatingWebhookConfiguration{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: config.WebhookName.Name, - Labels: labels, - }, - Webhooks: []admissionregistrationv1.ValidatingWebhook{ - { - AdmissionReviewVersions: []string{"v1beta1", "v1"}, - ClientConfig: admissionregistrationv1.WebhookClientConfig{ - Service: &admissionregistrationv1.ServiceReference{ - Name: config.ServiceName.Name, - Namespace: config.ServiceName.Namespace, - Port: ptr.To(webhookServicePort), - Path: &logPipelinePath, - }, - CABundle: caBundle, - }, - FailurePolicy: &failurePolicy, - MatchPolicy: &matchPolicy, - Name: "validation.logpipelines.telemetry.kyma-project.io", - SideEffects: &sideEffects, - TimeoutSeconds: &timeout, - Rules: []admissionregistrationv1.RuleWithOperations{ - { - Operations: operations, - Rule: admissionregistrationv1.Rule{ - APIGroups: apiGroups, - APIVersions: apiVersions, - Scope: &scope, - Resources: []string{"logpipelines"}, - }, - }, + createWebhook := func(name, path string, resources []string) admissionregistrationv1.ValidatingWebhook { + return admissionregistrationv1.ValidatingWebhook{ + AdmissionReviewVersions: []string{"v1beta1", "v1"}, + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Name: config.ServiceName.Name, + Namespace: config.ServiceName.Namespace, + Port: ptr.To(webhookServicePort), + Path: &path, }, + CABundle: caBundle, }, - { - AdmissionReviewVersions: []string{"v1beta1", "v1"}, - ClientConfig: admissionregistrationv1.WebhookClientConfig{ - Service: &admissionregistrationv1.ServiceReference{ - Name: config.ServiceName.Name, - Namespace: config.ServiceName.Namespace, - Port: ptr.To(webhookServicePort), - Path: &logParserPath, + FailurePolicy: ptr.To(admissionregistrationv1.Fail), + MatchPolicy: ptr.To(admissionregistrationv1.Exact), + Name: name, + SideEffects: ptr.To(admissionregistrationv1.SideEffectClassNone), + TimeoutSeconds: &webhookTimeout, + Rules: []admissionregistrationv1.RuleWithOperations{ + { + Operations: []admissionregistrationv1.OperationType{ + admissionregistrationv1.Create, + admissionregistrationv1.Update, }, - CABundle: caBundle, - }, - FailurePolicy: &failurePolicy, - MatchPolicy: &matchPolicy, - Name: "validation.logparsers.telemetry.kyma-project.io", - SideEffects: &sideEffects, - TimeoutSeconds: &timeout, - Rules: []admissionregistrationv1.RuleWithOperations{ - { - Operations: operations, - Rule: admissionregistrationv1.Rule{ - APIGroups: apiGroups, - APIVersions: apiVersions, - Scope: &scope, - Resources: []string{"logparsers"}, - }, + Rule: admissionregistrationv1.Rule{ + APIGroups: apiGroups, + APIVersions: apiVersions, + Scope: ptr.To(admissionregistrationv1.AllScopes), + Resources: resources, }, }, }, + } + } + + webhooks := []admissionregistrationv1.ValidatingWebhook{ + createWebhook("validation.logpipelines.telemetry.kyma-project.io", "/validate-logpipeline", []string{"logpipelines"}), + createWebhook("validation.logparsers.telemetry.kyma-project.io", "/validate-logparser", []string{"logparsers"}), + } + + return admissionregistrationv1.ValidatingWebhookConfiguration{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: config.WebhookName.Name, + Labels: labels, }, + Webhooks: webhooks, } } From 2ea47b235390433088896dc8fe523343e73a58fe Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Thu, 26 Sep 2024 08:15:02 +0200 Subject: [PATCH 30/35] Minor renamings --- .../{log_pipeline_webhook_configs.go => resources.go} | 6 +++--- internal/webhookcert/webhook_cert.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) rename internal/webhookcert/{log_pipeline_webhook_configs.go => resources.go} (93%) diff --git a/internal/webhookcert/log_pipeline_webhook_configs.go b/internal/webhookcert/resources.go similarity index 93% rename from internal/webhookcert/log_pipeline_webhook_configs.go rename to internal/webhookcert/resources.go index 3edb87928..da9e45b15 100644 --- a/internal/webhookcert/log_pipeline_webhook_configs.go +++ b/internal/webhookcert/resources.go @@ -18,9 +18,9 @@ const ( webhookServicePort int32 = 443 ) -// ensureLogPipelineWebhookConfigs creates or updates the ValidatingWebhookConfiguration for the LogPipeline resources. -// additionally it patches the conversion webhook configuration with the CA bundle. -func ensureLogPipelineWebhookConfigs(ctx context.Context, c client.Client, caBundle []byte, config Config) error { +// applyWebhookConfigResources creates or updates a ValidatingWebhookConfiguration for the LogPipeline/LogParser resources. +// additionally it patches a LogPipeline conversion webhook configuration. +func applyWebhookConfigResources(ctx context.Context, c client.Client, caBundle []byte, config Config) error { validatingWebhookConfig := makeValidatingWebhookConfig(caBundle, config) if err := k8sutils.CreateOrUpdateValidatingWebhookConfiguration(ctx, c, &validatingWebhookConfig); err != nil { return fmt.Errorf("failed to create or update validating webhook configuration: %w", err) diff --git a/internal/webhookcert/webhook_cert.go b/internal/webhookcert/webhook_cert.go index 183e4d246..630650fdd 100644 --- a/internal/webhookcert/webhook_cert.go +++ b/internal/webhookcert/webhook_cert.go @@ -32,7 +32,7 @@ func EnsureCertificate(ctx context.Context, client client.Client, config Config) return fmt.Errorf("failed to provide server cert/key: %w", err) } - return ensureLogPipelineWebhookConfigs(ctx, client, caCertPEM, config) + return applyWebhookConfigResources(ctx, client, caCertPEM, config) } func dnsNames(webhookService types.NamespacedName) (host string, alternativeDNSNames []string) { From 3d38875dea1e365d317b003966606695fbe2da07 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Thu, 26 Sep 2024 10:45:21 +0200 Subject: [PATCH 31/35] Fix rbac --- config/rbac/role.yaml | 1 + main.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 6804150b9..ac09b8634 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -58,6 +58,7 @@ rules: - customresourcedefinitions verbs: - get + - list - patch - apiGroups: - apps diff --git a/main.go b/main.go index f4daeb514..f7c0c0966 100644 --- a/main.go +++ b/main.go @@ -198,7 +198,7 @@ func getEnvOrDefault(envVar string, defaultValue string) string { //+kubebuilder:rbac:urls=/metrics,verbs=get //+kubebuilder:rbac:urls=/metrics/cadvisor,verbs=get -//+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;patch +//+kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;patch //+kubebuilder:rbac:groups=apps,namespace=system,resources=deployments,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=apps,namespace=system,resources=daemonsets,verbs=get;list;watch;create;update;patch;delete From c69587386bf9043c6f71a158abb43911c13f647d Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Thu, 26 Sep 2024 16:17:32 +0200 Subject: [PATCH 32/35] Strategic merge patch --- internal/webhookcert/resources.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/webhookcert/resources.go b/internal/webhookcert/resources.go index da9e45b15..67f769d09 100644 --- a/internal/webhookcert/resources.go +++ b/internal/webhookcert/resources.go @@ -118,7 +118,7 @@ func patchConversionWebhookConfig(ctx context.Context, c client.Client, conversi return fmt.Errorf("failed to get logpipelines CRD: %w", err) } - patch := client.MergeFrom(logPipelineCRD.DeepCopy()) + patch := client.StrategicMergeFrom(logPipelineCRD.DeepCopy()) logPipelineCRD.Spec.Conversion = &conversion From d965957f38599b25ac976cdc3c1e70d255633148 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Thu, 26 Sep 2024 17:18:07 +0200 Subject: [PATCH 33/35] Adapt threshold --- .testcoverage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.testcoverage.yml b/.testcoverage.yml index 86d05325a..dcb094626 100644 --- a/.testcoverage.yml +++ b/.testcoverage.yml @@ -21,7 +21,7 @@ override: # Temporarily decrease coverage threshold for certain packages, we need to increase the coverage and remove these entries from the list - threshold: 57 path: ^internal/istiostatus$ - - threshold: 86 + - threshold: 85 path: ^internal/otelcollector/config/otlpexporter$ - threshold: 73 path: ^internal/reconciler/logpipeline$ From 062e30bb0c867878ab12f08584ac9d07905c04d0 Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Thu, 26 Sep 2024 17:39:16 +0200 Subject: [PATCH 34/35] Adapt threshold --- .testcoverage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.testcoverage.yml b/.testcoverage.yml index dcb094626..020cdcf26 100644 --- a/.testcoverage.yml +++ b/.testcoverage.yml @@ -21,7 +21,7 @@ override: # Temporarily decrease coverage threshold for certain packages, we need to increase the coverage and remove these entries from the list - threshold: 57 path: ^internal/istiostatus$ - - threshold: 85 + - threshold: 86 path: ^internal/otelcollector/config/otlpexporter$ - threshold: 73 path: ^internal/reconciler/logpipeline$ @@ -37,7 +37,7 @@ override: path: ^internal/resources/selfmonitor$ - threshold: 75 path: ^internal/resourcelock$ - - threshold: 86 + - threshold: 85 path: ^internal/webhookcert$ - threshold: 70 path: ^webhook/logpipeline From b916b28d3d14cd002eb70ca7e20c6f7038fe6a3d Mon Sep 17 00:00:00 2001 From: Stanislav Khalash Date: Thu, 26 Sep 2024 19:57:47 +0200 Subject: [PATCH 35/35] Use simple merge --- internal/webhookcert/resources.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/webhookcert/resources.go b/internal/webhookcert/resources.go index 67f769d09..da9e45b15 100644 --- a/internal/webhookcert/resources.go +++ b/internal/webhookcert/resources.go @@ -118,7 +118,7 @@ func patchConversionWebhookConfig(ctx context.Context, c client.Client, conversi return fmt.Errorf("failed to get logpipelines CRD: %w", err) } - patch := client.StrategicMergeFrom(logPipelineCRD.DeepCopy()) + patch := client.MergeFrom(logPipelineCRD.DeepCopy()) logPipelineCRD.Spec.Conversion = &conversion