From 4bc0c62db3fc234921a4ac1258344d5540573c55 Mon Sep 17 00:00:00 2001 From: Andrew Hess Date: Thu, 26 Sep 2024 08:12:41 -0700 Subject: [PATCH] Revert "Disallow unknown fields in server.yaml (#1682)" (#1684) This reverts commit d6c2f9a66262648d2550800537729154679223f8. --- pkg/config/config.go | 12 +-- pkg/config/config_test.go | 162 ++++++++++++++++++++++++++++++++------ 2 files changed, 137 insertions(+), 37 deletions(-) diff --git a/pkg/config/config.go b/pkg/config/config.go index bf22a8583..9911bcccf 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -584,18 +584,8 @@ func ReadConfigFile(fileName string) (common.Configuration, error) { } func ExtractConfigData(yamlData []byte) (common.Configuration, error) { - if len(yamlData) == 0 { - // For some reason, decoding an empty yaml file gives an error, but if - // we pass "---" (the yaml doc separator) parsing works fine. When we - // have an empty config, we want to use all the defaults instead of - // erroring. - yamlData = []byte("---") - } - var config common.Configuration - decoder := yaml.NewDecoder(bytes.NewReader(yamlData)) - decoder.KnownFields(true) // If an unknown field is read, decoding should error - err := decoder.Decode(&config) + err := yaml.Unmarshal(yamlData, &config) if err != nil { log.Errorf("ExtractConfigData: Error parsing yaml data: %v, err: %v", string(yamlData), err) return config, err diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 95ec8d092..738f3a0f2 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -31,9 +31,8 @@ import ( func Test_ExtractConfigData(t *testing.T) { flag.Parse() cases := []struct { - input []byte - shouldError bool - expected common.Configuration + input []byte + expected common.Configuration }{ { // case 1 - For correct input parameters and values []byte(` @@ -41,27 +40,37 @@ func Test_ExtractConfigData(t *testing.T) { queryListenIP: "0.0.0.0" ingestPort: 9090 eventTypeKeywords: ["utm_content"] + baseLogDir: "./pkg/ingestor/httpserver/" queryNode: true ingestNode: true - dataPath: "data/" + seedNode: true + segreaderNode: true + metareaderNode: true + DataPath: "data/" s3: enabled: true bucketName: "test-1" bucketPrefix: "" regionName: "us-east-1" retentionHours: 90 - timestampKey: "timestamp" + TimeStampKey: "timestamp" maxSegFileSize: 10 - licenseKeyPath: "./" + licensekeyPath: "./" esVersion: "6.8.20" + maxVirtualTables: 10_000 + logFileRotationSizeMB: 100 + compressLogFile: false dataDiskThresholdPercent: 85 memoryThresholdPercent: 80 + partitionCountConsistentHasher: 271 + replicationFactorConsistentHasher: 40 + loadConsistentHasher: 1.2 s3IngestQueueName: "" s3IngestQueueRegion: "" s3IngestBufferSize: 1000 maxParallelS3IngestBuffers: 10 queryHostname: "abc:123" - pqsEnabled: bad string + PQSEnabled: bad string analyticsEnabled: false agileAggsEnabled: false safeMode: true @@ -75,7 +84,6 @@ func Test_ExtractConfigData(t *testing.T) { compressLogFile: false compressStatic: false `), - false, common.Configuration{ IngestListenIP: "0.0.0.0", QueryListenIP: "0.0.0.0", @@ -118,28 +126,133 @@ func Test_ExtractConfigData(t *testing.T) { }, { // case 2 - For wrong input type, show error message []byte(` - memoryThresholdPercent: not a number + ingestListenIP: "0.0.0.0" + queryListenIP: "0.0.0.0" + ingestPort: 9090 + queryPort: 9000 + eventTypeKeywords: ["utm_content"] + queryNode: true + ingestNode: true + seedNode: true + segreaderNode: true + metareaderNode: true + idleWipFlushIntervalSecs: 1200 + maxWaitWipFlushIntervalSecs: 300 + DataPath: "data/" + retentionHours: 123 + TimeStampKey: "timestamp" + maxSegFileSize: 12345 + licensekeyPath: "./" + esVersion: "6.8.20" + dataDiskThresholdPercent: 85 + memoryThresholdPercent: 80 + partitionCountConsistentHasher: 271 + replicationFactorConsistentHasher: 40 + loadConsistentHasher: 1.2 + S3IngestQueueName: "" + S3IngestQueueRegion: "" + S3IngestBufferSize: 1000 + MaxParallelS3IngestBuffers: 10 + PQSEnabled: F + dualCaseCheck: true + analyticsEnabled: bad string + AgileAggsEnabled: bad string + tracing: + endpoint: "" + serviceName: "" + smaplingPercentage: bad string + log: + logPrefix: "./pkg/ingestor/httpserver/" + logFileRotationSizeMB: 1000 + compressLogFile: true + compressStatic: bad string `), - true, - common.Configuration{}, + + common.Configuration{ + IngestListenIP: "0.0.0.0", + QueryListenIP: "0.0.0.0", + IngestPort: 9090, + QueryPort: 9000, + IngestUrl: "http://localhost:9090", + EventTypeKeywords: []string{"utm_content"}, + QueryNode: "true", + IngestNode: "true", + IdleWipFlushIntervalSecs: 60, + MaxWaitWipFlushIntervalSecs: 60, + DataPath: "data/", + S3: common.S3Config{Enabled: false, BucketName: "", BucketPrefix: "", RegionName: ""}, + RetentionHours: 123, + TimeStampKey: "timestamp", + MaxSegFileSize: 12345, + LicenseKeyPath: "./", + ESVersion: "6.8.20", + DataDiskThresholdPercent: 85, + MemoryThresholdPercent: 80, + S3IngestQueueName: "", + S3IngestQueueRegion: "", + S3IngestBufferSize: 1000, + MaxParallelS3IngestBuffers: 10, + QueryHostname: "localhost:9000", + PQSEnabled: "true", + PQSEnabledConverted: true, + DualCaseCheck: "true", + DualCaseCheckConverted: true, + AnalyticsEnabled: "true", + AnalyticsEnabledConverted: true, + AgileAggsEnabled: "true", + AgileAggsEnabledConverted: true, + SafeServerStart: false, + Log: common.LogConfig{LogPrefix: "./pkg/ingestor/httpserver/", LogFileRotationSizeMB: 1000, CompressLogFile: true}, + CompressStatic: "true", + CompressStaticConverted: true, + Tracing: common.TracingConfig{Endpoint: "", ServiceName: "siglens", SamplingPercentage: 0}, + }, }, { // case 3 - Error out on bad yaml []byte(` invalid input, we should error out `), - true, - common.Configuration{}, + + common.Configuration{ + IngestListenIP: "0.0.0.0", + QueryListenIP: "0.0.0.0", + IngestPort: 8081, + QueryPort: 0, + IngestUrl: "http://localhost:8081", + EventTypeKeywords: []string{"eventType"}, + QueryNode: "true", + IngestNode: "true", + IdleWipFlushIntervalSecs: 5, + MaxWaitWipFlushIntervalSecs: 30, + DataPath: "data/", + S3: common.S3Config{Enabled: false, BucketName: "", BucketPrefix: "", RegionName: ""}, + RetentionHours: 90, + TimeStampKey: "timestamp", + MaxSegFileSize: 1_073_741_824, + LicenseKeyPath: "./", + ESVersion: "6.8.20", + DataDiskThresholdPercent: 85, + MemoryThresholdPercent: 80, + S3IngestQueueName: "", + S3IngestQueueRegion: "", + + S3IngestBufferSize: 1000, + MaxParallelS3IngestBuffers: 10, + QueryHostname: "localhost:5122", + AnalyticsEnabled: "true", + AnalyticsEnabledConverted: true, + AgileAggsEnabled: "true", + AgileAggsEnabledConverted: true, + DualCaseCheck: "true", + DualCaseCheckConverted: true, + Log: common.LogConfig{LogPrefix: "", LogFileRotationSizeMB: 100, CompressLogFile: false}, + Tracing: common.TracingConfig{Endpoint: "", ServiceName: "siglens", SamplingPercentage: 1}, + }, }, - { // case 4 - Error out on invalid fields + { // case 4 - For no input, pick defaults []byte(` -invalidField: "invalid" +a: b `), - true, - common.Configuration{}, - }, - { // case 5 - For no input, pick defaults - []byte(``), - false, common.Configuration{ IngestListenIP: "0.0.0.0", QueryListenIP: "0.0.0.0", @@ -183,13 +296,10 @@ invalidField: "invalid" } for i, test := range cases { actualConfig, err := ExtractConfigData(test.input) - if test.shouldError { - assert.Error(t, err, fmt.Sprintf("test=%v should have errored", i+1)) + if i == 2 { + assert.Error(t, err) continue - } else { - assert.NoError(t, err, fmt.Sprintf("test=%v should not have errored", i+1)) } - if segutils.ConvertUintBytesToMB(memory.TotalMemory()) < SIZE_8GB_IN_MB { assert.Equal(t, uint64(50), actualConfig.MemoryThresholdPercent) // If memory is less than 8GB, config by default returns 50% as the threshold