Skip to content

Commit

Permalink
Revert "Disallow unknown fields in server.yaml (siglens#1682)" (sigle…
Browse files Browse the repository at this point in the history 
…ns#1684)

This reverts commit d6c2f9a.
  • Loading branch information
@AndrewHess
AndrewHess authored Sep 26, 2024
1 parent bf75587 commit 4bc0c62
Show file tree
Hide file tree
Showing 2 changed files with 137 additions and 37 deletions.
12 changes: 1 addition & 11 deletions pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -584,18 +584,8 @@ func ReadConfigFile(fileName string) (common.Configuration, error) {
}

func ExtractConfigData(yamlData []byte) (common.Configuration, error) {
if len(yamlData) == 0 {
// For some reason, decoding an empty yaml file gives an error, but if
// we pass "---" (the yaml doc separator) parsing works fine. When we
// have an empty config, we want to use all the defaults instead of
// erroring.
yamlData = []byte("---")
}

var config common.Configuration
decoder := yaml.NewDecoder(bytes.NewReader(yamlData))
decoder.KnownFields(true) // If an unknown field is read, decoding should error
err := decoder.Decode(&config)
err := yaml.Unmarshal(yamlData, &config)
if err != nil {
log.Errorf("ExtractConfigData: Error parsing yaml data: %v, err: %v", string(yamlData), err)
return config, err
Expand Down
162 changes: 136 additions & 26 deletions pkg/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,37 +31,46 @@ import (
func Test_ExtractConfigData(t *testing.T) {
flag.Parse()
cases := []struct {
input []byte
shouldError bool
expected common.Configuration
input []byte
expected common.Configuration
}{
{ // case 1 - For correct input parameters and values
[]byte(`
ingestListenIP: "0.0.0.0"
queryListenIP: "0.0.0.0"
ingestPort: 9090
eventTypeKeywords: ["utm_content"]
baseLogDir: "./pkg/ingestor/httpserver/"
queryNode: true
ingestNode: true
dataPath: "data/"
seedNode: true
segreaderNode: true
metareaderNode: true
DataPath: "data/"
s3:
enabled: true
bucketName: "test-1"
bucketPrefix: ""
regionName: "us-east-1"
retentionHours: 90
timestampKey: "timestamp"
TimeStampKey: "timestamp"
maxSegFileSize: 10
licenseKeyPath: "./"
licensekeyPath: "./"
esVersion: "6.8.20"
maxVirtualTables: 10_000
logFileRotationSizeMB: 100
compressLogFile: false
dataDiskThresholdPercent: 85
memoryThresholdPercent: 80
partitionCountConsistentHasher: 271
replicationFactorConsistentHasher: 40
loadConsistentHasher: 1.2
s3IngestQueueName: ""
s3IngestQueueRegion: ""
s3IngestBufferSize: 1000
maxParallelS3IngestBuffers: 10
queryHostname: "abc:123"
pqsEnabled: bad string
PQSEnabled: bad string
analyticsEnabled: false
agileAggsEnabled: false
safeMode: true
Expand All @@ -75,7 +84,6 @@ func Test_ExtractConfigData(t *testing.T) {
compressLogFile: false
compressStatic: false
`),
false,
common.Configuration{
IngestListenIP: "0.0.0.0",
QueryListenIP: "0.0.0.0",
Expand Down Expand Up @@ -118,28 +126,133 @@ func Test_ExtractConfigData(t *testing.T) {
},
{ // case 2 - For wrong input type, show error message
[]byte(`
memoryThresholdPercent: not a number
ingestListenIP: "0.0.0.0"
queryListenIP: "0.0.0.0"
ingestPort: 9090
queryPort: 9000
eventTypeKeywords: ["utm_content"]
queryNode: true
ingestNode: true
seedNode: true
segreaderNode: true
metareaderNode: true
idleWipFlushIntervalSecs: 1200
maxWaitWipFlushIntervalSecs: 300
DataPath: "data/"
retentionHours: 123
TimeStampKey: "timestamp"
maxSegFileSize: 12345
licensekeyPath: "./"
esVersion: "6.8.20"
dataDiskThresholdPercent: 85
memoryThresholdPercent: 80
partitionCountConsistentHasher: 271
replicationFactorConsistentHasher: 40
loadConsistentHasher: 1.2
S3IngestQueueName: ""
S3IngestQueueRegion: ""
S3IngestBufferSize: 1000
MaxParallelS3IngestBuffers: 10
PQSEnabled: F
dualCaseCheck: true
analyticsEnabled: bad string
AgileAggsEnabled: bad string
tracing:
endpoint: ""
serviceName: ""
smaplingPercentage: bad string
log:
logPrefix: "./pkg/ingestor/httpserver/"
logFileRotationSizeMB: 1000
compressLogFile: true
compressStatic: bad string
`),
true,
common.Configuration{},

common.Configuration{
IngestListenIP: "0.0.0.0",
QueryListenIP: "0.0.0.0",
IngestPort: 9090,
QueryPort: 9000,
IngestUrl: "http://localhost:9090",
EventTypeKeywords: []string{"utm_content"},
QueryNode: "true",
IngestNode: "true",
IdleWipFlushIntervalSecs: 60,
MaxWaitWipFlushIntervalSecs: 60,
DataPath: "data/",
S3: common.S3Config{Enabled: false, BucketName: "", BucketPrefix: "", RegionName: ""},
RetentionHours: 123,
TimeStampKey: "timestamp",
MaxSegFileSize: 12345,
LicenseKeyPath: "./",
ESVersion: "6.8.20",
DataDiskThresholdPercent: 85,
MemoryThresholdPercent: 80,
S3IngestQueueName: "",
S3IngestQueueRegion: "",
S3IngestBufferSize: 1000,
MaxParallelS3IngestBuffers: 10,
QueryHostname: "localhost:9000",
PQSEnabled: "true",
PQSEnabledConverted: true,
DualCaseCheck: "true",
DualCaseCheckConverted: true,
AnalyticsEnabled: "true",
AnalyticsEnabledConverted: true,
AgileAggsEnabled: "true",
AgileAggsEnabledConverted: true,
SafeServerStart: false,
Log: common.LogConfig{LogPrefix: "./pkg/ingestor/httpserver/", LogFileRotationSizeMB: 1000, CompressLogFile: true},
CompressStatic: "true",
CompressStaticConverted: true,
Tracing: common.TracingConfig{Endpoint: "", ServiceName: "siglens", SamplingPercentage: 0},
},
},
{ // case 3 - Error out on bad yaml
[]byte(`
invalid input, we should error out
`),
true,
common.Configuration{},

common.Configuration{
IngestListenIP: "0.0.0.0",
QueryListenIP: "0.0.0.0",
IngestPort: 8081,
QueryPort: 0,
IngestUrl: "http://localhost:8081",
EventTypeKeywords: []string{"eventType"},
QueryNode: "true",
IngestNode: "true",
IdleWipFlushIntervalSecs: 5,
MaxWaitWipFlushIntervalSecs: 30,
DataPath: "data/",
S3: common.S3Config{Enabled: false, BucketName: "", BucketPrefix: "", RegionName: ""},
RetentionHours: 90,
TimeStampKey: "timestamp",
MaxSegFileSize: 1_073_741_824,
LicenseKeyPath: "./",
ESVersion: "6.8.20",
DataDiskThresholdPercent: 85,
MemoryThresholdPercent: 80,
S3IngestQueueName: "",
S3IngestQueueRegion: "",

S3IngestBufferSize: 1000,
MaxParallelS3IngestBuffers: 10,
QueryHostname: "localhost:5122",
AnalyticsEnabled: "true",
AnalyticsEnabledConverted: true,
AgileAggsEnabled: "true",
AgileAggsEnabledConverted: true,
DualCaseCheck: "true",
DualCaseCheckConverted: true,
Log: common.LogConfig{LogPrefix: "", LogFileRotationSizeMB: 100, CompressLogFile: false},
Tracing: common.TracingConfig{Endpoint: "", ServiceName: "siglens", SamplingPercentage: 1},
},
},
{ // case 4 - Error out on invalid fields
{ // case 4 - For no input, pick defaults
[]byte(`
invalidField: "invalid"
a: b
`),
true,
common.Configuration{},
},
{ // case 5 - For no input, pick defaults
[]byte(``),
false,
common.Configuration{
IngestListenIP: "0.0.0.0",
QueryListenIP: "0.0.0.0",
Expand Down Expand Up @@ -183,13 +296,10 @@ invalidField: "invalid"
}
for i, test := range cases {
actualConfig, err := ExtractConfigData(test.input)
if test.shouldError {
assert.Error(t, err, fmt.Sprintf("test=%v should have errored", i+1))
if i == 2 {
assert.Error(t, err)
continue
} else {
assert.NoError(t, err, fmt.Sprintf("test=%v should not have errored", i+1))
}

if segutils.ConvertUintBytesToMB(memory.TotalMemory()) < SIZE_8GB_IN_MB {
assert.Equal(t, uint64(50), actualConfig.MemoryThresholdPercent)
// If memory is less than 8GB, config by default returns 50% as the threshold
Expand Down

0 comments on commit 4bc0c62

Please sign in to comment.