[dbnode] [coordinator] Refactor config to use defaults wherever possi…

…ble (#2839)

.golangci.yml

@@ -189,7 +189,6 @@ linters:
     - unconvert
     - unparam
     - varcheck
-    - wsl
   enable-all: false
   disable:
     - gomnd
@@ -208,7 +207,11 @@ linters:
     - nolintlint
     # Deprecated project due to being prone to bad suggestions.
     - interfacer
-    - unused
+    # Valid use for not explicitly setting every field when they are optional nil/empty.
+    - exhaustivestruct
+    # We allow cuddling assignment following conditions because there are valid 
+    # logical groupings for this use-case (e.g. when evaluating config values).
+    - wsl
   disable-all: false
   presets:
     # bodyclose, errcheck, gosec, govet, scopelint, staticcheck, typecheck

config/m3db/clustered-etcd/generated.yaml

@@ -73,4 +73,3 @@
     "sanitization": "prometheus"
   "writeNewSeriesAsync": true
   "writeNewSeriesBackoffDuration": "2ms"
-  "writeNewSeriesLimitPerSecond": 1048576

config/m3db/clustered-etcd/m3dbnode.libsonnet

@@ -101,7 +101,6 @@ function(cluster, coordinator={}, db={}) {
     },
     "gcPercentage": 100,
     "writeNewSeriesAsync": true,
-    "writeNewSeriesLimitPerSecond": 1048576,
     "writeNewSeriesBackoffDuration": "2ms",
     "cache": {
       "series": {

config/m3db/local-etcd/generated.yaml

@@ -68,4 +68,3 @@
     "sanitization": "prometheus"
   "writeNewSeriesAsync": true
   "writeNewSeriesBackoffDuration": "2ms"
-  "writeNewSeriesLimitPerSecond": 1048576

config/m3db/local-etcd/m3dbnode.libsonnet

@@ -60,7 +60,6 @@ function(coordinator={}, db={}) {
     },
     "gcPercentage": 100,
     "writeNewSeriesAsync": true,
-    "writeNewSeriesLimitPerSecond": 1048576,
     "writeNewSeriesBackoffDuration": "2ms",
     "cache": {
       "series": {

kube/m3dbnode-configmap.yaml

@@ -53,7 +53,6 @@ data:
       gcPercentage: 100
 
       writeNewSeriesAsync: true
-      writeNewSeriesLimitPerSecond: 1048576
       writeNewSeriesBackoffDuration: 2ms
 
       commitlog:

kube/terraform/main.tf

@@ -133,7 +133,7 @@ resource "kubernetes_config_map" "m3dbnode_config" {
     namespace = "m3db"
   }
   data {
-    m3dbnode.yml = "coordinator:\n  listenAddress: \"0.0.0.0:7201\"\n  local:\n    namespaces:\n        - namespace: default\n          type: unaggregated\n          retention: 48h\n  metrics:\n    scope:\n      prefix: \"coordinator\"\n    prometheus:\n      handlerPath: /metrics\n      listenAddress: 0.0.0.0:7203\n    sanitization: prometheus\n    samplingRate: 1.0\n    extended: none\n  tagOptions:\n    idScheme: quoted\n\ndb:\n  logging:\n    level: info\n\n  metrics:\n    prometheus:\n      handlerPath: /metrics\n    sanitization: prometheus\n    samplingRate: 1.0\n    extended: detailed\n\n  listenAddress: 0.0.0.0:9000\n  clusterListenAddress: 0.0.0.0:9001\n  httpNodeListenAddress: 0.0.0.0:9002\n  httpClusterListenAddress: 0.0.0.0:9003\n  debugListenAddress: 0.0.0.0:9004\n\n  hostID:\n    resolver: hostname\n\n  client:\n    writeConsistencyLevel: majority\n    readConsistencyLevel: unstrict_majority\n\n  gcPercentage: 100\n\n  writeNewSeriesAsync: true\n  writeNewSeriesLimitPerSecond: 1048576\n  writeNewSeriesBackoffDuration: 2ms\n\n  commitlog:\n    flushMaxBytes: 524288\n    flushEvery: 1s\n    queue:\n      calculationType: fixed\n      size: 2097152\n\n  filesystem:\n    filePathPrefix: /var/lib/m3db\n\n  config:\n    service:\n        env: default_env\n        zone: embedded\n        service: m3db\n        cacheDir: /var/lib/m3kv\n        etcdClusters:\n        - zone: embedded\n          endpoints:\n          - http://etcd-0.etcd:2379\n          - http://etcd-1.etcd:2379\n          - http://etcd-2.etcd:2379\n"
+    m3dbnode.yml = "coordinator:\n  listenAddress: \"0.0.0.0:7201\"\n  local:\n    namespaces:\n        - namespace: default\n          type: unaggregated\n          retention: 48h\n  metrics:\n    scope:\n      prefix: \"coordinator\"\n    prometheus:\n      handlerPath: /metrics\n      listenAddress: 0.0.0.0:7203\n    sanitization: prometheus\n    samplingRate: 1.0\n    extended: none\n  tagOptions:\n    idScheme: quoted\n\ndb:\n  logging:\n    level: info\n\n  metrics:\n    prometheus:\n      handlerPath: /metrics\n    sanitization: prometheus\n    samplingRate: 1.0\n    extended: detailed\n\n  listenAddress: 0.0.0.0:9000\n  clusterListenAddress: 0.0.0.0:9001\n  httpNodeListenAddress: 0.0.0.0:9002\n  httpClusterListenAddress: 0.0.0.0:9003\n  debugListenAddress: 0.0.0.0:9004\n\n  hostID:\n    resolver: hostname\n\n  client:\n    writeConsistencyLevel: majority\n    readConsistencyLevel: unstrict_majority\n\n  gcPercentage: 100\n\n  writeNewSeriesAsync: true\n  writeNewSeriesBackoffDuration: 2ms\n\n  commitlog:\n    flushMaxBytes: 524288\n    flushEvery: 1s\n    queue:\n      calculationType: fixed\n      size: 2097152\n\n  filesystem:\n    filePathPrefix: /var/lib/m3db\n\n  config:\n    service:\n        env: default_env\n        zone: embedded\n        service: m3db\n        cacheDir: /var/lib/m3kv\n        etcdClusters:\n        - zone: embedded\n          endpoints:\n          - http://etcd-0.etcd:2379\n          - http://etcd-1.etcd:2379\n          - http://etcd-2.etcd:2379\n"
   }
 }
 

scripts/development/m3_stack/m3dbnode.yml

@@ -1,7 +1,4 @@
 db:
-  logging:
-    level: info
-
   tracing:
     backend: jaeger
     jaeger:
@@ -11,49 +8,10 @@ db:
         type: const
         param: 1
 
-  metrics:
-    prometheus:
-      handlerPath: /metrics
-    sanitization: prometheus
-    samplingRate: 1.0
-    extended: detailed
-
-  listenAddress: 0.0.0.0:9000
-  clusterListenAddress: 0.0.0.0:9001
-  httpNodeListenAddress: 0.0.0.0:9002
-  httpClusterListenAddress: 0.0.0.0:9003
-  debugListenAddress: 0.0.0.0:9004
-
   hostID:
     resolver: environment
     envVarName: M3DB_HOST_ID
 
-  client:
-    writeConsistencyLevel: majority
-    readConsistencyLevel: unstrict_majority
-
-  gcPercentage: 100
-
-  writeNewSeriesAsync: true
-  writeNewSeriesLimitPerSecond: 1048576
-  writeNewSeriesBackoffDuration: 2ms
-
-  cache:
-    series:
-      policy: lru
-    postingsList:
-      size: 262144
-
-  commitlog:
-    flushMaxBytes: 524288
-    flushEvery: 1s
-    queue:
-      calculationType: fixed
-      size: 2097152
-
-  filesystem:
-    filePathPrefix: /var/lib/m3db
-
   config:
       service:
           env: default_env

scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/m3dbnode.yml

@@ -45,7 +45,6 @@ db:
   gcPercentage: 100
 
   writeNewSeriesAsync: true
-  writeNewSeriesLimitPerSecond: 1048576
   writeNewSeriesBackoffDuration: 2ms
 
   cache:

scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml

@@ -56,7 +56,6 @@ db:
   gcPercentage: 100
 
   writeNewSeriesAsync: true
-  writeNewSeriesLimitPerSecond: 1048576
   writeNewSeriesBackoffDuration: 2ms
 
   cache:

scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml

@@ -35,7 +35,6 @@ db:
   gcPercentage: 100
 
   writeNewSeriesAsync: true
-  writeNewSeriesLimitPerSecond: 1048576
   writeNewSeriesBackoffDuration: 2ms
 
   cache:

scripts/docker-integration-tests/repair/m3dbnode.yml

@@ -35,7 +35,6 @@ db:
   gcPercentage: 100
 
   writeNewSeriesAsync: true
-  writeNewSeriesLimitPerSecond: 1048576
   writeNewSeriesBackoffDuration: 2ms
 
   cache:

scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-a.yml

@@ -35,7 +35,6 @@ db:
   gcPercentage: 100
 
   writeNewSeriesAsync: true
-  writeNewSeriesLimitPerSecond: 1048576
   writeNewSeriesBackoffDuration: 2ms
 
   cache:

scripts/docker-integration-tests/repair_and_replication/m3dbnode-cluster-b.yml

@@ -35,7 +35,6 @@ db:
   gcPercentage: 100
 
   writeNewSeriesAsync: true
-  writeNewSeriesLimitPerSecond: 1048576
   writeNewSeriesBackoffDuration: 2ms
 
   cache:

scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml

@@ -35,7 +35,6 @@ db:
   gcPercentage: 100
 
   writeNewSeriesAsync: true
-  writeNewSeriesLimitPerSecond: 1048576
   writeNewSeriesBackoffDuration: 2ms
 
   cache:

scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml

@@ -35,7 +35,6 @@ db:
   gcPercentage: 100
 
   writeNewSeriesAsync: true
-  writeNewSeriesLimitPerSecond: 1048576
   writeNewSeriesBackoffDuration: 2ms
 
   cache:

site/content/docs/operational_guide/availability_consistency_durability.md

@@ -57,10 +57,12 @@ This instructs M3DB to handle writes for new timeseries (for a given time block)
 
 However, since new time series are created asynchronously, it's possible that there may be a brief delay inbetween when a write is acknowledged by the client and when that series becomes available for subsequent reads.
 
-M3DB also allows operators to rate limit the number of new series that can be created per second via the following configuration:
+M3DB also allows operators to rate limit the number of new series that can be created per second via the following configuration under the `db.limits` section:
 
 ```yaml
-writeNewSeriesLimitPerSecond: 1048576
+db:
+  limits:
+    writeNewSeriesPerSecond: 1048576
 ```
 
 This value can be set much lower than the default value for workloads in which a significant increase in cardinality usually indicates a misbehaving caller.
@@ -105,10 +107,12 @@ writeNewSeriesAsync: false
 
 This instructs M3DB to handle writes for new timeseries (for a given time block) synchronously. Creating a new timeseries in memory is much more expensive than simply appending a new write to an existing series, so this configuration could have an adverse effect on performance when many new timeseries are being inserted into M3DB concurrently.
 
-Since this operation is so expensive, M3DB allows operator to rate limit the number of new series that can be created per second via the following configuration (also a top-level key under the `db` section):
+Since this operation is so expensive, M3DB allows operator to rate limit the number of new series that can be created per second via the following configuration (also a top-level key under the `db.limits` section):
 
 ```yaml
-writeNewSeriesLimitPerSecond: 1048576
+db:
+  limits:
+    writeNewSeriesPerSecond: 1048576
 ```
 
 ### Ignoring Corrupt Commitlogs on Bootstrap