Merge pull request #470 from go-graphite/carbonserer/quota-friendly-f…

…ile-list-cache

carbonserver: introduce file list cache v2

README.md

@@ -377,7 +377,26 @@ trie-index = false
 # most of the indexing time if it contains a high number of metrics (10
 # - 40 millions). go-carbon only reads the cached file list once after
 # reboot and the cache result is updated after every scan. (EXPERIMENTAL)
-file-list-cache = ""
+#
+# file-list-cache = "/var/lib/carbon/carbonserver-file-list-cache.bin"
+
+# Supported FLC (file list cache) version values are: 2, 1 (default value is 1
+# for backward compatibility).
+#
+# File list cache v2 is is more advanced of file list cache for go-carbon, with
+# good quota support during restarts (by keeping physical, logical sizes, metric
+# creation time in the cache). For more details, see carbonserver/flc.go.
+#
+# V2 file is no longer a plain text file compressed with gzip, but a semi-binary
+# file. For reading flc v2 cache file, use go-carbon with flag -print-file-list-cache:
+#
+#     go-carbon -print-file-list-cache /var/lib/carbon/carbonserver-file-list-cache.bin
+#
+# For upgrade or downgrade to v2 or v1, you can just update the config,
+# go-carbon would transparently detect the existing cache file on restart. No
+# manual deletion needed.
+#
+# file-list-cache-version = 2
 
 # Enable concurrently building index without maintaining a new copy
 # index structure. More memory efficient.

carbon/app.go

@@ -502,6 +502,7 @@ func (app *App) Start(version string) (err error) {
 		carbonserver.SetTrieIndex(conf.Carbonserver.TrieIndex)
 		carbonserver.SetConcurrentIndex(conf.Carbonserver.ConcurrentIndex)
 		carbonserver.SetFileListCache(conf.Carbonserver.FileListCache)
+		carbonserver.SetFileListCacheVersion(conf.Carbonserver.FileListCacheVersion)
 		carbonserver.SetInternalStatsDir(conf.Carbonserver.InternalStatsDir)
 		carbonserver.SetPercentiles(conf.Carbonserver.Percentiles)
 		// carbonserver.SetQueryTimeout(conf.Carbonserver.QueryTimeout.Value())

carbon/config.go

@@ -126,10 +126,12 @@ type carbonserverConfig struct {
 	MaxMetricsGlobbed  int `toml:"max-metrics-globbed"`
 	MaxMetricsRendered int `toml:"max-metrics-rendered"`
 
-	TrieIndex       bool   `toml:"trie-index"`
-	ConcurrentIndex bool   `toml:"concurrent-index"`
-	RealtimeIndex   int    `toml:"realtime-index"`
-	FileListCache   string `toml:"file-list-cache"`
+	TrieIndex       bool `toml:"trie-index"`
+	ConcurrentIndex bool `toml:"concurrent-index"`
+	RealtimeIndex   int  `toml:"realtime-index"`
+
+	FileListCache        string `toml:"file-list-cache"`
+	FileListCacheVersion int    `toml:"file-list-cache-version"`
 
 	QuotaUsageReportFrequency *Duration `toml:"quota-usage-report-frequency"`
 
@@ -259,13 +261,14 @@ func NewConfig() *Config {
 			WriteTimeout: &Duration{
 				Duration: 60 * time.Second,
 			},
-			QueryCacheEnabled:  true,
-			QueryCacheSizeMB:   0,
-			FindCacheEnabled:   true,
-			TrigramIndex:       true,
-			CacheScan:          false,
-			MaxMetricsGlobbed:  10_000_000,
-			MaxMetricsRendered: 1_000_000,
+			QueryCacheEnabled:    true,
+			QueryCacheSizeMB:     0,
+			FindCacheEnabled:     true,
+			TrigramIndex:         true,
+			CacheScan:            false,
+			MaxMetricsGlobbed:    10_000_000,
+			MaxMetricsRendered:   1_000_000,
+			FileListCacheVersion: 1, // see carbonserver/flc.go
 		},
 		Carbonlink: carbonlinkConfig{
 			Listen:  "127.0.0.1:7002",