Added the ability to add remote_read and remote_write configurati…

…on sections through environment variables.

docs/config.md

@@ -61,14 +61,26 @@ SCRAPE_PORT_2=1234
 SERVICE_NAME_2=myservice2.acme.com
 ```
 
-As well you can add the service via `api` using the `reconfigure` entry point.
+You can also add a service via `api` using the `reconfigure` entry point.
 
 ```bash
 curl `[IP_OF_ONE_OF_SWARM_NODES]:8080/v1/docker-flow-monitor/reconfigure?scrapePort=[PORT]&serviceName=[IP_OR_DOMAIN]&scrapeType=static_configs
 ```
 
 Please consult [Prometheus Configuration](https://prometheus.io/docs/operating/configuration/) for more information about the available options.
 
+## Remote Read Configuration
+
+Environment variables prefixed with `REMOTE_READ_` are used instead Prometheus `remote_read` entries in the configuration.
+
+The formatting rules for the `REMOTE_READ` variables follow the same pattern as those used for [Global Configuration](#global-configuration)
+
+## Remote Write Configuration
+
+Environment variables prefixed with `REMOTE_WRITE_` are used instead Prometheus `remote_write` entries in the configuration.
+
+The formatting rules for the `REMOTE_WRITE` variables follow the same pattern as those used for [Global Configuration](#global-configuration)
+
 ## Scrapes
 
 Additional scrapes can be added through files prefixed with `scrape_`. By default, all such files located in `/run/secrets` are automatically added to the `scrape_configs` section of the configuration. The directory can be changed by setting a different value to the environment variable `CONFIGS_DIR`.

prometheus/config.go

@@ -6,42 +6,45 @@ import (
 	"os"
 	"strings"
 	"text/template"
+	"fmt"
 )
 
 // WriteConfig creates Prometheus configuration (`/etc/prometheus/prometheus.yml`) and rules (`/etc/prometheus/alert.rules`) files.
 func WriteConfig(scrapes map[string]Scrape, alerts map[string]Alert) {
 	FS.MkdirAll("/etc/prometheus", 0755)
-	gc := getGlobalConfig()
-	sc := getScrapeConfig(scrapes)
+	gc := GetGlobalConfig()
+	sc := GetScrapeConfig(scrapes)
+	rc := GetRemoteConfig()
 	ruleFiles := ""
 	if len(alerts) > 0 {
 		logPrintf("Writing to alert.rules")
 		ruleFiles = "\nrule_files:\n  - 'alert.rules'\n"
 		afero.WriteFile(FS, "/etc/prometheus/alert.rules", []byte(GetAlertConfig(alerts)), 0644)
 	}
-	config := gc + "\n" + sc + ruleFiles
+	config := gc + "\n" + sc + "\n" + rc + ruleFiles
 	logPrintf("Writing to prometheus.yml")
 	afero.WriteFile(FS, "/etc/prometheus/prometheus.yml", []byte(config), 0644)
 }
 
-func getGlobalConfig() string {
-	data := getGlobalConfigData()
-	config := `
-global:`
-	for key, values := range data {
-		if len(values[""]) > 0 {
-			config += "\n  " + key + ": " + values[""]
-		} else {
-			config += "\n  " + key + ":"
-			for subKey, value := range values {
-				config += "\n    " + subKey + ": " + value
-			}
-		}
-	}
+// GetRemoteConfig returns remote_write and remote_read configs
+func GetRemoteConfig() string {
+	rw := getDataFromEnvVars("REMOTE_WRITE")
+	config := getConfigSection("remote_write", rw)
+
+	rr := getDataFromEnvVars("REMOTE_READ")
+	config += getConfigSection("remote_read", rr)
+
 	return config
 }
 
-func getScrapeConfig(scrapes map[string]Scrape) string {
+// GetGlobalConfig returns global section of the configuration
+func GetGlobalConfig() string {
+	data := getDataFromEnvVars("GLOBAL")
+	return getConfigSection("global", data)
+}
+
+// GetScrapeConfig returns scrapes section of the configuration
+func GetScrapeConfig(scrapes map[string]Scrape) string {
 	config := getScrapeConfigFromMap(scrapes) + getScrapeConfigFromDir()
 	if len(config) > 0 {
 		if !strings.HasPrefix(config, "\n") {
@@ -53,10 +56,10 @@ scrape_configs:` + config
 	return config
 }
 
-func getGlobalConfigData() map[string]map[string]string {
+func getDataFromEnvVars(prefix string) map[string]map[string]string {
 	data := map[string]map[string]string{}
 	for _, e := range os.Environ() {
-		if key, value := getArgFromEnv(e, "GLOBAL"); len(key) > 0 {
+		if key, value := getArgFromEnv(e, prefix); len(key) > 0 {
 			realKey := key
 			subKey := ""
 			if strings.Contains(key, "-") {
@@ -122,3 +125,24 @@ func getScrapeConfigFromMap(scrapes map[string]Scrape) string {
 	}
 	return ""
 }
+
+func getConfigSection(section string, data map[string]map[string]string) string {
+	if len(data) == 0 {
+		return ""
+	}
+	config := fmt.Sprintf(`
+%s:`,
+		section,
+	)
+	for key, values := range data {
+		if len(values[""]) > 0 {
+			config += "\n  " + key + ": " + values[""]
+		} else {
+			config += "\n  " + key + ":"
+			for subKey, value := range values {
+				config += "\n    " + subKey + ": " + value
+			}
+		}
+	}
+	return config
+}

prometheus/config_test.go

@@ -23,6 +23,36 @@ func TestConfigUnitTestSuite(t *testing.T) {
 	suite.Run(t, s)
 }
 
+// GetRemoteConfig
+
+func (s *ConfigTestSuite) Test_GetRemoteConfig_ReturnsEmptyString_WhenEnvVarsAreNotSet() {
+	actual := GetRemoteConfig()
+
+	s.Empty(actual)
+}
+
+func (s *ConfigTestSuite) Test_GetRemoteConfig_ReturnsRemoteWriteUrl() {
+	defer func() { os.Unsetenv("REMOTE_WRITE_URL") }()
+	os.Setenv("REMOTE_WRITE_URL", "http://acme.com/write")
+	expected := `
+remote_write:
+  url: http://acme.com/write`
+	actual := GetRemoteConfig()
+
+	s.Equal(expected, actual)
+}
+
+func (s *ConfigTestSuite) Test_GetRemoteConfig_ReturnsRemoteReadUrl() {
+	defer func() { os.Unsetenv("REMOTE_READ_URL") }()
+	os.Setenv("REMOTE_READ_URL", "http://acme.com/read")
+	expected := `
+remote_read:
+  url: http://acme.com/read`
+	actual := GetRemoteConfig()
+
+	s.Equal(expected, actual)
+}
+
 // GetGlobalConfig
 
 func (s *ConfigTestSuite) Test_GlobalConfig_ReturnsConfigWithData() {
@@ -33,7 +63,7 @@ func (s *ConfigTestSuite) Test_GlobalConfig_ReturnsConfigWithData() {
 global:
   scrape_interval: 123s`
 
-	actual := getGlobalConfig()
+	actual := GetGlobalConfig()
 	s.Equal(expected, actual)
 }
 
@@ -57,7 +87,7 @@ global:
 
 	// Because of ordering, the config is not always the same so we're repeating a failure for a few times.
 	for i := 0; i < 5; i++ {
-		actual = getGlobalConfig()
+		actual = GetGlobalConfig()
 		if actual == expected {
 			return
 		}
@@ -91,7 +121,7 @@ scrape_configs:
 		"service-3": {ServiceName: "service-3", ScrapePort: 4321, ScrapeType: "static_configs"},
 	}
 
-	actual := getScrapeConfig(scrapes)
+	actual := GetScrapeConfig(scrapes)
 
 	s.Equal(expected, actual)
 }
@@ -128,7 +158,7 @@ scrape_configs:
 	afero.WriteFile(FS, "/run/secrets/scrape_job2", []byte(job2), 0644)
 	afero.WriteFile(FS, "/run/secrets/scrape_job3", []byte(job3), 0644)
 
-	actual := getScrapeConfig(scrapes)
+	actual := GetScrapeConfig(scrapes)
 
 	s.Equal(expected, actual)
 }
@@ -152,7 +182,7 @@ scrape_configs:
 	afero.WriteFile(FS, "/run/secrets/scrape_job", []byte(job), 0644)
 	afero.WriteFile(FS, "/run/secrets/job_without_scrape_prefix", []byte("something silly"), 0644)
 
-	actual := getScrapeConfig(scrapes)
+	actual := GetScrapeConfig(scrapes)
 
 	s.Equal(expected, actual)
 }
@@ -177,13 +207,13 @@ scrape_configs:
 	scrapes := map[string]Scrape{}
 	afero.WriteFile(FS, "/tmp/scrape_job", []byte(job), 0644)
 
-	actual := getScrapeConfig(scrapes)
+	actual := GetScrapeConfig(scrapes)
 
 	s.Equal(expected, actual)
 }
 
 func (s *ConfigTestSuite) Test_GetScrapeConfig_ReturnsEmptyString_WhenNoData() {
-	actual := getScrapeConfig(map[string]Scrape{})
+	actual := GetScrapeConfig(map[string]Scrape{})
 
 	s.Empty(actual)
 }
@@ -192,24 +222,37 @@ func (s *ConfigTestSuite) Test_GetScrapeConfig_ReturnsEmptyString_WhenNoData() {
 
 func (s *ConfigTestSuite) Test_WriteConfig_WritesConfig() {
 	fsOrig := FS
-	defer func() { FS = fsOrig }()
+	defer func() {
+		FS = fsOrig
+		os.Unsetenv("REMOTE_WRITE_URL")
+		os.Unsetenv("REMOTE_READ_URL")
+	}()
+	os.Setenv("REMOTE_WRITE_URL", "http://acme.com/write")
+	os.Setenv("REMOTE_READ_URL", "http://acme.com/read")
 	FS = afero.NewMemMapFs()
 	scrapes := map[string]Scrape{
 		"service-1": {ServiceName: "service-1", ScrapePort: 1234},
 		"service-2": {ServiceName: "service-2", ScrapePort: 5678},
 	}
-	gc := getGlobalConfig()
-	sc := getScrapeConfig(scrapes)
+	gc := GetGlobalConfig()
+	sc := GetScrapeConfig(scrapes)
+	rc := GetRemoteConfig()
 	expected := fmt.Sprintf(`%s
+%s
 %s`,
 		gc,
 		sc,
+		rc,
 	)
+	println("000")
+	println(rc)
+	println("111")
 
 	WriteConfig(scrapes, map[string]Alert{})
 
 	actual, _ := afero.ReadFile(FS, "/etc/prometheus/prometheus.yml")
 	s.Equal(expected, string(actual))
+//	s.Fail(string(actual))
 }
 
 func (s *ConfigTestSuite) Test_WriteConfig_WritesAlerts() {
@@ -223,9 +266,10 @@ func (s *ConfigTestSuite) Test_WriteConfig_WritesAlerts() {
 		AlertNameFormatted: "myservicealertname",
 		AlertIf:            "a>b",
 	}
-	gc := getGlobalConfig()
+	gc := GetGlobalConfig()
 	expectedConfig := fmt.Sprintf(`%s
 
+
 rule_files:
   - 'alert.rules'
 `,

server/server_test.go

@@ -97,6 +97,7 @@ func (s *ServerTestSuite) Test_Execute_WritesConfig() {
 	expected := `
 global:
   scrape_interval: 5s
+
 `
 	fsOrig := prometheus.FS
 	defer func() { prometheus.FS = fsOrig }()
@@ -547,6 +548,7 @@ scrape_configs:
         type: A
         port: 1234
 
+
 rule_files:
   - 'alert.rules'
 `
@@ -746,6 +748,7 @@ scrape_configs:
       - names: ["tasks.my-service"]
         type: A
         port: 1234
+
 `
 	rwMock := ResponseWriterMock{}
 	addr := "/v1/docker-flow-monitor?serviceName=my-service&scrapePort=1234"
@@ -763,6 +766,7 @@ scrape_configs:
 	expectedAfterDelete := `
 global:
   scrape_interval: 5s
+
 `
 	addr = "/v1/docker-flow-monitor?serviceName=my-service"
 	req, _ = http.NewRequest("DELETE", addr, nil)

stacks/exporters-aws.yml

@@ -51,7 +51,6 @@ services:
       - /:/rootfs
       - /etc/hostname:/etc/host_hostname
     deploy:
-      mode: global
       labels:
         - com.df.notify=true
         - com.df.scrapePort=9100
@@ -68,6 +67,7 @@ services:
       placement:
         constraints:
           - node.role == manager
+      mode: global
       resources:
         reservations:
           memory: 30M
@@ -88,7 +88,6 @@ services:
       - /:/rootfs
       - /etc/hostname:/etc/host_hostname
     deploy:
-      mode: global
       labels:
         - com.df.notify=true
         - com.df.scrapePort=9100
@@ -107,6 +106,7 @@ services:
       placement:
         constraints:
           - node.role == worker
+      mode: global
       resources:
         reservations:
           memory: 30M

stacks/go-demo-aws.yml

@@ -34,7 +34,7 @@ services:
         - com.df.alertIf.3=@resp_time_server_error:5m,0.001
         - com.df.alertName.4=replicas_running
         - com.df.alertIf.4=@replicas_running
-        - com.df.alertFor.4=10m
+        - com.df.alertFor.4=1m
       resources:
         reservations:
           memory: 5M

stacks/jenkins-aws-secret.yml

@@ -44,9 +44,9 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     secrets:
+      - aws
       - jenkins-user
       - jenkins-pass
-      - aws
     deploy:
       placement:
         constraints: [node.role == manager]
@@ -63,12 +63,12 @@ networks:
     external: false
 
 secrets:
+  aws:
+    external: true
   jenkins-user:
     external: true
   jenkins-pass:
     external: true
-  aws:
-    external: true
 
 volumes:
   master: