[FIX] How to handle prometheus remote-write case with influxdb-relay

.gitignore

@@ -3,3 +3,4 @@ build
 build/*
 influxdb-relay
 pkg
+.idea/

.gitlab-ci.yml

@@ -10,54 +10,33 @@ dockerlint:
   stage: linter
   image: vpgrp/linter:docker
   script:
-    - find . -type f -name "Dockerfile" | xargs -I{} dockerlint {}
+    - find . -type f -name "Dockerfile" | grep -v 'vendor' | xargs -I{} dockerlint {}
   allow_failure: true
 
 golint:
   stage: linter
   image: vpgrp/linter:golang
-  script:
+  before_script:
     - go get github.com/GeertJohan/fgt
-    - find . -type f -name "*\.go" | xargs -I{} fgt golint {}
-  allow_failure: true
-
-pylint:
-  stage: linter
-  image: vpgrp/linter:python
   script:
-    - find . -type f -name "*\.py" | xargs -I{} pylint {}
+    - find . -type f -name "*\.go" | grep -v 'vendor' | xargs -I{} fgt golint {}
   allow_failure: true
 
 # build
 go-get:
   stage: build
   image: vpgrp/golang:latest
-  allow_failure: true
   script:
     - go get -u github.com/vente-privee/influxdb-relay
 
 go-build:
   stage: build
   image: vpgrp/golang:latest
-  allow_failure: true
   before_script:
-    - apt-get update  -qq -y
-    - apt-get install -qq -y git libffi-dev make python python-boto rsync
-    - mkdir -p /go/src/github.com/vente-privee/influxdb-relay
-    - rsync -az /builds/noc/ /go/src/github.com/vente-privee
-  script:
-    - cd /go/src/github.com/vente-privee/influxdb-relay
-    - python /go/src/github.com/vente-privee/influxdb-relay/build.py
-
-docker-build:
-  stage: build
-  image: docker:latest
-  allow_failure: true
-  services:
-    - docker:dind
-  variables:
-    DOCKER_DRIVER: overlay
+    - mkdir -p ${GOPATH}/src/github.com/vente-privee
+    - ln -fsv ${CI_PROJECT_DIR} ${GOPATH}/src/github.com/vente-privee/influxdb-relay
   script:
-    - docker build --file Dockerfile.build --rm --tag influxdb-relay-builder:latest .
-    - docker run --rm --volume $(pwd):/go/src/github.com/vente-privee/influxdb-relay influxdb-relay-builder
+    - cd ${GOPATH}/src/github.com/vente-privee/influxdb-relay
+    - go build -a -ldflags '-extldflags "-static"' -o influxdb-relay
 # EOF
+

Gopkg.toml

@@ -29,10 +29,6 @@
   name = "github.com/influxdata/influxdb"
   version = "1.5.4"
 
-[[constraint]]
-  branch = "master"
-  name = "github.com/influxdata/influxdb-relay"
-
 [[constraint]]
   name = "github.com/naoina/toml"
   version = "0.1.1"

README.md

@@ -4,8 +4,8 @@
 
 Fork of [influxdb-relay](https://github.com/influxdata/influxdb-relay).
 
-This project adds  a basic high availability layer to  InfluxDB. With the right
-architecture and disaster recovery processes,  this achieves a highly available
+This project adds a basic high availability layer to InfluxDB. With the right
+architecture and disaster recovery processes, this achieves a highly available
 setup.
 
 *NOTE:* `influxdb-relay` must be built with Go 1.5+
@@ -34,6 +34,9 @@ name = "example-http"
 # TCP address to bind to, for HTTP server.
 bind-addr = "127.0.0.1:9096"
 
+# Ping response code, default is 204
+default-ping-response = 200
+
 # Enable HTTPS requests.
 ssl-combined-pem = "/etc/ssl/influxdb-relay.pem"
 
@@ -43,8 +46,11 @@ output = [
     # location: full URL of the /write endpoint of the backend
     # timeout: Go-parseable time duration. Fail writes if incomplete in this time.
     # skip-tls-verification: skip verification for HTTPS location. WARNING: it's insecure. Don't use in production.
+    # type: type of input source. OPTIONAL: see below for more information.
     { name="local1", location="http://127.0.0.1:8086/write", timeout="10s" },
     { name="local2", location="http://127.0.0.1:7086/write", timeout="10s" },
+    { name="local1_prom", location="http://127.0.0.1:8086/api/v1/prom/write", timeout="10s", type="prometheus" },
+    { name="local2_prom", location="http://127.0.0.1:7086/api/v1/prom/write", timeout="10s", type="prometheus" },
 ]
 
 [[udp]]
@@ -70,6 +76,17 @@ output = [
 ]
 ```
 
+## Types
+
+InfluxDB Relay is able to forward from a variety of input sources, including:
+
+  * `influxdb`
+  * `prometheus`
+
+The `type` parameter in the configuration file defaults to `influxdb`.
+
+*NOTE:* Types are not supported in UDP.
+
 ## Description
 
 The architecture is fairly simple and consists  of a load balancer, two or more

relay/config.go → config/config.go

@@ -1,18 +1,21 @@
-package relay
+package config
 
 import (
 	"os"
 
 	"github.com/naoina/toml"
 )
 
-// Config -TODO-
+// Config is an object created from a configuration file
+// It is a list of HTTP and/or UDP relays
+// Each relay has its own list of backends
 type Config struct {
 	HTTPRelays []HTTPConfig `toml:"http"`
 	UDPRelays  []UDPConfig  `toml:"udp"`
+	Verbose    bool
 }
 
-// HTTPConfig -TODO-
+// HTTPConfig represents an HTTP relay
 type HTTPConfig struct {
 	// Name identifies the HTTP relay
 	Name string `toml:"name"`
@@ -26,38 +29,45 @@ type HTTPConfig struct {
 	// Default retention policy to set for forwarded requests
 	DefaultRetentionPolicy string `toml:"default-retention-policy"`
 
+	DefaultPingResponse int `toml:"default-ping-response"`
+
 	// Outputs is a list of backed servers where writes will be forwarded
 	Outputs []HTTPOutputConfig `toml:"output"`
 }
 
-// HTTPOutputConfig -TODO-
+// HTTPOutputConfig represents the specification of an HTTP backend target
 type HTTPOutputConfig struct {
 	// Name of the backend server
 	Name string `toml:"name"`
 
 	// Location should be set to the URL of the backend server's write endpoint
 	Location string `toml:"location"`
 
-	// Timeout sets a per-backend timeout for write requests. (Default 10s)
+	// Timeout sets a per-backend timeout for write requests (default: 10s)
 	// The format used is the same seen in time.ParseDuration
 	Timeout string `toml:"timeout"`
 
-	// Buffer failed writes up to maximum count. (Default 0, retry/buffering disabled)
+	// Buffer failed writes up to maximum count (default: 0, retry/buffering disabled)
 	BufferSizeMB int `toml:"buffer-size-mb"`
 
-	// Maximum batch size in KB (Default 512)
+	// Maximum batch size in KB (default: 512)
 	MaxBatchKB int `toml:"max-batch-kb"`
 
-	// Maximum delay between retry attempts.
-	// The format used is the same seen in time.ParseDuration (Default 10s)
+	// Maximum delay between retry attempts
+	// The format used is the same seen in time.ParseDuration (default: 10s)
 	MaxDelayInterval string `toml:"max-delay-interval"`
 
-	// Skip TLS verification in order to use self signed certificate.
-	// WARNING: It's insecure. Use it only for developing and don't use in production.
+	// Skip TLS verification in order to use self signed certificate
+	// WARNING: It's insecure, use it only for developing and don't use in production
 	SkipTLSVerification bool `toml:"skip-tls-verification"`
+
+	// Where does the data come from ?
+	// This allows us to identify the data in the source code
+	// in order to apply a type-based treatment to it
+	InputType Input `toml:"type"`
 }
 
-// UDPConfig -TODO-
+// UDPConfig represents a UDP relay
 type UDPConfig struct {
 	// Name identifies the UDP relay
 	Name string `toml:"name"`
@@ -75,7 +85,7 @@ type UDPConfig struct {
 	Outputs []UDPOutputConfig `toml:"output"`
 }
 
-// UDPOutputConfig -TODO-
+// UDPOutputConfig represents the specification of a UDP backend target
 type UDPOutputConfig struct {
 	// Name identifies the UDP backend
 	Name string `toml:"name"`
@@ -88,12 +98,24 @@ type UDPOutputConfig struct {
 }
 
 // LoadConfigFile parses the specified file into a Config object
-func LoadConfigFile(filename string) (cfg Config, err error) {
+func LoadConfigFile(filename string) (Config, error) {
+	var cfg Config
+
 	f, err := os.Open(filename)
 	if err != nil {
 		return cfg, err
 	}
 	defer f.Close()
 
-	return cfg, toml.NewDecoder(f).Decode(&cfg)
+	if err = toml.NewDecoder(f).Decode(&cfg); err == nil {
+		for index, relay := range cfg.HTTPRelays {
+			for indexB, backend := range relay.Outputs {
+				if backend.InputType == "" {
+					cfg.HTTPRelays[index].Outputs[indexB].InputType = TypeInfluxdb
+				}
+			}
+		}
+	}
+
+	return cfg, err
 }

config/input.go

@@ -0,0 +1,11 @@
+package config
+
+// Input type of source
+type Input string
+
+const (
+	// TypeInfluxdb input type
+	TypeInfluxdb   Input = "influxdb"
+	// TypePrometheus input type
+	TypePrometheus Input = "prometheus"
+)

dev/Dockerfile.develop

@@ -0,0 +1,9 @@
+FROM golang
+
+WORKDIR /app
+
+COPY influxdb-relay .
+COPY test_env/test.toml .
+
+EXPOSE 9096
+CMD ["./influxdb-relay", "-config", "test.toml", "-v"]

dev/docker-compose.yml

@@ -0,0 +1,23 @@
+version: '3'
+
+services:
+  influxdb:
+    image: influxdb
+    ports:
+      - "8086:8086"
+    volumes:
+      - /tmp/influxdbrelay-influx:/var/lib/influxdb
+
+  prometheus:
+    image: prom/prometheus
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+
+  relay:
+    build:
+      context: ../
+      dockerfile: Dockerfile.develop
+    ports:
+      - "9096:9096"

dev/prepare_databases.sh

@@ -0,0 +1,10 @@
+#! /usr/bin/env bash
+
+curl -X POST "http://127.0.0.1:8086/query" --data-urlencode 'q=CREATE DATABASE NOT_prometheus'
+curl -X POST "http://127.0.0.1:8086/query" --data-urlencode 'q=CREATE DATABASE prometheus'
+
+# curl -X POST "http://127.0.0.1:8086/query" --data-urlencode 'db=prometheus' --data-urlencode 'q=SHOW SERIES'
+# curl -X POST "http://127.0.0.1:8086/query" --data-urlencode 'db=NOT_prometheus' --data-urlencode 'q=SHOW SERIES'
+
+# curl -X POST "http://127.0.0.1:9096/write?db=NOT_prometheus" --data-binary
+# 'cpu_load_short,host=server01,region=us-west value=0.64 1434055562000000000'

dev/prometheus.yml

@@ -0,0 +1,10 @@
+remote_write:
+  - url: "http://relay:9096/api/v1/prom/write?db=prometheus"
+
+scrape_configs:
+  - job_name: 'AUTOPARSE'
+    scrape_interval: 2s
+    static_configs:
+      - targets: ['localhost:9090']
+        labels:
+          group: 'production'

dev/test.toml

@@ -0,0 +1,7 @@
+[[http]]
+name = "test-config"
+bind-addr = "0.0.0.0:9096"
+output = [
+    { name="from_influx", location = "http://influxdb:8086/write" },
+    { name="from_prometheus", location = "http://influxdb:8086/api/v1/prom/write", type="prometheus" }
+]

sample.toml → examples/sample.toml

@@ -3,6 +3,7 @@
 [[http]]
 name = "example-http"
 bind-addr = "127.0.0.1:9096"
+default-ping-response = 204
 output = [
     { name="local1", location = "http://127.0.0.1:8086/write" },
     { name="local2", location = "http://127.0.0.1:7086/write" },