diff --git a/Dockerfile b/Dockerfile index 9fcb7f9..bfaf03a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,12 +11,6 @@ RUN \ apk upgrade --no-cache && \ apk add g++ git make musl-dev cairo-dev -# Install Grafana - -RUN mkdir /tmp/grafana \ - && wget -P /tmp/ https://dl.grafana.com/oss/release/grafana-${GRAFANA_VERSION}.linux-amd64.tar.gz \ - && tar xfz /tmp/grafana-${GRAFANA_VERSION}.linux-amd64.tar.gz --strip-components=1 -C /tmp/grafana - # Install go-carbon WORKDIR ${GOPATH} @@ -57,17 +51,22 @@ RUN \ make && \ mv carbonapi /tmp/carbonapi +# ------------------------------ BRUBECK -------------------------------------- +FROM alpine:3.8 AS brubeck-builder + +RUN \ + apk update --no-cache && \ + apk upgrade --no-cache && \ + apk add g++ git make musl-dev jansson-dev openssl-dev libmicrohttpd-dev git jq + +RUN git clone https://github.com/github/brubeck.git + +RUN cd brubeck && ./script/bootstrap # ------------------------------ RUN IMAGE -------------------------------------- -FROM alpine:3.13.2 +FROM alpine:3.8 -ENV TZ='Europe/Amsterdam' +ENV TZ='Etc/UTC' -COPY --from=builder /tmp/grafana/bin/grafana-cli /usr/bin/grafana-cli -COPY --from=builder /tmp/grafana/bin/grafana-server /usr/sbin/grafana-server -COPY --from=builder /tmp/grafana/conf /usr/share/grafana/conf -COPY --from=builder /tmp/grafana/public /usr/share/grafana/public -COPY --from=builder /tmp/grafana/plugins-bundled /usr/share/grafana/plugins-bundled -COPY --from=builder /tmp/grafana/scripts /usr/share/grafana/scripts COPY --from=builder /tmp/go-carbon /usr/bin/go-carbon COPY --from=builder /tmp/carbonapi /usr/bin/carbonapi @@ -80,16 +79,17 @@ RUN \ cairo \ shadow \ tzdata \ - nginx \ runit \ dcron \ logrotate \ libc6-compat \ ca-certificates \ su-exec \ - bash \ - && rm -rf \ - /etc/nginx/conf.d/default.conf /etc/nginx/sites-enabled/default && \ + openssl \ + jansson \ + libmicrohttpd \ + jq \ + bash && \ cp "/usr/share/zoneinfo/${TZ}" /etc/localtime && \ echo "${TZ}" > /etc/timezone && \ /usr/sbin/useradd \ @@ -105,12 +105,18 @@ RUN \ /tmp/* \ /var/cache/apk/* +COPY --from=brubeck-builder /brubeck/brubeck /bin/brubeck + +ADD conf/config.template.json /config.template.json + +ADD conf/generate_config.sh /bin/generate_config.sh + WORKDIR / -VOLUME ["/etc/go-carbon", "/etc/carbonapi", "/var/lib/graphite", "/etc/nginx", "/etc/grafana", "/etc/logrotate.d", "/var/log"] +VOLUME ["/etc/go-carbon", "/etc/carbonapi", "/var/lib/graphite", "/etc/logrotate.d", "/var/log"] ENV HOME /root -EXPOSE 80 2003 2003/udp 2004 8080 8081 +EXPOSE 80 2003 2003/udp 2004 8080 8081 8125 CMD ["/entrypoint.sh"] diff --git a/README.md b/README.md index e935053..86dc7eb 100644 --- a/README.md +++ b/README.md @@ -7,24 +7,25 @@ docker run -d\ --name go-graphite\ --restart=always\ -p 80:80\ + -p 8080:8081\ + -p 8125:8125/udp\ -p 2003-2004:2003-2004\ gographite/go-graphite ``` ### Includes the following components -* [Nginx](http://nginx.org/) - reverse proxies Grafana dashboard -* [Grafana](http://www.grafana.com/) - front-end dashboard * [Go-carbon](https://github.com/lomik/go-carbon) - Golang implementation of Graphite/Carbon server * [Carbonapi](https://github.com/go-graphite/carbonapi) - Golang implementation of Graphite-web +* [Brubeck](https://github.com/github/brubeck) - C implementation Statsd ### Mapped Ports Host | Container | Service ---- | --------- | ------------------------------------------------------------------------------------------------------------------- - 80 | 80 | [grafana](http://docs.grafana.org/) 2003 | 2003 | [carbon receiver - plaintext](http://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-plaintext-protocol) 2004 | 2004 | [carbon receiver - pickle](http://graphite.readthedocs.io/en/latest/feeding-carbon.html#the-pickle-protocol) +8125 | 8125 | [statsd receiver - udp](https://github.com/b/statsd_spec) ### Exposed Ports @@ -38,8 +39,6 @@ Host | Container | Notes ----------------- | -------------------------- | ------------------------------- DOCKER ASSIGNED | /etc/go-carbon | go-carbon configs (see ) DOCKER ASSIGNED | /var/lib/graphite | graphite file storage -DOCKER ASSIGNED | /etc/nginx | nginx config -DOCKER ASSIGNED | /etc/grafana | Grafana config DOCKER ASSIGNED | /etc/carbonapi | Carbonapi config DOCKER ASSIGNED | /etc/logrotate.d | logrotate config DOCKER ASSIGNED | /var/log | log files diff --git a/conf/config.template.json b/conf/config.template.json new file mode 100644 index 0000000..a484058 --- /dev/null +++ b/conf/config.template.json @@ -0,0 +1,27 @@ +{ + "logfile" : "./log/brubeck.log", + "sharding" : false, + "server_name" : "brubeck_debug", + "dumpfile" : "./brubeck.dump", + "capacity" : 15, + "expire" : 5, + "http" : ":9090", + + "backends" : [ + { + "type" : "carbon", + "address" : "0.0.0.0", + "port" : 2003, + "frequency" : 10 + } + ], + + "samplers" : [ + { + "type" : "statsd", + "address" : "0.0.0.0", + "port" : 8125, + "workers" : 4 + } + ] +} diff --git a/conf/etc/grafana/grafana.ini b/conf/etc/grafana/grafana.ini deleted file mode 100644 index b863f26..0000000 --- a/conf/etc/grafana/grafana.ini +++ /dev/null @@ -1,407 +0,0 @@ -##################### Grafana Configuration Example ##################### -# -# Everything has defaults so you only need to uncomment things you want to -# change - -# possible values : production, development -; app_mode = production - -# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty -; instance_name = ${HOSTNAME} - -#################################### Paths #################################### -[paths] -# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) -# -;data = /var/lib/grafana -# -# Directory where grafana can store logs -# -;logs = /var/log/grafana -# -# Directory where grafana will automatically scan and look for plugins -# -;plugins = /var/lib/grafana/plugins - -# -#################################### Server #################################### -[server] -# Protocol (http, https, socket) -;protocol = http - -# The ip address to bind to, empty will bind to all interfaces -;http_addr = - -# The http port to use -;http_port = 3000 - -# The public facing domain name used to access grafana from a browser -;domain = localhost - -# Redirect to correct domain if host header does not match domain -# Prevents DNS rebinding attacks -;enforce_domain = false - -# The full public facing url you use in browser, used for redirects and emails -# If you use reverse proxy and sub path specify full url (with sub path) -;root_url = http://localhost:3000 - -# Log web requests -;router_logging = false - -# the path relative working path -;static_root_path = public - -# enable gzip -;enable_gzip = false - -# https certs & key file -;cert_file = -;cert_key = - -# Unix socket path -;socket = - -#################################### Database #################################### -[database] -# You can configure the database connection by specifying type, host, name, user and password -# as seperate properties or as on string using the url propertie. - -# Either "mysql", "postgres" or "sqlite3", it's your choice -;type = sqlite3 -;host = 127.0.0.1:3306 -;name = grafana -;user = root -# If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;""" -;password = - -# Use either URL or the previous fields to configure the database -# Example: mysql://user:secret@host:port/database -;url = - -# For "postgres" only, either "disable", "require" or "verify-full" -;ssl_mode = disable - -# For "sqlite3" only, path relative to data_path setting -;path = grafana.db - -# Max conn setting default is 0 (mean not set) -;max_idle_conn = -;max_open_conn = - - -#################################### Session #################################### -[session] -# Either "memory", "file", "redis", "mysql", "postgres", default is "file" -;provider = file - -# Provider config options -# memory: not have any config yet -# file: session dir path, is relative to grafana data_path -# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=grafana` -# mysql: go-sql-driver/mysql dsn config string, e.g. `user:password@tcp(127.0.0.1:3306)/database_name` -# postgres: user=a password=b host=localhost port=5432 dbname=c sslmode=disable -;provider_config = sessions - -# Session cookie name -;cookie_name = grafana_sess - -# If you use session in https only, default is false -;cookie_secure = false - -# Session life time, default is 86400 -;session_life_time = 86400 - -#################################### Data proxy ########################### -[dataproxy] - -# This enables data proxy logging, default is false -;logging = false - - -#################################### Analytics #################################### -[analytics] -# Server reporting, sends usage counters to stats.grafana.org every 24 hours. -# No ip addresses are being tracked, only simple counters to track -# running instances, dashboard and error counts. It is very helpful to us. -# Change this option to false to disable reporting. -;reporting_enabled = true - -# Set to false to disable all checks to https://grafana.net -# for new vesions (grafana itself and plugins), check is used -# in some UI views to notify that grafana or plugin update exists -# This option does not cause any auto updates, nor send any information -# only a GET request to http://grafana.com to get latest versions -;check_for_updates = true - -# Google Analytics universal tracking code, only enabled if you specify an id here -;google_analytics_ua_id = - -#################################### Security #################################### -[security] -# default admin user, created on startup -;admin_user = admin - -# default admin password, can be changed before first start of grafana, or in profile settings -;admin_password = admin - -# used for signing -;secret_key = SW2YcwTIb9zpOOhoPsMm - -# Auto-login remember days -;login_remember_days = 7 -;cookie_username = grafana_user -;cookie_remember_name = grafana_remember - -# disable gravatar profile images -;disable_gravatar = false - -# data source proxy whitelist (ip_or_domain:port separated by spaces) -;data_source_proxy_whitelist = - -[snapshots] -# snapshot sharing options -;external_enabled = true -;external_snapshot_url = https://snapshots-origin.raintank.io -;external_snapshot_name = Publish to snapshot.raintank.io - -# remove expired snapshot -;snapshot_remove_expired = true - -# remove snapshots after 90 days -;snapshot_TTL_days = 90 - -#################################### Users #################################### -[users] -# disable user signup / registration -;allow_sign_up = true - -# Allow non admin users to create organizations -;allow_org_create = true - -# Set to true to automatically assign new users to the default organization (id 1) -;auto_assign_org = true - -# Default role new users will be automatically assigned (if disabled above is set to true) -;auto_assign_org_role = Viewer - -# Background text for the user field on the login page -;login_hint = email or username - -# Default UI theme ("dark" or "light") -;default_theme = dark - -[auth] -# Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false -;disable_login_form = false - -# Set to true to disable the signout link in the side menu. useful if you use auth.proxy, defaults to false -;disable_signout_menu = false - -#################################### Anonymous Auth ########################## -[auth.anonymous] -# enable anonymous access -;enabled = false - -# specify organization name that should be used for unauthenticated users -;org_name = Main Org. - -# specify role for unauthenticated users -;org_role = Viewer - -#################################### Github Auth ########################## -[auth.github] -;enabled = false -;allow_sign_up = true -;client_id = some_id -;client_secret = some_secret -;scopes = user:email,read:org -;auth_url = https://github.com/login/oauth/authorize -;token_url = https://github.com/login/oauth/access_token -;api_url = https://api.github.com/user -;team_ids = -;allowed_organizations = - -#################################### Google Auth ########################## -[auth.google] -;enabled = false -;allow_sign_up = true -;client_id = some_client_id -;client_secret = some_client_secret -;scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email -;auth_url = https://accounts.google.com/o/oauth2/auth -;token_url = https://accounts.google.com/o/oauth2/token -;api_url = https://www.googleapis.com/oauth2/v1/userinfo -;allowed_domains = - -#################################### Generic OAuth ########################## -[auth.generic_oauth] -;enabled = false -;name = OAuth -;allow_sign_up = true -;client_id = some_id -;client_secret = some_secret -;scopes = user:email,read:org -;auth_url = https://foo.bar/login/oauth/authorize -;token_url = https://foo.bar/login/oauth/access_token -;api_url = https://foo.bar/user -;team_ids = -;allowed_organizations = - -#################################### Grafana.com Auth #################### -[auth.grafana_com] -;enabled = false -;allow_sign_up = true -;client_id = some_id -;client_secret = some_secret -;scopes = user:email -;allowed_organizations = - -#################################### Auth Proxy ########################## -[auth.proxy] -;enabled = false -;header_name = X-WEBAUTH-USER -;header_property = username -;auto_sign_up = true -;ldap_sync_ttl = 60 -;whitelist = 192.168.1.1, 192.168.2.1 - -#################################### Basic Auth ########################## -[auth.basic] -;enabled = true - -#################################### Auth LDAP ########################## -[auth.ldap] -;enabled = false -;config_file = /etc/grafana/ldap.toml -;allow_sign_up = true - -#################################### SMTP / Emailing ########################## -[smtp] -;enabled = false -;host = localhost:25 -;user = -# If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;""" -;password = -;cert_file = -;key_file = -;skip_verify = false -;from_address = admin@grafana.localhost -;from_name = Grafana - -[emails] -;welcome_email_on_sign_up = false - -#################################### Logging ########################## -[log] -# Either "console", "file", "syslog". Default is console and file -# Use space to separate multiple modes, e.g. "console file" -;mode = console file - -# Either "debug", "info", "warn", "error", "critical", default is "info" -;level = info - -# optional settings to set different levels for specific loggers. Ex filters = sqlstore:debug -;filters = - - -# For "console" mode only -[log.console] -;level = - -# log line format, valid options are text, console and json -;format = console - -# For "file" mode only -[log.file] -;level = - -# log line format, valid options are text, console and json -;format = text - -# This enables automated log rotate(switch of following options), default is true -;log_rotate = true - -# Max line number of single file, default is 1000000 -;max_lines = 1000000 - -# Max size shift of single file, default is 28 means 1 << 28, 256MB -;max_size_shift = 28 - -# Segment log daily, default is true -;daily_rotate = true - -# Expired days of log file(delete after max days), default is 7 -;max_days = 7 - -[log.syslog] -;level = - -# log line format, valid options are text, console and json -;format = text - -# Syslog network type and address. This can be udp, tcp, or unix. If left blank, the default unix endpoints will be used. -;network = -;address = - -# Syslog facility. user, daemon and local0 through local7 are valid. -;facility = - -# Syslog tag. By default, the process' argv[0] is used. -;tag = - - -#################################### AMQP Event Publisher ########################## -[event_publisher] -;enabled = false -;rabbitmq_url = amqp://localhost/ -;exchange = grafana_events - -;#################################### Dashboard JSON files ########################## -[dashboards.json] -;enabled = false -;path = /var/lib/grafana/dashboards - -#################################### Alerting ############################ -[alerting] -# Disable alerting engine & UI features -;enabled = true -# Makes it possible to turn off alert rule execution but alerting UI is visible -;execute_alerts = true - -#################################### Internal Grafana Metrics ########################## -# Metrics available at HTTP API Url /api/metrics -[metrics] -# Disable / Enable internal metrics -;enabled = true - -# Publish interval -;interval_seconds = 10 - -# Send internal metrics to Graphite -[metrics.graphite] -# Enable by setting the address setting (ex localhost:2003) -;address = -;prefix = prod.grafana.%(instance_name)s. - -#################################### Grafana.com integration ########################## -# Url used to to import dashboards directly from Grafana.com -[grafana_com] -;url = https://grafana.com - -#################################### External image storage ########################## -[external_image_storage] -# Used for uploading images to public servers so they can be included in slack/email messages. -# you can choose between (s3, webdav) -;provider = - -[external_image_storage.s3] -;bucket_url = -;access_key = -;secret_key = - -[external_image_storage.webdav] -;url = -;public_url = -;username = -;password = diff --git a/conf/etc/grafana/provisioning/datasources/carbonapi.yaml b/conf/etc/grafana/provisioning/datasources/carbonapi.yaml deleted file mode 100644 index 25b6c91..0000000 --- a/conf/etc/grafana/provisioning/datasources/carbonapi.yaml +++ /dev/null @@ -1,43 +0,0 @@ -# config file version -apiVersion: 1 - -## list of datasources that should be deleted from the database -#deleteDatasources: -# - name: Graphite -# orgId: 1 - -# list of datasources to insert/update depending -# whats available in the database -datasources: - # name of the datasource. Required -- name: carbonapi - # datasource type. Required - type: graphite - # access mode. direct or proxy. Required - access: proxy - # org id. will default to orgId 1 if not specified - orgId: 1 - # url - url: http://127.0.0.1:8081 - # database password, if used - password: - # database user, if used - user: - # database name, if used - database: - # enable/disable basic auth - basicAuth: - # basic auth username - basicAuthUser: - # basic auth password - basicAuthPassword: - # enable/disable with credentials headers - withCredentials: - # mark as default datasource. Max one per org - isDefault: true - # fields that will be converted to json and stored in json_data - jsonData: - graphiteVersion: "1.1" - graphiteType": "default" - # allow users to edit datasources from the UI. - editable: true \ No newline at end of file diff --git a/conf/etc/nginx/nginx.conf b/conf/etc/nginx/nginx.conf deleted file mode 100644 index 80e0c16..0000000 --- a/conf/etc/nginx/nginx.conf +++ /dev/null @@ -1,96 +0,0 @@ -user nginx; -worker_processes 4; -pid /run/nginx.pid; -daemon off; - -events { - worker_connections 768; - # multi_accept on; -} - -http { - - ## - # Basic Settings - ## - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - # server_tokens off; - - # server_names_hash_bucket_size 64; - # server_name_in_redirect off; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - ## - # Logging Settings - ## - - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - - ## - # Gzip Settings - ## - - gzip on; - gzip_disable "msie6"; - - # gzip_vary on; - # gzip_proxied any; - # gzip_comp_level 6; - # gzip_buffers 16 8k; - # gzip_http_version 1.1; - # gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript; - - ## - # nginx-naxsi config - ## - # Uncomment it if you installed nginx-naxsi - ## - - #include /etc/nginx/naxsi_core.rules; - - ## - # nginx-passenger config - ## - # Uncomment it if you installed nginx-passenger - ## - - #passenger_root /usr; - #passenger_ruby /usr/bin/ruby; - - ## - # Virtual Host Configs - ## - - include /etc/nginx/conf.d/*.conf; - include /etc/nginx/sites-enabled/*; -} - - -#mail { -# # See sample authentication script at: -# # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript -# -# # auth_http localhost/auth.php; -# # pop3_capabilities "TOP" "USER"; -# # imap_capabilities "IMAP4rev1" "UIDPLUS"; -# -# server { -# listen localhost:110; -# protocol pop3; -# proxy on; -# } -# -# server { -# listen localhost:143; -# protocol imap; -# proxy on; -# } -#} diff --git a/conf/etc/nginx/sites-enabled/go-graphite.conf b/conf/etc/nginx/sites-enabled/go-graphite.conf deleted file mode 100644 index 60b2535..0000000 --- a/conf/etc/nginx/sites-enabled/go-graphite.conf +++ /dev/null @@ -1,9 +0,0 @@ -server { - listen 80; - root /usr/share/nginx/www; - index index.html index.htm; - - location / { - proxy_pass http://localhost:3000/; - } -} diff --git a/conf/etc/service/brubeck/run b/conf/etc/service/brubeck/run new file mode 100755 index 0000000..68c12d1 --- /dev/null +++ b/conf/etc/service/brubeck/run @@ -0,0 +1,5 @@ +#!/bin/bash + +/bin/generate_config.sh + +exec /bin/brubeck --config=/root/config.json diff --git a/conf/etc/service/grafana/run b/conf/etc/service/grafana/run deleted file mode 100755 index 30213f0..0000000 --- a/conf/etc/service/grafana/run +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -if [ ! -z "${GF_INSTALL_PLUGINS}" ]; then - OLDIFS=$IFS - IFS=',' - for plugin in ${GF_INSTALL_PLUGINS}; do - IFS=$OLDIFS - if [[ $plugin =~ .*\;.* ]]; then - pluginUrl=$(echo "$plugin" | cut -d';' -f 1) - pluginWithoutUrl=$(echo "$plugin" | cut -d';' -f 2) - /usr/bin/grafana-cli --pluginUrl "${pluginUrl}" --pluginsDir "${GF_PATHS_PLUGINS}" plugins install ${pluginWithoutUrl} - else - /usr/bin/grafana-cli --pluginsDir "${GF_PATHS_PLUGINS}" plugins install ${plugin} - fi - done -fi - -exec /usr/sbin/grafana-server --homepath=/usr/share/grafana/ \ - --config=/etc/grafana/grafana.ini \ - cfg:default.paths.provisioning="/etc/grafana/provisioning" \ - cfg:default.log.mode="console" >> /var/log/grafana.log 2>&1 - diff --git a/conf/etc/service/nginx/run b/conf/etc/service/nginx/run deleted file mode 100755 index 7691c03..0000000 --- a/conf/etc/service/nginx/run +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -mkdir -p /var/log/nginx -exec /usr/sbin/nginx -c /etc/nginx/nginx.conf diff --git a/conf/generate_config.sh b/conf/generate_config.sh new file mode 100755 index 0000000..05cabef --- /dev/null +++ b/conf/generate_config.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +STATSD_PORT=${STATSD_PORT-8125} +STATSD_WORKERS=${STATSD_WORKERS-4} +CARBON_HOST=${CARBON_HOST-127.0.0.1} +CARBON_PORT=${CARBON_PORT-2003} + +cat /config.template.json \ + | jq ".samplers[0].port = ${STATSD_PORT}" \ + | jq ".samplers[0].workers = ${STATSD_WORKERS}" \ + | jq ".backends[0].address = \"${CARBON_HOST}\"" \ + | jq ".backends[0].port = ${CARBON_PORT}" \ + > /root/config.json