From dbb186a1f6f88b1cba967d1b392fc4976c49607d Mon Sep 17 00:00:00 2001 From: Denis Talakevich Date: Wed, 30 Oct 2019 22:06:17 +0200 Subject: [PATCH] add optional export metrics to prometheus --- Gemfile | 2 ++ Gemfile.lock | 2 ++ bin/prometheus_exporter | 28 ++++++++++++++++++ config/initializers/prometheus.rb | 39 +++++++++++++++++++++++++ config/puma.rb | 14 +++++++++ config/puma_production.rb | 14 +++++++++ config/yeti_web.yml.distr | 7 +++++ debian/yeti-web.yeti-prometheus.service | 23 +++++++++++++++ spec/config/yeti_web_spec.rb | 6 ++++ 9 files changed, 135 insertions(+) create mode 100755 bin/prometheus_exporter create mode 100644 config/initializers/prometheus.rb create mode 100644 debian/yeti-web.yeti-prometheus.service diff --git a/Gemfile b/Gemfile index d54c86166..3a54007bc 100644 --- a/Gemfile +++ b/Gemfile @@ -77,6 +77,8 @@ gem 'puma_worker_killer' gem 'syslog-logger' gem 'zip-zip' +gem 'prometheus_exporter', require: false + group :development do gem 'annotate' gem 'sourcify' diff --git a/Gemfile.lock b/Gemfile.lock index acf577702..88c4293e9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -354,6 +354,7 @@ GEM ast (~> 2.4.0) pg (1.0.0) powerpack (0.1.2) + prometheus_exporter (0.4.14) public_suffix (3.0.3) puma (3.12.1) puma_worker_killer (0.1.0) @@ -566,6 +567,7 @@ DEPENDENCIES parallel_tests pg postgres_ext! + prometheus_exporter puma puma_worker_killer pundit diff --git a/bin/prometheus_exporter b/bin/prometheus_exporter new file mode 100755 index 000000000..94400457a --- /dev/null +++ b/bin/prometheus_exporter @@ -0,0 +1,28 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# +# This file was generated by Bundler. +# +# The application 'prometheus_exporter' is installed as part of a gem, and +# this file is here to facilitate running it. +# + +require 'pathname' +ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', Pathname.new(__FILE__).realpath) + +bundle_binstub = File.expand_path('bundle', __dir__) + +if File.file?(bundle_binstub) + if /This file was generated by Bundler/.match?(File.read(bundle_binstub, 300)) + load(bundle_binstub) + else + abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run. +Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.") + end +end + +require 'rubygems' +require 'bundler/setup' + +load Gem.bin_path('prometheus_exporter', 'prometheus_exporter') diff --git a/config/initializers/prometheus.rb b/config/initializers/prometheus.rb new file mode 100644 index 000000000..153acae61 --- /dev/null +++ b/config/initializers/prometheus.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +if !Rails.env.test? && Rails.configuration.yeti_web['prometheus']['enabled'] && !defined?(::Rake) + + # NOTE: as we have different processes but want 1 interface for prometheus to scrape the metrics + # we send the metrics to a collector container + # this process need to be started as a separate process + # + # Prometheus Ports: + # https://github.com/prometheus/prometheus/wiki/Default-port-allocations + + require 'prometheus_exporter' + + require 'prometheus_exporter/metric' + PrometheusExporter::Metric::Base.default_labels = Rails.configuration.yeti_web['prometheus']['default_labels'] + + # Connect to Prometheus Collector Process + require 'prometheus_exporter/client' + my_client = PrometheusExporter::Client.new( + host: Rails.configuration.yeti_web['prometheus']['host'], + port: Rails.configuration.yeti_web['prometheus']['port'], + custom_labels: Rails.configuration.yeti_web['prometheus']['default_labels'] + ) + + # Set Default Client + PrometheusExporter::Client.default = my_client + # This reports stats per request like HTTP status and timings + require 'prometheus_exporter/middleware' + Rails.application.middleware.unshift PrometheusExporter::Middleware + + # this reports basic process stats like RSS and GC info + require 'prometheus_exporter/instrumentation' + PrometheusExporter::Instrumentation::DelayedJob.register_plugin + + PrometheusExporter::Instrumentation::Process.start( + type: 'master', + labels: Rails.configuration.yeti_web['prometheus']['default_labels'] + ) +end diff --git a/config/puma.rb b/config/puma.rb index 108d11bb7..7834aea6c 100644 --- a/config/puma.rb +++ b/config/puma.rb @@ -32,6 +32,12 @@ config.pre_term = ->(worker) { puts "Worker #{worker.inspect} being killed" } end PumaWorkerKiller.start + + if Rails.configuration.yeti_web['prometheus']['enabled'] + require 'prometheus_exporter/client' + require 'prometheus_exporter/instrumentation' + PrometheusExporter::Instrumentation::Puma.start + end end on_worker_boot do @@ -39,4 +45,12 @@ ActiveRecord::Base.establish_connection SecondBase::Base.establish_connection end + + if Rails.configuration.yeti_web['prometheus']['enabled'] + require 'prometheus_exporter/instrumentation' + PrometheusExporter::Instrumentation::Process.start( + type: 'web', + labels: Rails.configuration.yeti_web['prometheus']['default_labels'] + ) + end end diff --git a/config/puma_production.rb b/config/puma_production.rb index 9e02e8e9e..2dbc051b5 100644 --- a/config/puma_production.rb +++ b/config/puma_production.rb @@ -31,6 +31,12 @@ config.pre_term = ->(worker) { puts "Worker #{worker.inspect} being killed" } end PumaWorkerKiller.start + + if Rails.configuration.yeti_web['prometheus']['enabled'] + require 'prometheus_exporter/client' + require 'prometheus_exporter/instrumentation' + PrometheusExporter::Instrumentation::Puma.start + end end on_worker_boot do @@ -38,4 +44,12 @@ ActiveRecord::Base.establish_connection SecondBase::Base.establish_connection end + + if Rails.configuration.yeti_web['prometheus']['enabled'] + require 'prometheus_exporter/instrumentation' + PrometheusExporter::Instrumentation::Process.start( + type: 'web', + labels: Rails.configuration.yeti_web['prometheus']['default_labels'] + ) + end end diff --git a/config/yeti_web.yml.distr b/config/yeti_web.yml.distr index b6c6ce6e7..e7a70847e 100644 --- a/config/yeti_web.yml.distr +++ b/config/yeti_web.yml.distr @@ -15,3 +15,10 @@ partition_remove_delay: auth_log.auth_log: 7 rtp_statistics.streams: 3 logs.api_requests: 90 + +prometheus: + enabled: false + host: localhost + port: 9394 + default_labels: + host: my-host diff --git a/debian/yeti-web.yeti-prometheus.service b/debian/yeti-web.yeti-prometheus.service new file mode 100644 index 000000000..d319dcf42 --- /dev/null +++ b/debian/yeti-web.yeti-prometheus.service @@ -0,0 +1,23 @@ +[Unit] +Description=YETI system web interface daemon for Prometheus +Documentation=https://yeti-switch.org/docs/ + +[Install] +WantedBy=multi-user.target + +[Service] +User=yeti-web +Group=yeti-web +LimitNOFILE=65536 +LimitCORE=infinity + +Environment=RAILS_ENV=production +Environment=RACK_ENV=production +Environment=RAKE_ENV=production +Environment=BUNDLE_GEMFILE=/opt/yeti-web/Gemfile +Environment=GEM_PATH=/opt/yeti-web/vendor/bundler + +RuntimeDirectory=yeti-delayed-job +ExecStart=/usr/bin/ruby /opt/yeti-web/vendor/bundler/bin/bundle exec /opt/yeti-web/bin/prometheus_exporter +Type=simple +Restart=on-abnormal diff --git a/spec/config/yeti_web_spec.rb b/spec/config/yeti_web_spec.rb index 1f333660f..14c43fb51 100644 --- a/spec/config/yeti_web_spec.rb +++ b/spec/config/yeti_web_spec.rb @@ -27,6 +27,12 @@ 'auth_log.auth_log': anything, 'rtp_statistics.streams': anything, 'logs.api_requests': anything + }, + prometheus: { + enabled: anything, + host: a_kind_of(String), + port: a_kind_of(Integer), + default_labels: a_kind_of(Hash) } } end