add optional export metrics to prometheus

Gemfile

@@ -77,6 +77,8 @@ gem 'puma_worker_killer'
 gem 'syslog-logger'
 gem 'zip-zip'
 
+gem 'prometheus_exporter', require: false
+
 group :development do
   gem 'annotate'
   gem 'sourcify'

Gemfile.lock

@@ -354,6 +354,7 @@ GEM
       ast (~> 2.4.0)
     pg (1.0.0)
     powerpack (0.1.2)
+    prometheus_exporter (0.4.14)
     public_suffix (3.0.3)
     puma (3.12.1)
     puma_worker_killer (0.1.0)
@@ -566,6 +567,7 @@ DEPENDENCIES
   parallel_tests
   pg
   postgres_ext!
+  prometheus_exporter
   puma
   puma_worker_killer
   pundit

bin/prometheus_exporter

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'prometheus_exporter' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path('bundle', __dir__)
+
+if File.file?(bundle_binstub)
+  if /This file was generated by Bundler/.match?(File.read(bundle_binstub, 300))
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('prometheus_exporter', 'prometheus_exporter')

config/initializers/prometheus.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+if !Rails.env.test? && Rails.configuration.yeti_web['prometheus']['enabled'] && !defined?(::Rake)
+
+  # NOTE: as we have different processes but want 1 interface for prometheus to scrape the metrics
+  # we send the metrics to a collector container
+  # this process need to be started as a separate process
+  #
+  # Prometheus Ports:
+  # https://github.com/prometheus/prometheus/wiki/Default-port-allocations
+
+  require 'prometheus_exporter'
+
+  require 'prometheus_exporter/metric'
+  PrometheusExporter::Metric::Base.default_labels = Rails.configuration.yeti_web['prometheus']['default_labels']
+
+  # Connect to Prometheus Collector Process
+  require 'prometheus_exporter/client'
+  my_client = PrometheusExporter::Client.new(
+    host: Rails.configuration.yeti_web['prometheus']['host'],
+    port: Rails.configuration.yeti_web['prometheus']['port'],
+    custom_labels: Rails.configuration.yeti_web['prometheus']['default_labels']
+  )
+
+  # Set Default Client
+  PrometheusExporter::Client.default = my_client
+  # This reports stats per request like HTTP status and timings
+  require 'prometheus_exporter/middleware'
+  Rails.application.middleware.unshift PrometheusExporter::Middleware
+
+  # this reports basic process stats like RSS and GC info
+  require 'prometheus_exporter/instrumentation'
+  PrometheusExporter::Instrumentation::DelayedJob.register_plugin
+
+  PrometheusExporter::Instrumentation::Process.start(
+    type: 'master',
+    labels: Rails.configuration.yeti_web['prometheus']['default_labels']
+  )
+end

config/puma.rb

@@ -32,11 +32,25 @@
     config.pre_term = ->(worker) { puts "Worker #{worker.inspect} being killed" }
   end
   PumaWorkerKiller.start
+
+  if Rails.configuration.yeti_web['prometheus']['enabled']
+    require 'prometheus_exporter/client'
+    require 'prometheus_exporter/instrumentation'
+    PrometheusExporter::Instrumentation::Puma.start
+  end
 end
 
 on_worker_boot do
   ActiveSupport.on_load(:active_record) do
     ActiveRecord::Base.establish_connection
     SecondBase::Base.establish_connection
   end
+
+  if Rails.configuration.yeti_web['prometheus']['enabled']
+    require 'prometheus_exporter/instrumentation'
+    PrometheusExporter::Instrumentation::Process.start(
+      type: 'web',
+      labels: Rails.configuration.yeti_web['prometheus']['default_labels']
+    )
+  end
 end

config/puma_production.rb

@@ -31,11 +31,25 @@
     config.pre_term = ->(worker) { puts "Worker #{worker.inspect} being killed" }
   end
   PumaWorkerKiller.start
+
+  if Rails.configuration.yeti_web['prometheus']['enabled']
+    require 'prometheus_exporter/client'
+    require 'prometheus_exporter/instrumentation'
+    PrometheusExporter::Instrumentation::Puma.start
+  end
 end
 
 on_worker_boot do
   ActiveSupport.on_load(:active_record) do
     ActiveRecord::Base.establish_connection
     SecondBase::Base.establish_connection
   end
+
+  if Rails.configuration.yeti_web['prometheus']['enabled']
+    require 'prometheus_exporter/instrumentation'
+    PrometheusExporter::Instrumentation::Process.start(
+      type: 'web',
+      labels: Rails.configuration.yeti_web['prometheus']['default_labels']
+    )
+  end
 end

config/yeti_web.yml.distr

@@ -15,3 +15,10 @@ partition_remove_delay:
   auth_log.auth_log: 7
   rtp_statistics.streams: 3
   logs.api_requests: 90
+
+prometheus:
+  enabled: false
+  host: localhost
+  port: 9394
+  default_labels:
+    host: my-host

debian/yeti-web.yeti-prometheus.service

@@ -0,0 +1,23 @@
+[Unit]
+Description=YETI system web interface daemon for Prometheus
+Documentation=https://yeti-switch.org/docs/
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+User=yeti-web
+Group=yeti-web
+LimitNOFILE=65536
+LimitCORE=infinity
+
+Environment=RAILS_ENV=production
+Environment=RACK_ENV=production
+Environment=RAKE_ENV=production
+Environment=BUNDLE_GEMFILE=/opt/yeti-web/Gemfile
+Environment=GEM_PATH=/opt/yeti-web/vendor/bundler
+
+RuntimeDirectory=yeti-delayed-job
+ExecStart=/usr/bin/ruby /opt/yeti-web/vendor/bundler/bin/bundle exec /opt/yeti-web/bin/prometheus_exporter
+Type=simple
+Restart=on-abnormal

spec/config/yeti_web_spec.rb

@@ -27,6 +27,12 @@
         'auth_log.auth_log': anything,
         'rtp_statistics.streams': anything,
         'logs.api_requests': anything
+      },
+      prometheus: {
+        enabled: anything,
+        host: a_kind_of(String),
+        port: a_kind_of(Integer),
+        default_labels: a_kind_of(Hash)
       }
     }
   end