diff --git a/Envfile b/Envfile index 944a3fac..854188e0 100644 --- a/Envfile +++ b/Envfile @@ -148,6 +148,11 @@ variable :TRENDING_TAGS, :String, default: "git,beginners" variable :FACEBOOK_PIXEL_ID, :String, default: "Optional" variable :SMARTY_STREETS_WEB_KEY, :String, default: "Optional" +variable :SKYLIGHT_AUTHENTICATION, :String, default: "Optional" +variable :NEWRELIC_AUTHENTICATION, :String, default: "Optional" + +variable :MINI_PROFILER, :String, default: "Optional" + group :production do variable :SECRET_KEY_BASE, :String diff --git a/Gemfile b/Gemfile index 1a5f4c0b..1f2793f6 100644 --- a/Gemfile +++ b/Gemfile @@ -102,6 +102,9 @@ gem "uglifier", "~> 4.1" gem "validate_url", "~> 1.0" gem "webpacker", "~> 3.5" gem "webpush", "~> 0.3" +gem 'newrelic_rpm' +gem 'rack-mini-profiler' +gem "prometheus_exporter" group :development do gem "better_errors", "~> 2.5" @@ -115,6 +118,7 @@ group :development do gem "guard-rspec", "~> 4.7", require: false gem "rb-fsevent", "~> 0.10", require: false gem "web-console", "~> 3.7" + gem 'meta_request' end group :development, :test do diff --git a/Gemfile.lock b/Gemfile.lock index c7c050cf..17438503 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -592,6 +592,9 @@ GEM memoizable (0.4.2) thread_safe (~> 0.3, >= 0.3.1) memory_profiler (0.9.12) + meta_request (0.7.4) + rack-contrib (>= 1.1, < 3) + railties (>= 3.0.0, < 7.1) method_source (0.9.2) mime-types (3.2.2) mime-types-data (~> 3.2015) @@ -612,6 +615,7 @@ GEM net-http-persistent (3.0.0) connection_pool (~> 2.2) netrc (0.11.0) + newrelic_rpm (9.1.0) nio4r (2.3.1) nokogiri (1.10.1) mini_portile2 (~> 2.4.0) @@ -651,6 +655,8 @@ GEM ast (~> 2.4.0) pg (1.1.4) powerpack (0.1.2) + prometheus_exporter (2.0.8) + webrick pry (0.12.2) coderay (~> 1.1.0) method_source (~> 0.9.0) @@ -675,8 +681,12 @@ GEM pusher-signature (0.1.8) raabro (1.1.6) rack (2.0.6) + rack-contrib (2.3.0) + rack (~> 2.0) rack-host-redirect (1.3.0) rack + rack-mini-profiler (3.0.0) + rack (>= 1.2.0) rack-protection (2.0.4) rack rack-proxy (0.6.5) @@ -932,6 +942,7 @@ GEM webpush (0.3.2) hkdf (~> 0.2) jwt + webrick (1.8.1) websocket-driver (0.6.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.3) @@ -1012,7 +1023,9 @@ DEPENDENCIES libhoney (~> 1.11) liquid (~> 4.0) memory_profiler (~> 0.9) + meta_request nakayoshi_fork + newrelic_rpm nokogiri (~> 1.10) octokit (~> 4.13) omniauth (~> 1.9) @@ -1020,6 +1033,7 @@ DEPENDENCIES omniauth-twitter (~> 1.4) parallel_tests (~> 2.27) pg (~> 1.1) + prometheus_exporter pry (~> 0.12) pry-byebug (~> 3.7) pry-rails (~> 0.3) @@ -1029,6 +1043,7 @@ DEPENDENCIES pusher (~> 1.3) pusher-push-notifications (~> 1.0) rack-host-redirect (~> 1.3) + rack-mini-profiler rack-timeout (~> 0.5) rails (~> 5.1.6) rails-assets-airbrake-js-client (~> 1.5)! diff --git a/Procfile.profile b/Procfile.profile new file mode 100644 index 00000000..64554f21 --- /dev/null +++ b/Procfile.profile @@ -0,0 +1,3 @@ +web: bin/rails s -p 3000 -e profile +job: bin/rake jobs:work +prometheus_exporter: bundle exec prometheus_exporter -a prometheus/custom_collector.rb diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 1531dc97..20130f2c 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -1,6 +1,10 @@ class ApplicationController < ActionController::Base protect_from_forgery with: :exception, prepend: true + before_action do + Rack::MiniProfiler.authorize_request if cookies[:mini_profiler] == ApplicationConfig["MINI_PROFILER"] + end + include Pundit include Instrumentation diff --git a/app/views/stories/_main_stories_feed.html.erb b/app/views/stories/_main_stories_feed.html.erb index b6cd0a65..9fb395e7 100644 --- a/app/views/stories/_main_stories_feed.html.erb +++ b/app/views/stories/_main_stories_feed.html.erb @@ -55,7 +55,9 @@ <% if !user_signed_in? && i == 4 %> <%= render "stories/sign_in_invitation" %> <% end %> - <%= render "articles/single_story", story: story %> + <% cache(story) do %> + <%= render "articles/single_story", story: story %> + <% end %> <% end %> <% end %> <% if @stories.size > 1 %> diff --git a/bin/startup b/bin/startup index c7e84efb..de393fa5 100755 --- a/bin/startup +++ b/bin/startup @@ -11,6 +11,12 @@ def system!(*args) end chdir APP_ROOT do - puts "== STARTING UP ==" - system! "foreman start -f Procfile.dev" + env = ENV.fetch('RAILS_ENV', 'development') + puts "== STARTING UP #{env}==" + + if env == 'profile' + system! "foreman start -f Procfile.profile" + else + system! "foreman start -f Procfile.dev" + end end diff --git a/case-study.md b/case-study.md new file mode 100644 index 00000000..6b9b3c1d --- /dev/null +++ b/case-study.md @@ -0,0 +1,128 @@ +## Цели + +- Попрактиковаться в настройке мониторинга +- Попрактиковаться в поиске возможностей для оптимизации + +- Попрактиковаться в проверке гипотез и обосновании предложений по оптимизации +- Познакомиться с интересным живым Rails open-source проектом + +## Настройка систем профилирования + +- Подключил NewRelic +- Подключил Skylight +- Подключил Prometheus + Grafana + +- Настроил rack-mini-profiler +- Настроил rails-panel + +## Настройка local_production окружения + +Добавил новый environment profile - для профилирования в среде максимально приближенной к Production + +## Нагрузочное тестирование + +Для проверки гипотиз и выстраивания максимально эффективного фидбэк лупа benchmark осуществлял при помощи утилиты ApacheBench. + +## Оптимизация + +Все инструменты мониторинга показывают, что самой горячей точкой является главная страница, StoriesController#index. + +В частности, заметное время занимает рендеринг partial-ов \_single_story.html.erb. + +Статистика 200 запросов в ab: + +```Benchmarking 127.0.0.1 (be patient) +Completed 100 requests +Completed 200 requests +Finished 200 requests + + +Server Software: +Server Hostname: 127.0.0.1 +Server Port: 3000 + +Document Path: / +Document Length: 142263 bytes + +Concurrency Level: 5 +Time taken for tests: 10.545 seconds +Complete requests: 200 +Failed requests: 0 +Total transferred: 28537200 bytes +HTML transferred: 28452600 bytes +Requests per second: 18.97 [#/sec] (mean) +Time per request: 263.633 [ms] (mean) +Time per request: 52.727 [ms] (mean, across all concurrent requests) +Transfer rate: 2642.73 [Kbytes/sec] received + +Connection Times (ms) +min mean[+/-sd] median max +Connect: 0 0 0.4 0 3 +Processing: 84 258 92.6 260 536 +Waiting: 84 247 89.2 246 498 +Total: 85 258 92.6 260 536 + +Percentage of the requests served within a certain time (ms) +50% 260 +66% 301 +75% 328 +80% 339 +90% 378 +95% 412 +98% 455 +99% 489 +100% 536 (longest request) +``` + +Скрин NewRelic до оптимизации: https://disk.yandex.ru/i/ocwtqArpKiEQbQ + +Рассмотрел гипотезу о том, что можно закешировать <%= render "articles/single_story", story: story %> в \_main_stories_feed.html.erb и это даст заметный эффект. + +Статистика 200 запросов в ab после отимизации: + +```Benchmarking 127.0.0.1 (be patient) +Completed 100 requests +Completed 200 requests +Finished 200 requests + + +Server Software: +Server Hostname: 127.0.0.1 +Server Port: 3000 + +Document Path: / +Document Length: 142311 bytes + +Concurrency Level: 5 +Time taken for tests: 6.603 seconds +Complete requests: 200 +Failed requests: 0 +Total transferred: 28546800 bytes +HTML transferred: 28462200 bytes +Requests per second: 30.29 [#/sec] (mean) +Time per request: 165.066 [ms] (mean) +Time per request: 33.013 [ms] (mean, across all concurrent requests) +Transfer rate: 4222.22 [Kbytes/sec] received + +Connection Times (ms) +min mean[+/-sd] median max +Connect: 0 0 0.2 0 2 +Processing: 57 162 59.0 154 384 +Waiting: 55 153 56.2 143 374 +Total: 57 162 59.0 154 384 + +Percentage of the requests served within a certain time (ms) +50% 154 +66% 173 +75% 184 +80% 203 +90% 254 +95% 289 +98% 319 +99% 347 +100% 384 (longest request) +``` + +Изменение графика в NewRelic помле оптимизации https://disk.yandex.ru/i/d40doyOEVx7P2Q + +Время ответа для 200 запросов уменьшилось с 10.545 до 6.603 секунд diff --git a/config/environments/development.rb b/config/environments/development.rb index ad3b97cb..ff56e943 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -6,7 +6,7 @@ def yarn_integrity_enabled? Rails.application.configure do # Verifies that versions and hashed value of the package contents in the project's package.json - config.webpacker.check_yarn_integrity = yarn_integrity_enabled? + config.webpacker.check_yarn_integrity = false # Settings specified here will take precedence over those in config/application.rb. diff --git a/config/environments/profile.rb b/config/environments/profile.rb new file mode 100644 index 00000000..ce8e11c5 --- /dev/null +++ b/config/environments/profile.rb @@ -0,0 +1,123 @@ +Rails.application.configure do + # config.middleware.use(Rack::RubyProf, :path => 'ruby-prof-results') + # config.middleware.use ProfileMiddleware + + # Verifies that versions and hashed value of the package contents in the project's package.json + config.webpacker.check_yarn_integrity = false + + # Settings specified here will take precedence over those in config/application.rb. + + # Code is not reloaded between requests. + config.cache_classes = true + + # Eager load code on boot. This eager loads most of Rails and + # your application in memory, allowing both threaded web servers + # and those relying on copy on write to perform better. + # Rake tasks automatically ignore this option for performance. + config.eager_load = true + + # Full error reports are disabled and caching is turned on. + config.consider_all_requests_local = false + config.action_controller.perform_caching = true + + # Enable Rack::Cache to put a simple HTTP cache in front of your application + # Add `rack-cache` to your Gemfile before enabling this. + # For large-scale production use, consider using a caching reverse proxy like + # NGINX, varnish or squid. + # config.action_dispatch.rack_cache = true + config.read_encrypted_secrets = true + + # Disable serving static files from the `/public` folder by default since + # Apache or NGINX already handles this. + config.public_file_server.enabled = ENV["RAILS_SERVE_STATIC_FILES"].present? + config.public_file_server.headers = { + "Cache-Control" => "public, s-maxage=2592000, max-age=86400" + } + + # Compress JavaScripts and CSS. + config.assets.js_compressor = Uglifier.new(harmony: true) + # config.assets.css_compressor = :sass + + # Do not fallback to assets pipeline if a precompiled asset is missed. + config.assets.compile = true + + # Asset digests allow you to set far-future HTTP expiration dates on all assets, + # yet still be able to expire them through the digest params. + config.assets.digest = true + + # `config.assets.precompile` and `config.assets.version` + # have moved to config/initializers/assets.rb + + # Specifies the header that your server uses for sending files. + # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache + # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX + + # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. + # config.force_ssl = true + + # Use the lowest log level to ensure availability of diagnostic information + # when problems arise. + config.log_level = :debug + + # Prepend all log lines with the following tags. + config.log_tags = [:request_id] + + # Use a different logger for distributed setups. + # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new) + + # Use a different cache store in production. + # config.cache_store = :mem_cache_store + config.cache_store = :memory_store + config.public_file_server.headers = { + "Cache-Control" => "public, max-age=172800" + } + + config.action_mailer.perform_caching = false + + # Ignore bad email addresses and do not raise email delivery errors. + # Set this to true and configure the email server for immediate delivery to raise delivery errors. + config.action_mailer.raise_delivery_errors = false + + # Enable locale fallbacks for I18n (makes lookups for any locale fall back to + # the I18n.default_locale when a translation cannot be found). + config.i18n.fallbacks = [I18n.default_locale] + + # Send deprecation notices to registered listeners. + config.active_support.deprecation = :notify + + # Use default logging formatter so that PID and timestamp are not suppressed. + # config.log_formatter = ::Logger::Formatter.new + config.log_formatter = ::Logger::Formatter.new + if ENV["RAILS_LOG_TO_STDOUT"].present? + logger = ActiveSupport::Logger.new(STDOUT) + logger.formatter = config.log_formatter + config.logger = ActiveSupport::TaggedLogging.new(logger) + end + + # Timber.io logger + send_logs_to_timber = ENV["SEND_LOGS_TO_TIMBER"] || "false" # <---- set to false to stop sending dev logs to Timber.io + log_device = send_logs_to_timber == "true" ? Timber::LogDevices::HTTP.new(ENV["TIMBER"]) : STDOUT + logger = Timber::Logger.new(log_device) + logger.level = config.log_level + config.logger = ActiveSupport::TaggedLogging.new(logger) + + # Do not dump schema after migrations. + config.active_record.dump_schema_after_migration = false + + config.app_domain = "localhost:3000" + + config.action_mailer.delivery_method = :smtp + config.action_mailer.perform_deliveries = true + config.action_mailer.default_url_options = { host: config.app_domain } + config.action_mailer.smtp_settings = { + address: "smtp.gmail.com", + port: "587", + enable_starttls_auto: true, + user_name: ENV["DEVELOPMENT_EMAIL_USERNAME"], + password: ENV["DEVELOPMENT_EMAIL_PASSWORD"], + authentication: :plain, + domain: "localhost:3000" + } + + config.action_mailer.preview_path = "#{Rails.root}/spec/mailers/previews" +end diff --git a/config/initializers/airbrake.rb b/config/initializers/airbrake.rb index 0a3fffc7..10fd3a89 100644 --- a/config/initializers/airbrake.rb +++ b/config/initializers/airbrake.rb @@ -41,7 +41,7 @@ # environments. # NOTE: This option *does not* work if you don't set the 'environment' option. # https://github.com/airbrake/airbrake-ruby#ignore_environments - c.ignore_environments = %w[test development] + c.ignore_environments = %w[test development profile] # A list of parameters that should be filtered out of what is sent to # Airbrake. By default, all "password" attributes will have their contents diff --git a/config/initializers/prometheus.rb b/config/initializers/prometheus.rb new file mode 100644 index 00000000..40e3a5b2 --- /dev/null +++ b/config/initializers/prometheus.rb @@ -0,0 +1,6 @@ +if Rails.env != "test" + require 'prometheus_exporter/middleware' + + # This reports stats per request like HTTP status and timings + Rails.application.middleware.unshift PrometheusExporter::Middleware +end diff --git a/config/initializers/timeout.rb b/config/initializers/timeout.rb index 076288f3..2c230b31 100644 --- a/config/initializers/timeout.rb +++ b/config/initializers/timeout.rb @@ -1,4 +1,4 @@ -if Rails.env.development? && ENV["RACK_TIMEOUT_WAIT_TIMEOUT"].nil? +if (Rails.env.development? || Rails.env.profile?) && ENV["RACK_TIMEOUT_WAIT_TIMEOUT"].nil? ENV["RACK_TIMEOUT_WAIT_TIMEOUT"] = "100000" ENV["RACK_TIMEOUT_SERVICE_TIMEOUT"] = "100000" end diff --git a/config/newrelic.yml b/config/newrelic.yml new file mode 100644 index 00000000..41cfbd9e --- /dev/null +++ b/config/newrelic.yml @@ -0,0 +1,70 @@ +# +# This file configures the New Relic Agent. New Relic monitors Ruby, Java, +# .NET, PHP, Python, Node, and Go applications with deep visibility and low +# overhead. For more information, visit www.newrelic.com. +# +# Generated April 3, 2023 +# +# This configuration file is custom generated for NewRelic Administration +# +# For full documentation of agent configuration options, please refer to +# https://docs.newrelic.com/docs/agents/ruby-agent/installation-configuration/ruby-agent-configuration + +common: &default_settings + # Required license key associated with your New Relic account. + license_key: <%= ENV["NEWRELIC_AUTHENTICATION"] %> + + # Your application name. Renaming here affects where data displays in New + # Relic. For more details, see https://docs.newrelic.com/docs/apm/new-relic-apm/maintenance/renaming-applications + app_name: biggy poppy + + distributed_tracing: + enabled: true + + # To disable the agent regardless of other settings, uncomment the following: + # agent_enabled: false + + # Logging level for log/newrelic_agent.log + log_level: info + + application_logging: + # If `true`, all logging-related features for the agent can be enabled or disabled + # independently. If `false`, all logging-related features are disabled. + enabled: true + forwarding: + # If `true`, the agent captures log records emitted by this application. + enabled: true + # Defines the maximum number of log records to buffer in memory at a time. + max_samples_stored: 10000 + metrics: + # If `true`, the agent captures metrics related to logging for this application. + enabled: true + local_decorating: + # If `true`, the agent decorates logs with metadata to link to entities, hosts, traces, and spans. + # This requires a log forwarder to send your log files to New Relic. + # This should not be used when forwarding is enabled. + enabled: false + +# Environment-specific settings are in this section. +# RAILS_ENV or RACK_ENV (as appropriate) is used to determine the environment. +# If your application has other named environments, configure them here. +development: + <<: *default_settings + app_name: biggy poppy (Development) + +test: + <<: *default_settings + # It doesn't make sense to report to New Relic from automated test runs. + monitor_mode: false + +staging: + <<: *default_settings + app_name: biggy poppy (Staging) + +profile: + <<: *default_settings + app_name: biggy poppy (Profile) + +production: + <<: *default_settings + diff --git a/config/secrets.yml b/config/secrets.yml index 73f5e05c..f009b75c 100644 --- a/config/secrets.yml +++ b/config/secrets.yml @@ -22,3 +22,6 @@ test: production: secret_key_base: <%= ENV["SECRET_KEY_BASE"] %> +profile: + secret_key_base: 42dd7834039ebbea271af22635a6782ee15e519b14629c5276bfcdd4cff841e9926994784bb43a335a8f8c9739bb254ea3afe831839d4dc65654ec7516ec25f0 + diff --git a/config/skylight.yml b/config/skylight.yml deleted file mode 100644 index d40c2c26..00000000 --- a/config/skylight.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -# The authentication token for the application. -authentication: <%= ENV['SKYLIGHT_AUTHENTICATION'] %> diff --git a/config/webpacker.yml b/config/webpacker.yml index 2dfcd170..472b39f8 100644 --- a/config/webpacker.yml +++ b/config/webpacker.yml @@ -54,3 +54,8 @@ production: # Cache manifest.json for performance cache_manifest: true + +profile: + <<: *default + compile: false + cache_manifest: true diff --git a/prometheus/custom_collector.rb b/prometheus/custom_collector.rb new file mode 100644 index 00000000..e13a2704 --- /dev/null +++ b/prometheus/custom_collector.rb @@ -0,0 +1,15 @@ +class CustomCollector < PrometheusExporter::Server::TypeCollector + unless defined? Rails + require File.expand_path("../../config/environment", __FILE__) + end + + def type + "spajic_posts" + end + + def metrics + spajic_posts_gague = PrometheusExporter::Metric::Gauge.new('spajic_posts', 'number of spajic posts') + spajic_posts_gague.observe User.find_by_name('Dorthea Paucek').articles.count + [spajic_posts_gague] + end +end diff --git a/prometheus/docker-compose.yml b/prometheus/docker-compose.yml new file mode 100644 index 00000000..28a0c662 --- /dev/null +++ b/prometheus/docker-compose.yml @@ -0,0 +1,24 @@ +version: '2' +services: + dockerhost: + image: qoomon/docker-host + cap_add: [ 'NET_ADMIN', 'NET_RAW' ] + mem_limit: 8M + restart: on-failure + prometheus: + depends_on: [ dockerhost ] + image: prom/prometheus:0.18.0 + volumes: + - ./prometheus.yml:/etc/prometheus/prometheus.yml + command: + - '-config.file=/etc/prometheus/prometheus.yml' + ports: + - '9090:9090' + grafana: + image: grafana/grafana:3.0.0-beta7 + environment: + - GF_SECURITY_ADMIN_PASSWORD=pass + depends_on: + - prometheus + ports: + - "3030:3000" diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 00000000..de2a3fe6 --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,11 @@ +global: + scrape_interval: 5s + external_labels: + monitor: 'my-monitor' +scrape_configs: + - job_name: 'prometheus' + target_groups: + - targets: ['localhost:9090'] + - job_name: 'devdev' + target_groups: + - targets: ['dockerhost:9394']