Namespacing: Profiling (#1849)

Co-authored-by: Ivo Anjo <[email protected]>

.rubocop.yml

@@ -21,7 +21,7 @@ AllCops:
     - 'lib/datadog/**/vendor/**/*.rb'
     - 'integration/apps/*/bin/*'
     - 'integration/apps/*/Gemfile'
-    - 'lib/ddtrace/profiling/pprof/pprof_pb.rb'
+    - 'lib/datadog/profiling/pprof/pprof_pb.rb'
   NewCops: disable # Don't allow new cops to be enabled implicitly.
 
 # 80 characters is a nice goal, but not worth currently changing in existing

.rubocop_todo.yml

@@ -90,13 +90,13 @@ Lint/MissingSuper:
     - 'lib/ddtrace/opentracer/scope.rb'
     - 'lib/ddtrace/opentracer/span.rb'
     - 'lib/ddtrace/opentracer/span_context.rb'
-    - 'lib/ddtrace/profiling/collectors/stack.rb'
-    - 'lib/ddtrace/profiling/pprof/converter.rb'
-    - 'lib/ddtrace/profiling/pprof/template.rb'
-    - 'lib/ddtrace/profiling/recorder.rb'
-    - 'lib/ddtrace/profiling/scheduler.rb'
-    - 'lib/ddtrace/profiling/transport/http/api/instance.rb'
-    - 'lib/ddtrace/profiling/transport/http/api/spec.rb'
+    - 'lib/datadog/profiling/collectors/stack.rb'
+    - 'lib/datadog/profiling/pprof/converter.rb'
+    - 'lib/datadog/profiling/pprof/template.rb'
+    - 'lib/datadog/profiling/recorder.rb'
+    - 'lib/datadog/profiling/scheduler.rb'
+    - 'lib/datadog/profiling/transport/http/api/instance.rb'
+    - 'lib/datadog/profiling/transport/http/api/spec.rb'
     - 'lib/ddtrace/sampler.rb'
     - 'lib/ddtrace/sampling/matcher.rb'
     - 'lib/ddtrace/sampling/rate_limiter.rb'
@@ -115,7 +115,7 @@ Lint/MissingSuper:
 # Offense count: 1
 Lint/StructNewOverride:
   Exclude:
-    - 'lib/ddtrace/profiling/pprof/converter.rb'
+    - 'lib/datadog/profiling/pprof/converter.rb'
 
 # Offense count: 1
 # Configuration parameters: AllowComments.
@@ -164,10 +164,10 @@ Performance/MethodObjectAsBlock:
     - 'lib/datadog/ci/contrib/cucumber/formatter.rb'
     - 'lib/ddtrace/contrib/delayed_job/plugin.rb'
     - 'lib/ddtrace/pipeline.rb'
-    - 'lib/ddtrace/profiling/collectors/stack.rb'
-    - 'lib/ddtrace/profiling/pprof/builder.rb'
-    - 'lib/ddtrace/profiling/pprof/stack_sample.rb'
-    - 'lib/ddtrace/profiling/pprof/template.rb'
+    - 'lib/datadog/profiling/collectors/stack.rb'
+    - 'lib/datadog/profiling/pprof/builder.rb'
+    - 'lib/datadog/profiling/pprof/stack_sample.rb'
+    - 'lib/datadog/profiling/pprof/template.rb'
     - 'spec/datadog/ci/contrib/cucumber/patcher_spec.rb'
 
 # Offense count: 6
@@ -363,29 +363,29 @@ RSpec/VerifiedDoubles:
     - 'spec/ddtrace/opentracer/tracer_spec.rb'
     - 'spec/ddtrace/patcher_spec.rb'
     - 'spec/ddtrace/pin_spec.rb'
-    - 'spec/ddtrace/profiling/backtrace_location_spec.rb'
-    - 'spec/ddtrace/profiling/collectors/stack_spec.rb'
-    - 'spec/ddtrace/profiling/encoding/profile_spec.rb'
-    - 'spec/ddtrace/profiling/events/stack_spec.rb'
-    - 'spec/ddtrace/profiling/exporter_spec.rb'
-    - 'spec/ddtrace/profiling/ext/cthread_spec.rb'
-    - 'spec/ddtrace/profiling/ext/forking_spec.rb'
-    - 'spec/ddtrace/profiling/flush_spec.rb'
-    - 'spec/ddtrace/profiling/pprof/builder_spec.rb'
-    - 'spec/ddtrace/profiling/pprof/converter_spec.rb'
-    - 'spec/ddtrace/profiling/pprof/message_set_spec.rb'
-    - 'spec/ddtrace/profiling/pprof/payload_spec.rb'
-    - 'spec/ddtrace/profiling/pprof/template_spec.rb'
-    - 'spec/ddtrace/profiling/transport/http/api/endpoint_spec.rb'
-    - 'spec/ddtrace/profiling/transport/http/api/instance_spec.rb'
-    - 'spec/ddtrace/profiling/transport/http/builder_spec.rb'
-    - 'spec/ddtrace/profiling/transport/http/client_spec.rb'
-    - 'spec/ddtrace/profiling/transport/http/response_spec.rb'
-    - 'spec/ddtrace/profiling/transport/http_spec.rb'
-    - 'spec/ddtrace/profiling/transport/io/client_spec.rb'
-    - 'spec/ddtrace/profiling/transport/io/response_spec.rb'
-    - 'spec/ddtrace/profiling/transport/io_spec.rb'
-    - 'spec/ddtrace/profiling/transport/parcel_spec.rb'
+    - 'spec/datadog/profiling/backtrace_location_spec.rb'
+    - 'spec/datadog/profiling/collectors/stack_spec.rb'
+    - 'spec/datadog/profiling/encoding/profile_spec.rb'
+    - 'spec/datadog/profiling/events/stack_spec.rb'
+    - 'spec/datadog/profiling/exporter_spec.rb'
+    - 'spec/datadog/profiling/ext/cthread_spec.rb'
+    - 'spec/datadog/profiling/ext/forking_spec.rb'
+    - 'spec/datadog/profiling/flush_spec.rb'
+    - 'spec/datadog/profiling/pprof/builder_spec.rb'
+    - 'spec/datadog/profiling/pprof/converter_spec.rb'
+    - 'spec/datadog/profiling/pprof/message_set_spec.rb'
+    - 'spec/datadog/profiling/pprof/payload_spec.rb'
+    - 'spec/datadog/profiling/pprof/template_spec.rb'
+    - 'spec/datadog/profiling/transport/http/api/endpoint_spec.rb'
+    - 'spec/datadog/profiling/transport/http/api/instance_spec.rb'
+    - 'spec/datadog/profiling/transport/http/builder_spec.rb'
+    - 'spec/datadog/profiling/transport/http/client_spec.rb'
+    - 'spec/datadog/profiling/transport/http/response_spec.rb'
+    - 'spec/datadog/profiling/transport/http_spec.rb'
+    - 'spec/datadog/profiling/transport/io/client_spec.rb'
+    - 'spec/datadog/profiling/transport/io/response_spec.rb'
+    - 'spec/datadog/profiling/transport/io_spec.rb'
+    - 'spec/datadog/profiling/transport/parcel_spec.rb'
     - 'spec/ddtrace/profiling_spec.rb'
     - 'spec/ddtrace/runtime/metrics_spec.rb'
     - 'spec/ddtrace/tasks/exec_spec.rb'

Gemfile

@@ -71,7 +71,7 @@ gem 'opentracing', '>= 0.4.1'
 
 # Profiler optional dependencies
 # NOTE: We're excluding versions 3.7.0 and 3.7.1 for the reasons documented in #1424 and the big comment in
-#       lib/ddtrace/profiling.rb: it breaks for some older rubies in CI without BUNDLE_FORCE_RUBY_PLATFORM=true.
+#       lib/datadog/profiling.rb: it breaks for some older rubies in CI without BUNDLE_FORCE_RUBY_PLATFORM=true.
 #       Since most of our customers won't have BUNDLE_FORCE_RUBY_PLATFORM=true, it's not something we want to add
 #       to our CI, so we just shortcut and exclude specific versions that were affecting our CI.
 if RUBY_PLATFORM != 'java'

benchmarks/profiler_sample_loop.rb

@@ -23,10 +23,10 @@ def create_profiler
     end
 
     # Stop background threads
-    Datadog.profiler.shutdown!
+    Datadog.shutdown!
 
     # Call collection directly
-    @stack_collector = Datadog.profiler.collectors.first
+    @stack_collector = Datadog.send(:components).profiler.collectors.first
     @recorder = @stack_collector.recorder
   end
 

benchmarks/profiler_submission.rb

@@ -37,10 +37,10 @@ def create_profiler
     end
 
     # Stop background threads
-    Datadog.profiler.shutdown!
+    Datadog.shutdown!
 
     # Call exporter directly
-    @exporter = Datadog.profiler.scheduler.exporters.first
+    @exporter = Datadog.send(:components).profiler.scheduler.exporters.first
     @flush = Marshal.load(
       Zlib::GzipReader.new(File.open(ENV['FLUSH_DUMP_FILE'] || 'benchmarks/data/profiler-submission-marshal.gz'))
     )

bin/ddtracerb

@@ -1,15 +1,15 @@
 #!/usr/bin/env ruby
-require 'ddtrace/tasks/exec'
-require 'ddtrace/tasks/help'
+require 'datadog/profiling/tasks/exec'
+require 'datadog/profiling/tasks/help'
 
 command = ARGV.shift
 
 case command
 when 'exec'
-  Datadog::Tasks::Exec.new(ARGV).run
+  Datadog::Profiling::Tasks::Exec.new(ARGV).run
 when 'help', '--help'
-  Datadog::Tasks::Help.new.run
+  Datadog::Profiling::Tasks::Help.new.run
 else
   puts "Command '#{command}' is not valid for ddtrace."
-  Datadog::Tasks::Help.new.run
+  Datadog::Profiling::Tasks::Help.new.run
 end

docs/ProfilingDevelopment.md

@@ -6,7 +6,7 @@ For a more practical view of getting started with development of `ddtrace`, see
 
 ## Profiling components high-level view
 
-Components below live inside <../lib/ddtrace/profiling>:
+Components below live inside <../lib/datadog/profiling>:
 
 * `Collectors::Stack`: Collects stack trace samples from Ruby threads for both CPU-time (if available) and wall-clock.
   Runs on its own background thread.
@@ -15,9 +15,9 @@ Components below live inside <../lib/ddtrace/profiling>:
 * `Events::Stack`, `Events::StackSample`: Entity classes used to represent stacks.
 * `Ext::Forking`: Monkey patches `Kernel#fork`, adding a `Kernel#at_fork` callback mechanism which is used to restore
   profiling abilities after the VM forks (such as re-instrumenting the main thread, and restarting profiler threads).
-* `Pprof::*` (in <../lib/ddtrace/profiling/pprof>): Converts samples captured in the `Recorder` into the pprof format.
+* `Pprof::*` (in <../lib/datadog/profiling/pprof>): Converts samples captured in the `Recorder` into the pprof format.
 * `Tasks::Setup`: Takes care of loading our extensions/monkey patches to handle fork().
-* `Transport::*` (in <../lib/ddtrace/profiling/transport>): Implements transmission of profiling payloads to the Datadog agent
+* `Transport::*` (in <../lib/datadog/profiling/transport>): Implements transmission of profiling payloads to the Datadog agent
   or backend.
 * `TraceIdentifiers::*`: Used to retrieve trace id and span id from tracers, to be used to connect traces to profiles.
 * `BacktraceLocation`: Entity class used to represent an entry in a stack trace.
@@ -34,8 +34,8 @@ Components below live inside <../lib/ddtrace/profiling>:
 When started via `ddtracerb exec` (together with `DD_PROFILING_ENABLED=true`), initialization goes through the following
 flow:
 
-1. <../lib/ddtrace/profiling/preload.rb> triggers the creation of the `Datadog.profiler` instance by calling the method
-2. `Datadog.profiler` is handled by `Datadog::Configuration`, which triggers the configuration of `ddtrace` components
+1. <../lib/datadog/profiling/preload.rb> triggers the creation of the profiler instance by calling the method `Datadog::Profiling.start_if_enabled`
+2. The profiler instance is handled by `Datadog::Configuration`, which triggers the configuration of `ddtrace` components
    in `#build_components`
 3. Inside `Datadog::Components`, the `build_profiler` method triggers the execution of the `Tasks::Setup`
 4. The `Setup` task activates our extensions

integration/apps/rack/app/acme.rb

@@ -89,7 +89,7 @@ def check(request)
       def detailed_check(request)
         ['200', { 'Content-Type' => 'application/json'}, [JSON.pretty_generate(
           webserver_process: $PROGRAM_NAME,
-          profiler_available: !!Datadog.profiler,
+          profiler_available: Datadog::Profiling.start_if_enabled,
           # NOTE: Threads can't be named on Ruby 2.1 and 2.2
           profiler_threads: ((Thread.list.map(&:name).select { |it| it && it.include?('Profiling') }) unless RUBY_VERSION < '2.3')
         )], "\n"]

integration/apps/rack/app/resque_background_job.rb

@@ -28,7 +28,7 @@ def self.perform(key, value)
       key: key,
       value: value,
       resque_process: $PROGRAM_NAME,
-      profiler_available: !!Datadog.profiler,
+      profiler_available: Datadog::Profiling.start_if_enabled,
       # NOTE: Threads can't be named on Ruby 2.1 and 2.2
       profiler_threads: ((Thread.list.map(&:name).select { |it| it && it.include?('Profiling') }) unless RUBY_VERSION < '2.3')
     ))

integration/apps/rack/app/sidekiq_background_job.rb

@@ -26,7 +26,7 @@ def perform(key, value)
       key: key,
       value: value,
       sidekiq_process: $PROGRAM_NAME,
-      profiler_available: !!Datadog.profiler,
+      profiler_available: Datadog::Profiling.start_if_enabled,
       # NOTE: Threads can't be named on Ruby 2.1 and 2.2
       profiler_threads: ((Thread.list.map(&:name).select { |it| it && it.include?('Profiling') }) unless RUBY_VERSION < '2.3')
     ))

integration/apps/rails-five/app/controllers/health_controller.rb

@@ -12,7 +12,7 @@ def check
   def detailed_check
     render json: {
       webserver_process: $PROGRAM_NAME,
-      profiler_available: !!Datadog.profiler,
+      profiler_available: Datadog::Profiling.start_if_enabled,
       profiler_threads: Thread.list.map(&:name).select { |it| it && it.include?('Profiling') }
     }
   end

lib/datadog/core/configuration.rb

@@ -153,8 +153,7 @@ def configuration_for(target, option = nil)
 
       def_delegators \
         :components,
-        :health_metrics,
-        :profiler
+        :health_metrics
 
       def logger
         # avoid initializing components if they didn't already exist

lib/datadog/core/configuration/components.rb

@@ -2,7 +2,7 @@
 require 'datadog/core/configuration/agent_settings_resolver'
 require 'datadog/core/diagnostics/health'
 require 'datadog/core/logger'
-require 'ddtrace/profiling'
+require 'datadog/profiling'
 require 'datadog/core/runtime/metrics'
 require 'ddtrace/tracer'
 require 'ddtrace/trace_flush'
@@ -225,7 +225,7 @@ def build_profiler(settings, agent_settings, tracer)
             exporters = build_profiler_exporters(settings, agent_settings)
             scheduler = build_profiler_scheduler(settings, recorder, exporters)
 
-            Datadog::Profiler.new(collectors, scheduler)
+            Datadog::Profiling::Profiler.new(collectors, scheduler)
           end
 
           private

lib/datadog/core/configuration/settings.rb

@@ -6,7 +6,7 @@
 require 'ddtrace/ext/distributed'
 require 'datadog/core/environment/ext'
 require 'datadog/core/runtime/ext'
-require 'ddtrace/ext/profiling'
+require 'datadog/profiling/ext'
 require 'ddtrace/ext/sampling'
 require 'ddtrace/ext/test'
 
@@ -251,7 +251,7 @@ def initialize(*_)
           # @default `DD_PROFILING_ENABLED` environment variable, otherwise `false`
           # @return [Boolean]
           option :enabled do |o|
-            o.default { env_to_bool(Ext::Profiling::ENV_ENABLED, false) }
+            o.default { env_to_bool(Profiling::Ext::ENV_ENABLED, false) }
             o.lazy
           end
 
@@ -270,7 +270,7 @@ def initialize(*_)
             # Controls the maximum number of frames for each thread sampled. Can be tuned to avoid omitted frames in the
             # produced profiles. Increasing this may increase the overhead of profiling.
             option :max_frames do |o|
-              o.default { env_to_int(Ext::Profiling::ENV_MAX_FRAMES, 400) }
+              o.default { env_to_int(Profiling::Ext::ENV_MAX_FRAMES, 400) }
               o.lazy
             end
 
@@ -283,7 +283,7 @@ def initialize(*_)
                 # @default `DD_PROFILING_ENDPOINT_COLLECTION_ENABLED` environment variable, otherwise `true`
                 # @return [Boolean]
                 option :enabled do |o|
-                  o.default { env_to_bool(Ext::Profiling::ENV_ENDPOINT_COLLECTION_ENABLED, true) }
+                  o.default { env_to_bool(Profiling::Ext::ENV_ENDPOINT_COLLECTION_ENABLED, true) }
                   o.lazy
                 end
               end
@@ -298,7 +298,7 @@ def initialize(*_)
           settings :upload do
             option :timeout_seconds do |o|
               o.setter { |value| value.nil? ? 30.0 : value.to_f }
-              o.default { env_to_float(Ext::Profiling::ENV_UPLOAD_TIMEOUT, 30.0) }
+              o.default { env_to_float(Profiling::Ext::ENV_UPLOAD_TIMEOUT, 30.0) }
               o.lazy
             end
           end

lib/ddtrace/profiling.rb → lib/datadog/profiling.rb

@@ -82,6 +82,22 @@ def self.configuration
       )
     end
 
+    # Starts the profiler, if the profiler is supported by in
+    # this runtime environment and if the profiler has been enabled
+    # in configuration.
+    #
+    # @return [Boolean] `true` if the profiler has successfully started, otherwise `false`.
+    # @public_api
+    def self.start_if_enabled
+      # If the profiler was not previously touched, getting the profiler instance triggers start as a side-effect
+      # otherwise we get nil
+      profiler = Datadog.send(:components).profiler
+      # ...but we still try to start it BECAUSE if the process forks, the profiler will exist but may
+      # not yet have been started in the fork
+      profiler.start if profiler
+      !!profiler
+    end
+
     private_class_method def self.ruby_engine_unsupported?
       'JRuby is not supported' if RUBY_ENGINE == 'jruby'
     end
@@ -184,19 +200,19 @@ def self.configuration
     private_class_method def self.load_profiling
       return false unless supported?
 
-      require 'ddtrace/profiling/ext/forking'
-      require 'ddtrace/profiling/collectors/code_provenance'
-      require 'ddtrace/profiling/collectors/stack'
-      require 'ddtrace/profiling/exporter'
-      require 'ddtrace/profiling/recorder'
-      require 'ddtrace/profiling/scheduler'
-      require 'ddtrace/profiling/tasks/setup'
-      require 'ddtrace/profiling/transport/io'
-      require 'ddtrace/profiling/transport/http'
-      require 'ddtrace/profiling/profiler'
-      require 'ddtrace/profiling/native_extension'
-      require 'ddtrace/profiling/trace_identifiers/helper'
-      require 'ddtrace/profiling/pprof/pprof_pb'
+      require 'datadog/profiling/ext/forking'
+      require 'datadog/profiling/collectors/code_provenance'
+      require 'datadog/profiling/collectors/stack'
+      require 'datadog/profiling/exporter'
+      require 'datadog/profiling/recorder'
+      require 'datadog/profiling/scheduler'
+      require 'datadog/profiling/tasks/setup'
+      require 'datadog/profiling/transport/io'
+      require 'datadog/profiling/transport/http'
+      require 'datadog/profiling/profiler'
+      require 'datadog/profiling/native_extension'
+      require 'datadog/profiling/trace_identifiers/helper'
+      require 'datadog/profiling/pprof/pprof_pb'
 
       true
     end

lib/ddtrace/profiling/collectors/stack.rb → lib/datadog/profiling/collectors/stack.rb

@@ -1,8 +1,8 @@
 # typed: false
 
-require 'ddtrace/profiling/native_extension'
-require 'ddtrace/profiling/backtrace_location'
-require 'ddtrace/profiling/events/stack'
+require 'datadog/profiling/native_extension'
+require 'datadog/profiling/backtrace_location'
+require 'datadog/profiling/events/stack'
 require 'datadog/core/utils/only_once'
 require 'datadog/core/utils/time'
 require 'datadog/core/worker'

lib/ddtrace/profiling/encoding/profile.rb → lib/datadog/profiling/encoding/profile.rb

@@ -2,8 +2,8 @@
 require 'set'
 require 'time'
 
-require 'ddtrace/profiling/flush'
-require 'ddtrace/profiling/pprof/template'
+require 'datadog/profiling/flush'
+require 'datadog/profiling/pprof/template'
 
 module Datadog
   module Profiling

lib/ddtrace/profiling/events/stack.rb → lib/datadog/profiling/events/stack.rb

@@ -1,5 +1,5 @@
 # typed: true
-require 'ddtrace/profiling/event'
+require 'datadog/profiling/event'
 
 module Datadog
   module Profiling

lib/ddtrace/profiling/exporter.rb → lib/datadog/profiling/exporter.rb

@@ -1,5 +1,5 @@
 # typed: true
-require 'ddtrace/profiling/transport/io/client'
+require 'datadog/profiling/transport/io/client'
 
 module Datadog
   module Profiling