Add Pitchfork/Raindrops middleware (#66)

* Add Periodic Stats

Add listener_address as Instance attributes of Promenade::Raindrops::Stats

Fix guard clauses

💇‍♀️

Update dev dependencies

Update Bundler version

* Fix coverage for Raindrops stats

* Test coverage for failing periodic instrumentation

* Ignore rubocop

I think the complexity in the setup method is unavoidable,
and trying to factor this logic into another method doesn't improve
anything.

The assignment in if conditions stuff is a bit suspect, but I don't
really mind it ...

* Add a spec to check the pid_provider selection logic

* Add Periodic Stats

Add listener_address as Instance attributes of Promenade::Raindrops::Stats

Fix guard clauses

💇‍♀️

Update dev dependencies

Update Bundler version

* Add Pitchfork/Raindrops middleware

Remove Periodic Stats

Fix rebase

Ignore railties

add codecove yml path

Remove unused settings

Ignore Railtie from codecove

Update codecove

Add Codecov yml path

Ignore railtie with Simplecov

---------

Co-authored-by: Ed Robinson <edward-robinson@cookpad.com>

Gemfile.lock

@@ -188,6 +188,7 @@ GEM
       thor (~> 1.0, >= 1.2.2)
       zeitwerk (~> 2.6)
     rainbow (3.1.1)
+    raindrops (0.20.1)
     rake (13.1.0)
     rb_sys (0.9.87)
     rdoc (6.6.3.1)
@@ -271,6 +272,7 @@ GEM
     zeitwerk (2.6.13)
 
 PLATFORMS
+  arm64-darwin-21
   arm64-darwin-22
   x86_64-darwin-22
   x86_64-linux
@@ -281,6 +283,7 @@ DEPENDENCIES
   climate_control
   promenade!
   rails (> 3.0, < 8.0)
+  raindrops
   rake
   rspec (~> 3.11)
   rspec-rails (~> 5.1)

lib/promenade/pitchfork/middleware.rb

@@ -0,0 +1,22 @@
+require "promenade/pitchfork/stats"
+
+module Promenade
+  module Pitchfork
+    class Middleware
+      RACK_AFTER_REPLY = "rack.after_reply".freeze
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        if env.key?(RACK_AFTER_REPLY)
+          env[RACK_AFTER_REPLY] << -> {
+            ::Promenade::Pitchfork::Stats.instrument
+          }
+        end
+        @app.call(env)
+      end
+    end
+  end
+end

lib/promenade/pitchfork/stats.rb

@@ -0,0 +1,62 @@
+require "promenade/raindrops/stats"
+
+module Promenade
+  module Pitchfork
+    class Stats
+      Promenade.gauge :pitchfork_workers_count do
+        doc "Number of workers configured"
+      end
+
+      Promenade.gauge :pitchfork_live_workers_count do
+        doc "Number of live / booted workers"
+      end
+
+      Promenade.gauge :pitchfork_capacity do
+        doc "Number of workers that are currently idle"
+      end
+
+      Promenade.gauge :pitchfork_busy_percent do
+        doc "Percentage of workers that are currently busy"
+      end
+
+      def initialize
+        return unless defined?(::Pitchfork) && defined?(::Pitchfork::Info)
+
+        @workers_count = ::Pitchfork::Info.workers_count
+        @live_workers_count = ::Pitchfork::Info.live_workers_count
+
+        raindrops_stats = Raindrops::Stats.new
+
+        @active_workers = raindrops_stats.active_workers || 0
+        @queued_requests = raindrops_stats.queued_requests || 0
+      end
+
+      def instrument
+        Promenade.metric(:pitchfork_workers_count).set({}, workers_count)
+        Promenade.metric(:pitchfork_live_workers_count).set({}, live_workers_count)
+        Promenade.metric(:pitchfork_capacity).set({}, capacity)
+        Promenade.metric(:pitchfork_busy_percent).set({}, busy_percent)
+      end
+
+      def self.instrument
+        new.instrument
+      end
+
+      private
+
+        attr_reader :workers_count, :live_workers_count, :active_workers, :queued_requests
+
+        def capacity
+          return 0 if live_workers_count.nil? || live_workers_count == 0
+
+          live_workers_count - active_workers
+        end
+
+        def busy_percent
+          return 0 if live_workers_count == 0
+
+          (active_workers.to_f / live_workers_count) * 100
+        end
+    end
+  end
+end

lib/promenade/railtie.rb

@@ -13,6 +13,16 @@ class Railtie < ::Rails::Railtie
         Promenade::Client::Rack::HTTPRequestDurationCollector
       Rails.application.config.middleware.insert 0,
         Promenade::Client::Rack::HTTPRequestQueueTimeCollector
+
+      if defined?(::Raindrops)
+        require "promenade/raindrops/middleware"
+        Rails.application.config.middleware.use Promenade::Raindrops::Middleware
+      end
+
+      if defined?(::Pitchfork)
+        require "promenade/pitchfork/middleware"
+        Rails.application.config.middleware.use Promenade::Pitchfork::Middleware
+      end
     end
   end
 end

lib/promenade/raindrops/middleware.rb

@@ -0,0 +1,32 @@
+require "promenade/raindrops/stats"
+
+module Promenade
+  module Raindrops
+    class Middleware
+      RACK_AFTER_REPLY = "rack.after_reply".freeze
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        if env.key?(RACK_AFTER_REPLY)
+          env[RACK_AFTER_REPLY] << -> { instrument }
+        end
+        @app.call(env)
+      end
+
+      private
+
+        def tcp_listener_names
+          ::Pitchfork.listener_names
+        end
+
+        def instrument
+          tcp_listener_names.each do |name|
+            Promenade::Raindrops::Stats.instrument(listener_address: name)
+          end
+        end
+    end
+  end
+end

lib/promenade/raindrops/stats.rb

@@ -0,0 +1,42 @@
+begin
+  require "raindrops"
+rescue LoadError
+  # No raindrops available, dont do anything
+end
+
+module Promenade
+  module Raindrops
+    Promenade.gauge :rack_active_workers do
+      doc "Number of active workers in the Application Server"
+    end
+
+    Promenade.gauge :rack_queued_requests do
+      doc "Number of requests waiting to be processed by the Application Server"
+    end
+
+    class Stats
+      attr_reader :active_workers, :queued_requests, :listener_address
+
+      def initialize(listener_address: nil)
+        return unless defined?(::Raindrops)
+        return unless defined?(::Raindrops::Linux.tcp_listener_stats)
+
+        @listener_address = listener_address || "127.0.0.1:#{ENV.fetch('PORT', 3000)}"
+
+        stats = ::Raindrops::Linux.tcp_listener_stats([@listener_address])[@listener_address]
+
+        @active_workers = stats.active
+        @queued_requests = stats.queued
+      end
+
+      def instrument
+        Promenade.metric(:rack_active_workers).set({}, active_workers) if active_workers
+        Promenade.metric(:rack_queued_requests).set({}, queued_requests) if queued_requests
+      end
+
+      def self.instrument(listener_address: nil)
+        new(listener_address: listener_address).instrument
+      end
+    end
+  end
+end

promenade.gemspec

@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "byebug"
   spec.add_development_dependency "climate_control"
   spec.add_development_dependency "rails", "> 3.0", "< 8.0"
+  spec.add_development_dependency "raindrops"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec", "~> 3.11"
   spec.add_development_dependency "rspec-rails", "~> 5.1"

spec/promenade/pitchfork/middleware_spec.rb

@@ -0,0 +1,15 @@
+require "promenade/pitchfork/middleware"
+
+RSpec.describe Promenade::Pitchfork::Middleware do
+  let(:app) { double(:app, call: nil) }
+
+  it "is add it's instrumentaion to the rack.after_reply" do
+    stats = class_spy("Promenade::Pitchfork::Stats").as_stubbed_const
+
+    after_reply = []
+    described_class.new(app).call({ "rack.after_reply" => after_reply })
+    after_reply.each(&:call)
+
+    expect(stats).to have_received(:instrument)
+  end
+end

spec/promenade/pitchfork/stats_spec.rb

@@ -0,0 +1,52 @@
+require "spec_helper"
+require "promenade/pitchfork/stats"
+
+RSpec.describe Promenade::Pitchfork::Stats do
+  let(:pitchfork_info) { class_double("Pitchfork::Info") }
+  let(:raindrops_stats) { instance_double("Promenade::Raindrops::Stats", active_workers: 6, queued_requests: 2) }
+
+  before do
+    stub_const("Pitchfork::Info", pitchfork_info)
+    allow(pitchfork_info).to receive(:workers_count).and_return(10)
+    allow(pitchfork_info).to receive(:live_workers_count).and_return(8)
+
+    allow(Promenade::Raindrops::Stats).to receive(:new).and_return(raindrops_stats)
+  end
+
+  describe "#instrument" do
+    let(:metric) { instance_double("Promenade::Metric") }
+
+    before do
+      allow(Promenade).to receive(:metric).and_return(metric)
+      allow(metric).to receive(:set)
+    end
+
+    it "sets the metrics correctly" do
+      stats = Promenade::Pitchfork::Stats.new
+
+      expect(Promenade).to receive(:metric).with(:pitchfork_workers_count).and_return(metric)
+      expect(Promenade).to receive(:metric).with(:pitchfork_live_workers_count).and_return(metric)
+      expect(Promenade).to receive(:metric).with(:pitchfork_capacity).and_return(metric)
+      expect(Promenade).to receive(:metric).with(:pitchfork_busy_percent).and_return(metric)
+
+      expect(metric).to receive(:set).with({}, 10)
+      expect(metric).to receive(:set).with({}, 8)
+      expect(metric).to receive(:set).with({}, 2)
+      expect(metric).to receive(:set).with({}, 75.0)
+
+      stats.instrument
+    end
+  end
+
+  describe ".instrument" do
+    it "calls the instance method instrument" do
+      stats_instance = instance_double("Promenade::Pitchfork::Stats")
+      allow(Promenade::Pitchfork::Stats).to receive(:new).and_return(stats_instance)
+      allow(stats_instance).to receive(:instrument)
+
+      Promenade::Pitchfork::Stats.instrument
+
+      expect(stats_instance).to have_received(:instrument)
+    end
+  end
+end

spec/promenade/raindrops/middleware_spec.rb

@@ -0,0 +1,21 @@
+require "promenade/raindrops/middleware"
+
+RSpec.describe Promenade::Raindrops::Middleware do
+  let(:app) { double(:app, call: nil) }
+  let(:listener_address) { "127.0.0.1:#{ENV.fetch('PORT', 3000)}" }
+  let(:pitchfork) { class_double("Pitchfork").as_stubbed_const }
+
+  before do
+    allow(pitchfork).to receive(:listener_names).and_return([listener_address])
+  end
+
+  it "is add it's instrumentaion to the rack.after_reply" do
+    stats = class_spy("Promenade::Raindrops::Stats").as_stubbed_const
+
+    after_reply = []
+    described_class.new(app).call({ "rack.after_reply" => after_reply })
+    after_reply.each(&:call)
+
+    expect(stats).to have_received(:instrument).with(listener_address: listener_address)
+  end
+end

spec/promenade/raindrops/stats_spec.rb

@@ -0,0 +1,30 @@
+require "spec_helper"
+require "promenade/raindrops/stats"
+
+RSpec.describe Promenade::Raindrops::Stats do
+  let(:listen_stats) { instance_double("Raindrops::Linux::ListenStats", active: 1, queued: 1) }
+  let(:listener_address) { "127.0.0.1:#{ENV.fetch('PORT', 3000)}" }
+
+  before do
+    allow(Raindrops::Linux).to receive(:tcp_listener_stats).and_return({ listener_address => listen_stats })
+  end
+
+  describe "#instrument" do
+    let(:metric) { instance_double("Promenade::Metric") }
+
+    before do
+      allow(Promenade).to receive(:metric).and_return(metric)
+      allow(metric).to receive(:set)
+    end
+
+    it "sets the metrics correctly" do
+      expect(Promenade).to receive(:metric).with(:rack_active_workers).and_return(metric)
+      expect(Promenade).to receive(:metric).with(:rack_queued_requests).and_return(metric)
+
+      expect(metric).to receive(:set).with({}, 1)
+      expect(metric).to receive(:set).with({}, 1)
+
+      described_class.instrument
+    end
+  end
+end

spec/spec_helper.rb

@@ -2,6 +2,7 @@
 SimpleCov.minimum_coverage 99
 SimpleCov.start do
   add_filter "/spec/"
+  add_filter "/lib/promenade/railtie.rb"
 end
 
 if ENV["CI"]