Merge pull request #22 from rails/rm-alert-on-stale-lock

Return system as down when the lock file is stale (older than 2 hours)

Author: rafaelfranca

.github/workflows/ci.yml

@@ -0,0 +1,21 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Ruby from .ruby-version
+      uses: ruby/setup-ruby@v1
+      with:
+        bundler-cache: true
+
+    - name: Run tests
+      run: bundle exec rake test

Gemfile

@@ -5,3 +5,8 @@ gem "puma", "~> 6.0"
 
 gem "capistrano3-puma"
 gem "capistrano-rvm"
+
+group :test do
+  gem "minitest", "~> 5.0"
+  gem "rack-test", "~> 2.0"
+end

Gemfile.lock

@@ -22,6 +22,7 @@ GEM
     i18n (1.14.7)
       concurrent-ruby (~> 1.0)
     logger (1.7.0)
+    minitest (5.26.0)
     net-scp (4.1.0)
       net-ssh (>= 2.6.5, < 8.0.0)
     net-sftp (4.0.0)
@@ -32,6 +33,8 @@ GEM
     puma (6.6.1)
       nio4r (~> 2.0)
     rack (2.2.20)
+    rack-test (2.2.0)
+      rack (>= 1.3)
     rake (13.3.0)
     sshkit (1.24.0)
       base64
@@ -47,8 +50,10 @@ PLATFORMS
 DEPENDENCIES
   capistrano-rvm
   capistrano3-puma
+  minitest (~> 5.0)
   puma (~> 6.0)
   rack (~> 2.2)
+  rack-test (~> 2.0)
 
 BUNDLED WITH
    2.5.22

README.md

@@ -9,3 +9,12 @@ Server management in encapsulated in the script `bin/server`:
     bin/server stop
 
 This webhook just touches a file meaning "we have been called". The docs server is responsible for monitoring the presence of said file somehow, and act accordingly.
+
+## Testing
+
+The application includes a comprehensive test suite using minitest and rack-test:
+
+```bash
+# Run all tests
+bundle exec rake test
+```

Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "."
+  t.test_files = FileList["test/**/*_test.rb"]
+  t.verbose = true
+end
+
+task default: :test

config.ru

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
-require "fileutils"
-require "rack"
 require "logger"
+require "rack"
+require_relative "lib/rails_master_hook_app"
 
 # Setup logger - log to STDOUT for systemd
-logger = Logger.new(STDOUT)
+logger = Logger.new($stdout)
 logger.level = ENV["LOG_LEVEL"] ? Logger.const_get(ENV["LOG_LEVEL"].upcase) : Logger::INFO
 logger.formatter = proc do |severity, datetime, progname, msg|
   "#{severity}: #{msg}\n"
@@ -14,39 +14,4 @@ end
 # Use Rack::CommonLogger for HTTP request logging
 use Rack::CommonLogger, logger
 
-run_file  = ENV["RUN_FILE"] || "#{__dir__}/run-rails-master-hook"
-scheduled = <<EOS
-Rails master hook tasks scheduled:
-
-* updates the local checkout
-* updates Rails Contributors
-* generates and publishes edge docs
-
-If a new stable tag is detected it also
-
-* generates and publishes stable docs
-
-This needs typically a few minutes.
-EOS
-
-map "/rails-master-hook" do
-  run ->(env) do
-    request_method = env["REQUEST_METHOD"]
-
-    if request_method == "POST"
-      logger.info "Triggering Rails master hook by touching #{run_file}"
-      FileUtils.touch(run_file)
-      logger.info "Rails master hook scheduled successfully"
-      [200, {"Content-Type" => "text/plain", "Content-Length" => scheduled.length.to_s}, [scheduled]]
-    else
-      logger.warn "Rejected non-POST request (#{request_method}) to /rails-master-hook"
-      [404, {"Content-Type" => "text/plain", "Content-Length" => "0"}, []]
-    end
-  end
-end
-
-map "/" do
-  run ->(_env) do
-    [200, {"Content-Type" => "text/plain", "Content-Length" => "4"}, ["PONG"]]
-  end
-end
+run RailsMasterHookApp.new(logger: logger)

config/deploy/production.rb

@@ -8,6 +8,7 @@
 
 set :puma_service_unit_env_vars, %w[
   RUN_FILE=/home/rails/rails-master-hook/run-rails-master-hook
+  LOCK_FILE=/home/rails/rails-master-hook/lock-rails-master-hook
 ]
 set :puma_access_log, "journal"
 set :puma_error_log, "journal"

lib/lockfile_checker.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# Utility class for checking if lock files are stale
+#
+# A lock file is considered stale if it's older than 2 hours,
+# which indicates a potentially stuck or long-running process.
+class LockfileChecker
+  # @param lock_file [String, nil] Path to the lock file to check
+  def initialize(lock_file)
+    @file_age_seconds = calculate_file_age(lock_file)
+  end
+
+  # Check if the lock file is stale (older than 2 hours)
+  #
+  # @return [Boolean] true if the lock file is stale, false otherwise
+  def stale?
+    return false if @file_age_seconds.nil?
+
+    @file_age_seconds > 7200 # 2 hours in seconds
+  end
+
+  # Get the age of the lock file in minutes
+  #
+  # @return [Float, nil] Age in minutes, or nil if file doesn't exist
+  def age_in_minutes
+    return nil if @file_age_seconds.nil?
+
+    (@file_age_seconds / 60).round(1)
+  end
+
+  private
+
+  def calculate_file_age(lock_file)
+    return nil unless lock_file && File.exist?(lock_file)
+
+    Time.now - File.mtime(lock_file)
+  end
+end

lib/rails_master_hook_app.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# Application wrapper for testing
+# This extracts the core application logic without loading config.ru
+
+require "fileutils"
+require "rack"
+require "logger"
+require_relative "lockfile_checker"
+
+class RailsMasterHookApp
+  def initialize(run_file: nil, lock_file: nil, logger:)
+    @run_file = run_file || ENV["RUN_FILE"] || File.expand_path("../run-rails-master-hook", __dir__)
+    @lock_file = lock_file || ENV["LOCK_FILE"]
+    @logger = logger
+  end
+
+  def call(env)
+    request = Rack::Request.new(env)
+
+    # Handle rails-master-hook routes (with or without trailing slash)
+    if request.path_info == "/rails-master-hook" || request.path_info == "/rails-master-hook/"
+      handle_rails_master_hook(request)
+    else
+      handle_root(request)
+    end
+  end
+
+  private
+
+  def handle_rails_master_hook(request)
+    if request.request_method == "POST"
+      @logger.info "Triggering Rails master hook by touching #{@run_file}"
+      FileUtils.touch(@run_file)
+      @logger.info "Rails master hook scheduled successfully"
+
+      scheduled = <<~EOS
+        Rails master hook tasks scheduled:
+
+        * updates the local checkout
+        * updates Rails Contributors
+        * generates and publishes edge docs
+
+        If a new stable tag is detected it also
+
+        * generates and publishes stable docs
+
+        This needs typically a few minutes.
+      EOS
+
+      [200, {"Content-Type" => "text/plain", "Content-Length" => scheduled.length.to_s}, [scheduled]]
+    else
+      @logger.warn "Rejected non-POST request (#{request.request_method}) to /rails-master-hook"
+      [404, {"Content-Type" => "text/plain", "Content-Length" => "0"}, []]
+    end
+  end
+
+  def handle_root(request)
+    lockfile_checker = LockfileChecker.new(@lock_file)
+
+    if lockfile_checker.stale?
+      age_minutes = lockfile_checker.age_in_minutes
+      error_msg = "System down: Lock file has been present for more than 2 hours"
+      @logger.error "#{error_msg} (actual age: #{age_minutes} minutes)"
+      [503, {"Content-Type" => "text/plain", "Content-Length" => error_msg.length.to_s}, [error_msg]]
+    else
+      [200, {"Content-Type" => "text/plain", "Content-Length" => "4"}, ["PONG"]]
+    end
+  end
+end

test/integration_test.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "test_helper"
+
+class IntegrationTest < TestCase
+  def test_config_ru_app_behaves_correctly
+    app, _options = Rack::Builder.parse_file(File.expand_path("../config.ru", __dir__))
+
+    env = Rack::MockRequest.env_for("/")
+    status, headers, body = app.call(env)
+
+    assert_equal 200, status
+    assert_equal "PONG", body.first
+    assert_equal "text/plain", headers["Content-Type"]
+  end
+
+  def test_config_ru_rails_master_hook_endpoint
+    capture_io do
+      app, _options = Rack::Builder.parse_file(File.expand_path("../config.ru", __dir__))
+
+      env = Rack::MockRequest.env_for("/rails-master-hook", method: "POST")
+        status, headers, body = app.call(env)
+
+      assert_equal 200, status
+      assert_match(/Rails master hook tasks scheduled/, body.first)
+      assert_equal "text/plain", headers["Content-Type"]
+    end
+  end
+end