Merge pull request #1344 from Shopify/kwbeqr/retry

Add GitHub Secondary Rate Limit Wait and Retries

app/jobs/shipit/background_job.rb

@@ -5,9 +5,15 @@ class << self
       attr_accessor :timeout
     end
 
+    DEFAULT_RETRY_TIME_IN_SECONDS = 30
+
     # Write actions can sometimes fail intermittently, particulary for large and/or busy repositories
     retry_on(Octokit::BadGateway, Octokit::InternalServerError)
 
+    rescue_from(Octokit::TooManyRequests, Octokit::AbuseDetected) do |exception|
+      retry_job wait: exception.response_headers.fetch("Retry-After", DEFAULT_RETRY_TIME_IN_SECONDS)
+    end
+
     def perform(*)
       with_timeout do
         super

test/jobs/shipit/background_job_test.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+require 'test_helper'
+
+module Shipit
+  class BackgroundJobTest < ActiveSupport::TestCase
+    setup do
+      @stack = shipit_stacks(:shipit)
+      @last_commit = @stack.commits.last
+      @job = CacheDeploySpecJob.new
+      @user = shipit_users(:walrus)
+    end
+
+    test "#perform retries on Octokit secondary rate limit exceptions" do
+      freeze_time do
+        Octokit::Forbidden.any_instance.expects(:response_headers)
+          .returns({ "Retry-After" => 45 })
+
+        Shipit.github.api.expects(:user).with(@user.github_id).raises(Octokit::TooManyRequests)
+
+        assert_enqueued_with(job: BackgroundStubJob, at: Time.now + 45.seconds) do
+          BackgroundStubJob.perform_now(@user)
+        end
+      end
+    end
+
+    class BackgroundStubJob < BackgroundJob
+      queue_as :default
+
+      def perform(user)
+        Shipit.github.api.user(user.github_id)
+      end
+    end
+  end
+end