diff --git a/app/jobs/shipit/background_job.rb b/app/jobs/shipit/background_job.rb index ee7d69e91..eb86c8850 100644 --- a/app/jobs/shipit/background_job.rb +++ b/app/jobs/shipit/background_job.rb @@ -5,9 +5,15 @@ class << self attr_accessor :timeout end + DEFAULT_RETRY_TIME_IN_SECONDS = 30 + # Write actions can sometimes fail intermittently, particulary for large and/or busy repositories retry_on(Octokit::BadGateway, Octokit::InternalServerError) + rescue_from(Octokit::TooManyRequests, Octokit::AbuseDetected) do |exception| + retry_job wait: exception.response_headers.fetch("Retry-After", DEFAULT_RETRY_TIME_IN_SECONDS) + end + def perform(*) with_timeout do super diff --git a/test/jobs/shipit/background_job_test.rb b/test/jobs/shipit/background_job_test.rb new file mode 100644 index 000000000..3c8be33b6 --- /dev/null +++ b/test/jobs/shipit/background_job_test.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true +require 'test_helper' + +module Shipit + class BackgroundJobTest < ActiveSupport::TestCase + setup do + @stack = shipit_stacks(:shipit) + @last_commit = @stack.commits.last + @job = CacheDeploySpecJob.new + @user = shipit_users(:walrus) + end + + test "#perform retries on Octokit secondary rate limit exceptions" do + freeze_time do + Octokit::Forbidden.any_instance.expects(:response_headers) + .returns({ "Retry-After" => 45 }) + + Shipit.github.api.expects(:user).with(@user.github_id).raises(Octokit::TooManyRequests) + + assert_enqueued_with(job: BackgroundStubJob, at: Time.now + 45.seconds) do + BackgroundStubJob.perform_now(@user) + end + end + end + + class BackgroundStubJob < BackgroundJob + queue_as :default + + def perform(user) + Shipit.github.api.user(user.github_id) + end + end + end +end