Add stoplight for object storage failures, return HTTP 503 (#13043)

This commit is contained in:
Eugen Rochko 2020-12-15 12:55:29 +01:00 committed by GitHub
parent 75d2762fdf
commit 1045549f85
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 34 additions and 2 deletions

View file

@ -40,7 +40,7 @@ class Api::BaseController < ApplicationController
render json: { error: 'This action is not allowed' }, status: 403 render json: { error: 'This action is not allowed' }, status: 403
end end
rescue_from Mastodon::RaceConditionError do rescue_from Mastodon::RaceConditionError, Seahorse::Client::NetworkingError, Stoplight::Error::RedLight do
render json: { error: 'There was a temporary problem serving your request, please try again' }, status: 503 render json: { error: 'There was a temporary problem serving your request, please try again' }, status: 503
end end

View file

@ -28,7 +28,7 @@ class ApplicationController < ActionController::Base
rescue_from ActiveRecord::RecordNotFound, with: :not_found rescue_from ActiveRecord::RecordNotFound, with: :not_found
rescue_from Mastodon::NotPermittedError, with: :forbidden rescue_from Mastodon::NotPermittedError, with: :forbidden
rescue_from HTTP::Error, OpenSSL::SSL::SSLError, with: :internal_server_error rescue_from HTTP::Error, OpenSSL::SSL::SSLError, with: :internal_server_error
rescue_from Mastodon::RaceConditionError, with: :service_unavailable rescue_from Mastodon::RaceConditionError, Seahorse::Client::NetworkingError, Stoplight::Error::RedLight, with: :service_unavailable
rescue_from Mastodon::RateLimitExceededError, with: :too_many_requests rescue_from Mastodon::RateLimitExceededError, with: :too_many_requests
before_action :store_current_location, except: :raise_not_found, unless: :devise_controller? before_action :store_current_location, except: :raise_not_found, unless: :devise_controller?

View file

@ -228,6 +228,8 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
emoji ||= CustomEmoji.new(domain: @account.domain, shortcode: shortcode, uri: uri) emoji ||= CustomEmoji.new(domain: @account.domain, shortcode: shortcode, uri: uri)
emoji.image_remote_url = image_url emoji.image_remote_url = image_url
emoji.save emoji.save
rescue Seahorse::Client::NetworkingError
nil
end end
def process_attachments def process_attachments
@ -250,6 +252,8 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
media_attachment.save media_attachment.save
rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError
RedownloadMediaWorker.perform_in(rand(30..600).seconds, media_attachment.id) RedownloadMediaWorker.perform_in(rand(30..600).seconds, media_attachment.id)
rescue Seahorse::Client::NetworkingError
nil
end end
end end

View file

@ -113,3 +113,14 @@ else
end end
Paperclip.options[:content_type_mappings] = { csv: Import::FILE_TYPES } Paperclip.options[:content_type_mappings] = { csv: Import::FILE_TYPES }
# In some places in the code, we rescue this exception, but we don't always
# load the S3 library, so it may be an undefined constant:
unless defined?(Seahorse)
module Seahorse
module Client
class NetworkingError < StandardError; end
end
end
end

View file

@ -39,6 +39,23 @@ module Paperclip
def default_url(style_name = default_style) def default_url(style_name = default_style)
@url_generator.for_as_default(style_name) @url_generator.for_as_default(style_name)
end end
STOPLIGHT_THRESHOLD = 10
STOPLIGHT_COOLDOWN = 30
# We overwrite this method to put a circuit breaker around
# calls to object storage, to stop hitting APIs that are slow
# to respond or don't respond at all and as such minimize the
# impact of object storage outages on application throughput
def save
Stoplight('object-storage') { super }.with_threshold(STOPLIGHT_THRESHOLD).with_cool_off_time(STOPLIGHT_COOLDOWN).with_error_handler do |error, handle|
if error.is_a?(Seahorse::Client::NetworkingError)
handle.call(error)
else
raise error
end
end.run
end
end end
end end