Fix performance of follow import (#13836)

This commit is contained in:
Takeshi Umeda 2020-06-09 17:26:58 +09:00 committed by GitHub
parent 72a7cfaa39
commit d890abfcab
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 44 additions and 2 deletions

View file

@ -1,5 +1,16 @@
# frozen_string_literal: true # frozen_string_literal: true
# Monkey-patch on monkey-patch.
# Because it conflicts with the request.rb patch.
class HTTP::Timeout::PerOperationOriginal < HTTP::Timeout::PerOperation
def connect(socket_class, host, port, nodelay = false)
::Timeout.timeout(@connect_timeout, HTTP::TimeoutError) do
@socket = socket_class.open(host, port)
@socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
end
end
end
module WebfingerHelper module WebfingerHelper
def webfinger!(uri) def webfinger!(uri)
hidden_service_uri = /\.(onion|i2p)(:\d+)?$/.match(uri) hidden_service_uri = /\.(onion|i2p)(:\d+)?$/.match(uri)
@ -12,6 +23,14 @@ module WebfingerHelper
headers: { headers: {
'User-Agent': Mastodon::Version.user_agent, 'User-Agent': Mastodon::Version.user_agent,
}, },
timeout_class: HTTP::Timeout::PerOperationOriginal,
timeout_options: {
write_timeout: 10,
connect_timeout: 5,
read_timeout: 10,
},
} }
Goldfinger::Client.new(uri, opts.merge(Rails.configuration.x.http_client_proxy)).finger Goldfinger::Client.new(uri, opts.merge(Rails.configuration.x.http_client_proxy)).finger

View file

@ -81,7 +81,9 @@ class ImportService < BaseService
end end
end end
Import::RelationshipWorker.push_bulk(items) do |acct, extra| head_items = items.uniq { |acct, _| acct.split('@')[1] }
tail_items = items - head_items
Import::RelationshipWorker.push_bulk(head_items + tail_items) do |acct, extra|
[@account.id, acct, action, extra] [@account.id, acct, action, extra]
end end
end end

View file

@ -112,6 +112,8 @@ class ResolveAccountService < BaseService
end end
def webfinger_update_due? def webfinger_update_due?
return false if @options[:check_delivery_availability] && !DeliveryFailureTracker.available?(@domain)
@account.nil? || ((!@options[:skip_webfinger] || @account.ostatus?) && @account.possibly_stale?) @account.nil? || ((!@options[:skip_webfinger] || @account.ostatus?) && @account.possibly_stale?)
end end

View file

@ -7,7 +7,8 @@ class Import::RelationshipWorker
def perform(account_id, target_account_uri, relationship, options = {}) def perform(account_id, target_account_uri, relationship, options = {})
from_account = Account.find(account_id) from_account = Account.find(account_id)
target_account = ResolveAccountService.new.call(target_account_uri) target_domain = domain(target_account_uri)
target_account = stoplight_wrap_request(target_domain) { ResolveAccountService.new.call(target_account_uri, { check_delivery_availability: true }) }
options.symbolize_keys! options.symbolize_keys!
return if target_account.nil? return if target_account.nil?
@ -29,4 +30,22 @@ class Import::RelationshipWorker
rescue ActiveRecord::RecordNotFound rescue ActiveRecord::RecordNotFound
true true
end end
def domain(uri)
domain = uri.is_a?(Account) ? uri.domain : uri.split('@')[1]
TagManager.instance.local_domain?(domain) ? nil : TagManager.instance.normalize_domain(domain)
end
def stoplight_wrap_request(domain, &block)
if domain.present?
Stoplight("source:#{domain}", &block)
.with_fallback { nil }
.with_threshold(1)
.with_cool_off_time(5.minutes.seconds)
.with_error_handler { |error, handle| error.is_a?(HTTP::Error) || error.is_a?(OpenSSL::SSL::SSLError) ? handle.call(error) : raise(error) }
.run
else
block.call
end
end
end end