Speed up some rake tasks by moving execution to Sidekiq (#7678)

* Speed up some rake tasks by moving execution to Sidekiq

mastodon:media:remove_silenced
mastodon:media:remove_remote
mastodon:media:redownload_avatars
mastodon:feeds:build

* Fix code style issue
This commit is contained in:
Eugen Rochko 2018-05-30 21:09:30 +02:00 committed by GitHub
parent a29f196f95
commit ad40bf5e0c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 57 additions and 16 deletions

View file

@ -0,0 +1,14 @@
# frozen_string_literal: true
class Maintenance::DestroyMediaWorker
include Sidekiq::Worker
sidekiq_options queue: 'pull'
def perform(media_attachment_id)
media = MediaAttachment.find(media_attachment_id)
media.destroy
rescue ActiveRecord::RecordNotFound
true
end
end

View file

@ -0,0 +1,16 @@
# frozen_string_literal: true
class Maintenance::RedownloadAccountMediaWorker
include Sidekiq::Worker
sidekiq_options queue: 'pull', retry: false
def perform(account_id)
account = Account.find(account_id)
account.reset_avatar!
account.reset_header!
account.save
rescue ActiveRecord::RecordNotFound
true
end
end

View file

@ -0,0 +1,18 @@
# frozen_string_literal: true
class Maintenance::UncacheMediaWorker
include Sidekiq::Worker
sidekiq_options queue: 'pull'
def perform(media_attachment_id)
media = MediaAttachment.find(media_attachment_id)
return unless media.file.exists?
media.file.destroy
media.save
rescue ActiveRecord::RecordNotFound
true
end
end

View file

@ -502,18 +502,17 @@ namespace :mastodon do
desc 'Remove media attachments attributed to silenced accounts' desc 'Remove media attachments attributed to silenced accounts'
task remove_silenced: :environment do task remove_silenced: :environment do
MediaAttachment.where(account: Account.silenced).find_each(&:destroy) MediaAttachment.where(account: Account.silenced).select(:id).find_in_batches do |media_attachments|
Maintenance::DestroyMediaWorker.push_bulk(media_attachments.map(&:id))
end
end end
desc 'Remove cached remote media attachments that are older than NUM_DAYS. By default 7 (week)' desc 'Remove cached remote media attachments that are older than NUM_DAYS. By default 7 (week)'
task remove_remote: :environment do task remove_remote: :environment do
time_ago = ENV.fetch('NUM_DAYS') { 7 }.to_i.days.ago time_ago = ENV.fetch('NUM_DAYS') { 7 }.to_i.days.ago
MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).find_each do |media| MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).find_in_batches do |media_attachments|
next unless media.file.exists? Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id))
media.file.destroy
media.save
end end
end end
@ -529,14 +528,8 @@ namespace :mastodon do
accounts = Account.remote accounts = Account.remote
accounts = accounts.where(domain: ENV['DOMAIN']) if ENV['DOMAIN'].present? accounts = accounts.where(domain: ENV['DOMAIN']) if ENV['DOMAIN'].present?
accounts.find_each do |account| accounts.select(:id).find_in_batches do |accounts_batch|
begin Maintenance::RedownloadAccountMediaWorker.push_bulk(accounts_batch.map(&:id))
account.reset_avatar!
account.reset_header!
account.save
rescue Paperclip::Error
puts "Error resetting avatar and header for account #{username}@#{domain}"
end
end end
end end
end end
@ -568,8 +561,8 @@ namespace :mastodon do
desc 'Generates home timelines for users who logged in in the past two weeks' desc 'Generates home timelines for users who logged in in the past two weeks'
task build: :environment do task build: :environment do
User.active.includes(:account).find_each do |u| User.active.select(:account_id).find_in_batches do |users|
PrecomputeFeedService.new.call(u.account) RegenerationWorker.push_bulk(users.map(&:account_id))
end end
end end
end end