227 lines
10 KiB
Ruby
227 lines
10 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require_relative '../../config/boot'
|
|
require_relative '../../config/environment'
|
|
require_relative 'cli_helper'
|
|
|
|
module Mastodon
|
|
class StatusesCLI < Thor
|
|
include CLIHelper
|
|
include ActionView::Helpers::NumberHelper
|
|
|
|
def self.exit_on_failure?
|
|
true
|
|
end
|
|
|
|
option :days, type: :numeric, default: 90
|
|
option :batch_size, type: :numeric, default: 1_000, aliases: [:b], desc: 'Number of records in each batch'
|
|
option :continue, type: :boolean, default: false, desc: 'If remove is not completed, execute from the previous continuation'
|
|
option :clean_followed, type: :boolean, default: false, desc: 'Include the status of remote accounts that are followed by local accounts as candidates for remove'
|
|
option :skip_status_remove, type: :boolean, default: false, desc: 'Skip status remove (run only cleanup tasks)'
|
|
option :skip_media_remove, type: :boolean, default: false, desc: 'Skip remove orphaned media attachments'
|
|
option :compress_database, type: :boolean, default: false, desc: 'Compress database and update the statistics. This option locks the table for a long time, so run it offline'
|
|
desc 'remove', 'Remove unreferenced statuses'
|
|
long_desc <<~LONG_DESC
|
|
Remove statuses that are not referenced by local user activity, such as
|
|
ones that came from relays, or belonging to users that were once followed
|
|
by someone locally but no longer are.
|
|
|
|
It also removes orphaned records and performs additional cleanup tasks
|
|
such as updating statistics and recovering disk space.
|
|
|
|
This is a computationally heavy procedure that creates extra database
|
|
indices before commencing, and removes them afterward.
|
|
LONG_DESC
|
|
def remove
|
|
if options[:batch_size] < 1
|
|
say('Cannot run with this batch_size setting, must be at least 1', :red)
|
|
exit(1)
|
|
end
|
|
|
|
remove_statuses
|
|
vacuum_and_analyze_statuses
|
|
remove_orphans_media_attachments
|
|
remove_orphans_conversations
|
|
vacuum_and_analyze_conversations
|
|
end
|
|
|
|
private
|
|
|
|
def remove_statuses
|
|
return if options[:skip_status_remove]
|
|
|
|
say('Creating temporary database indices...')
|
|
|
|
ActiveRecord::Base.connection.add_index(:media_attachments, :remote_url, name: :index_media_attachments_remote_url, where: 'remote_url is not null', algorithm: :concurrently, if_not_exists: true)
|
|
|
|
max_id = Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false)
|
|
start_at = Time.now.to_f
|
|
|
|
unless options[:continue] && ActiveRecord::Base.connection.table_exists?('statuses_to_be_deleted')
|
|
ActiveRecord::Base.connection.add_index(:accounts, :id, name: :index_accounts_local, where: 'domain is null', algorithm: :concurrently, if_not_exists: true)
|
|
ActiveRecord::Base.connection.add_index(:status_pins, :status_id, name: :index_status_pins_status_id, algorithm: :concurrently, if_not_exists: true)
|
|
|
|
say('Extract the deletion target from statuses... This might take a while...')
|
|
|
|
ActiveRecord::Base.connection.create_table('statuses_to_be_deleted', force: true)
|
|
|
|
# Skip accounts followed by local accounts
|
|
clean_followed_sql = 'AND NOT EXISTS (SELECT 1 FROM follows WHERE statuses.account_id = follows.target_account_id)' unless options[:clean_followed]
|
|
|
|
ActiveRecord::Base.connection.exec_insert(<<-SQL.squish, 'SQL', [[nil, max_id]])
|
|
INSERT INTO statuses_to_be_deleted (id)
|
|
SELECT statuses.id FROM statuses WHERE deleted_at IS NULL AND NOT local AND uri IS NOT NULL AND (id < $1)
|
|
AND NOT EXISTS (SELECT 1 FROM statuses AS statuses1 WHERE statuses.id = statuses1.in_reply_to_id)
|
|
AND NOT EXISTS (SELECT 1 FROM statuses AS statuses1 WHERE statuses1.id = statuses.reblog_of_id AND (statuses1.uri IS NULL OR statuses1.local))
|
|
AND NOT EXISTS (SELECT 1 FROM statuses AS statuses1 WHERE statuses.id = statuses1.reblog_of_id AND (statuses1.uri IS NULL OR statuses1.local OR statuses1.id >= $1))
|
|
AND NOT EXISTS (SELECT 1 FROM status_pins WHERE statuses.id = status_id)
|
|
AND NOT EXISTS (SELECT 1 FROM mentions WHERE statuses.id = mentions.status_id AND mentions.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))
|
|
AND NOT EXISTS (SELECT 1 FROM favourites WHERE statuses.id = favourites.status_id AND favourites.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))
|
|
AND NOT EXISTS (SELECT 1 FROM bookmarks WHERE statuses.id = bookmarks.status_id AND bookmarks.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))
|
|
#{clean_followed_sql}
|
|
SQL
|
|
|
|
say('Removing temporary database indices to restore write performance...')
|
|
|
|
ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local, if_exists: true)
|
|
ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id, if_exists: true)
|
|
end
|
|
|
|
say('Beginning statuses removal... This might take a while...')
|
|
|
|
klass = Class.new(ApplicationRecord) do |c|
|
|
c.table_name = 'statuses_to_be_deleted'
|
|
end
|
|
|
|
Object.const_set('StatusToBeDeleted', klass)
|
|
|
|
scope = StatusToBeDeleted
|
|
processed = 0
|
|
removed = 0
|
|
progress = create_progress_bar(scope.count.fdiv(options[:batch_size]).ceil)
|
|
|
|
scope.reorder(nil).in_batches(of: options[:batch_size]) do |relation|
|
|
ids = relation.pluck(:id)
|
|
processed += ids.count
|
|
removed += Status.unscoped.where(id: ids).delete_all
|
|
progress.increment
|
|
end
|
|
|
|
progress.stop
|
|
|
|
ActiveRecord::Base.connection.drop_table('statuses_to_be_deleted')
|
|
|
|
say("Done after #{Time.now.to_f - start_at}s, removed #{removed} out of #{processed} statuses.", :green)
|
|
ensure
|
|
say('Removing temporary database indices to restore write performance...')
|
|
|
|
ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local, if_exists: true)
|
|
ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id, if_exists: true)
|
|
ActiveRecord::Base.connection.remove_index(:media_attachments, name: :index_media_attachments_remote_url, if_exists: true)
|
|
end
|
|
|
|
def remove_orphans_media_attachments
|
|
return if options[:skip_media_remove]
|
|
|
|
start_at = Time.now.to_f
|
|
|
|
say('Beginning removal of now-orphaned media attachments to free up disk space...')
|
|
|
|
scope = MediaAttachment.reorder(nil).unattached.where('created_at < ?', options[:days].pred.days.ago)
|
|
processed = 0
|
|
removed = 0
|
|
progress = create_progress_bar(scope.count)
|
|
|
|
scope.find_each do |media_attachment|
|
|
media_attachment.destroy!
|
|
|
|
removed += 1
|
|
rescue => e
|
|
progress.log pastel.red("Error processing #{media_attachment.id}: #{e}")
|
|
ensure
|
|
progress.increment
|
|
processed += 1
|
|
end
|
|
|
|
progress.stop
|
|
|
|
say("Done after #{Time.now.to_f - start_at}s, removed #{removed} out of #{processed} media_attachments.", :green)
|
|
end
|
|
|
|
def remove_orphans_conversations
|
|
start_at = Time.now.to_f
|
|
|
|
unless options[:continue] && ActiveRecord::Base.connection.table_exists?('conversations_to_be_deleted')
|
|
say('Creating temporary database indices...')
|
|
|
|
ActiveRecord::Base.connection.add_index(:statuses, :conversation_id, name: :index_statuses_conversation_id, algorithm: :concurrently, if_not_exists: true)
|
|
|
|
say('Extract the deletion target from conversations... This might take a while...')
|
|
|
|
ActiveRecord::Base.connection.create_table('conversations_to_be_deleted', force: true)
|
|
|
|
ActiveRecord::Base.connection.exec_insert(<<-SQL.squish, 'SQL')
|
|
INSERT INTO conversations_to_be_deleted (id)
|
|
SELECT id FROM conversations WHERE NOT EXISTS (SELECT 1 FROM statuses WHERE statuses.conversation_id = conversations.id)
|
|
SQL
|
|
|
|
say('Removing temporary database indices to restore write performance...')
|
|
ActiveRecord::Base.connection.remove_index(:statuses, name: :index_statuses_conversation_id, if_exists: true)
|
|
end
|
|
|
|
say('Beginning orphans removal... This might take a while...')
|
|
|
|
klass = Class.new(ApplicationRecord) do |c|
|
|
c.table_name = 'conversations_to_be_deleted'
|
|
end
|
|
|
|
Object.const_set('ConversationsToBeDeleted', klass)
|
|
|
|
scope = ConversationsToBeDeleted
|
|
processed = 0
|
|
removed = 0
|
|
progress = create_progress_bar(scope.count.fdiv(options[:batch_size]).ceil)
|
|
|
|
scope.in_batches(of: options[:batch_size]) do |relation|
|
|
ids = relation.pluck(:id)
|
|
processed += ids.count
|
|
removed += Conversation.unscoped.where(id: ids).delete_all
|
|
progress.increment
|
|
end
|
|
|
|
progress.stop
|
|
|
|
ActiveRecord::Base.connection.drop_table('conversations_to_be_deleted')
|
|
|
|
say("Done after #{Time.now.to_f - start_at}s, removed #{removed} out of #{processed} conversations.", :green)
|
|
ensure
|
|
say('Removing temporary database indices to restore write performance...')
|
|
ActiveRecord::Base.connection.remove_index(:statuses, name: :index_statuses_conversation_id, if_exists: true)
|
|
end
|
|
|
|
def vacuum_and_analyze_statuses
|
|
if options[:compress_database]
|
|
say('Run VACUUM FULL ANALYZE to statuses...')
|
|
ActiveRecord::Base.connection.execute('VACUUM FULL ANALYZE statuses')
|
|
say('Run REINDEX to statuses...')
|
|
ActiveRecord::Base.connection.execute('REINDEX TABLE statuses')
|
|
else
|
|
say('Run ANALYZE to statuses...')
|
|
ActiveRecord::Base.connection.execute('ANALYZE statuses')
|
|
end
|
|
end
|
|
|
|
def vacuum_and_analyze_conversations
|
|
if options[:compress_database]
|
|
say('Run VACUUM FULL ANALYZE to conversations...')
|
|
ActiveRecord::Base.connection.execute('VACUUM FULL ANALYZE conversations')
|
|
say('Run REINDEX to conversations...')
|
|
ActiveRecord::Base.connection.execute('REINDEX TABLE conversations')
|
|
else
|
|
say('Run ANALYZE to conversations...')
|
|
ActiveRecord::Base.connection.execute('ANALYZE conversations')
|
|
end
|
|
end
|
|
end
|
|
end
|