chinwag/chinwagsocial
chinwag/chinwagsocial
2
0
Fork 0
Code Issues Pull requests Releases 14 Wiki Activity

Merge tag 'v4.1.8' into chinwag-4.1

Browse source
This commit is contained in:
Mike Barnes 2023-09-20 13:37:47 +10:00
commit d9dfd09ac8
995 changed files with 53450 additions and 19456 deletions
Download patch file Download diff file Expand all files Collapse all files

189
lib/mastodon/accounts_cli.rb
View file

@ -200,21 +200,44 @@ module Mastodon
end
end
desc 'delete USERNAME', 'Delete a user'
option :email
option :dry_run, type: :boolean
desc 'delete [USERNAME]', 'Delete a user'
long_desc <<-LONG_DESC
Remove a user account with a given USERNAME.
LONG_DESC
def delete(username)
account = Account.find_local(username)
if account.nil?
say('No user with such username', :red)
With the --email option, the user is selected based on email
rather than username.
LONG_DESC
def delete(username = nil)
if username.present? && options[:email].present?
say('Use username or --email, not both', :red)
exit(1)
elsif username.blank? && options[:email].blank?
say('No username provided', :red)
exit(1)
end
say("Deleting user with #{account.statuses_count} statuses, this might take a while...")
DeleteAccountService.new.call(account, reserve_email: false)
say('OK', :green)
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
account = nil
if username.present?
account = Account.find_local(username)
if account.nil?
say('No user with such username', :red)
exit(1)
end
else
account = Account.left_joins(:user).find_by(user: { email: options[:email] })
if account.nil?
say('No user with such email', :red)
exit(1)
end
end
say("Deleting user with #{account.statuses_count} statuses, this might take a while...#{dry_run}")
DeleteAccountService.new.call(account, reserve_email: false) unless options[:dry_run]
say("OK#{dry_run}", :green)
end
option :force, type: :boolean, aliases: [:f], description: 'Override public key check'
@ -349,16 +372,16 @@ module Mastodon
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :verbose, type: :boolean, aliases: [:v]
option :dry_run, type: :boolean
desc 'refresh [USERNAME]', 'Fetch remote user data and files'
desc 'refresh [USERNAMES]', 'Fetch remote user data and files'
long_desc <<-LONG_DESC
Fetch remote user data and files for one or multiple accounts.
With the --all option, all remote accounts will be processed.
Through the --domain option, this can be narrowed down to a
specific domain only. Otherwise, a single remote account must
be specified with USERNAME.
specific domain only. Otherwise, remote accounts must be
specified with space-separated USERNAMES.
LONG_DESC
def refresh(username = nil)
def refresh(*usernames)
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
if options[:domain] || options[:all]
@ -374,19 +397,25 @@ module Mastodon
end
say("Refreshed #{processed} accounts#{dry_run}", :green, true)
elsif username.present?
username, domain = username.split('@')
account = Account.find_remote(username, domain)
elsif !usernames.empty?
usernames.each do |user|
user, domain = user.split('@')
account = Account.find_remote(user, domain)
if account.nil?
say('No such account', :red)
exit(1)
end
if account.nil?
say('No such account', :red)
exit(1)
end
unless options[:dry_run]
account.reset_avatar!
account.reset_header!
account.save
next if options[:dry_run]
begin
account.reset_avatar!
account.reset_header!
account.save
rescue Mastodon::UnexpectedResponseError
say("Account failed: #{user}@#{domain}", :red)
end
end
say("OK#{dry_run}", :green)
@ -530,6 +559,116 @@ module Mastodon
end
end
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :dry_run, type: :boolean
desc 'prune', 'Prune remote accounts that never interacted with local users'
long_desc <<-LONG_DESC
Prune remote account that
- follows no local accounts
- is not followed by any local accounts
- has no statuses on local
- has not been mentioned
- has not been favourited local posts
- not muted/blocked by us
LONG_DESC
def prune
dry_run = options[:dry_run] ? ' (dry run)' : ''
query = Account.remote.where.not(actor_type: %i(Application Service))
query = query.where('NOT EXISTS (SELECT 1 FROM mentions WHERE account_id = accounts.id)')
query = query.where('NOT EXISTS (SELECT 1 FROM favourites WHERE account_id = accounts.id)')
query = query.where('NOT EXISTS (SELECT 1 FROM statuses WHERE account_id = accounts.id)')
query = query.where('NOT EXISTS (SELECT 1 FROM follows WHERE account_id = accounts.id OR target_account_id = accounts.id)')
query = query.where('NOT EXISTS (SELECT 1 FROM blocks WHERE account_id = accounts.id OR target_account_id = accounts.id)')
query = query.where('NOT EXISTS (SELECT 1 FROM mutes WHERE target_account_id = accounts.id)')
query = query.where('NOT EXISTS (SELECT 1 FROM reports WHERE target_account_id = accounts.id)')
query = query.where('NOT EXISTS (SELECT 1 FROM follow_requests WHERE account_id = accounts.id OR target_account_id = accounts.id)')
_, deleted = parallelize_with_progress(query) do |account|
next if account.bot? || account.group?
next if account.suspended?
next if account.silenced?
account.destroy unless options[:dry_run]
1
end
say("OK, pruned #{deleted} accounts#{dry_run}", :green)
end
option :force, type: :boolean
option :replay, type: :boolean
option :target
desc 'migrate USERNAME', 'Migrate a local user to another account'
long_desc <<~LONG_DESC
With --replay, replay the last migration of the specified account, in
case some remote server may not have properly processed the associated
`Move` activity.
With --target, specify another account to migrate to.
With --force, perform the migration even if the selected account
redirects to a different account that the one specified.
LONG_DESC
def migrate(username)
if options[:replay].present? && options[:target].present?
say('Use --replay or --target, not both', :red)
exit(1)
end
if options[:replay].blank? && options[:target].blank?
say('Use either --replay or --target', :red)
exit(1)
end
account = Account.find_local(username)
if account.nil?
say("No such account: #{username}", :red)
exit(1)
end
migration = nil
if options[:replay]
migration = account.migrations.last
if migration.nil?
say('The specified account has not performed any migration', :red)
exit(1)
end
unless options[:force] || migration.target_account_id == account.moved_to_account_id
say('The specified account is not redirecting to its last migration target. Use --force if you want to replay the migration anyway', :red)
exit(1)
end
end
if options[:target]
target_account = ResolveAccountService.new.call(options[:target])
if target_account.nil?
say("The specified target account could not be found: #{options[:target]}", :red)
exit(1)
end
unless options[:force] || account.moved_to_account_id.nil? || account.moved_to_account_id == target_account.id
say('The specified account is redirecting to a different target account. Use --force if you want to change the migration target', :red)
exit(1)
end
begin
migration = account.migrations.create!(acct: target_account.acct)
rescue ActiveRecord::RecordInvalid => e
say("Error: #{e.message}", :red)
exit(1)
end
end
MoveService.new.call(migration)
say("OK, migrated #{account.acct} to #{migration.target_account.acct}", :green)
end
private
def rotate_keys_for_account(account, delay = 0)
@ -541,7 +680,7 @@ module Mastodon
old_key = account.private_key
new_key = OpenSSL::PKey::RSA.new(2048)
account.update(private_key: new_key.to_pem, public_key: new_key.public_key.to_pem)
ActivityPub::UpdateDistributionWorker.perform_in(delay, account.id, sign_with: old_key)
ActivityPub::UpdateDistributionWorker.perform_in(delay, account.id, { 'sign_with' => old_key })
end
end
end

77
lib/mastodon/domains_cli.rb
View file

@ -18,6 +18,8 @@ module Mastodon
option :dry_run, type: :boolean
option :limited_federation_mode, type: :boolean
option :by_uri, type: :boolean
option :include_subdomains, type: :boolean
option :purge_domain_blocks, type: :boolean
desc 'purge [DOMAIN...]', 'Remove accounts from a DOMAIN without a trace'
long_desc <<-LONG_DESC
Remove all accounts from a given DOMAIN without leaving behind any
@ -33,40 +35,75 @@ module Mastodon
that has the handle `foo@bar.com` but whose profile is at the URL
`https://mastodon-bar.com/users/foo`, would be purged by either
`tootctl domains purge bar.com` or `tootctl domains purge --by-uri mastodon-bar.com`.
When the --include-subdomains option is given, not only DOMAIN is deleted, but all
subdomains as well. Note that this may be considerably slower.
When the --purge-domain-blocks option is given, also purge matching domain blocks.
LONG_DESC
def purge(*domains)
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
domains = domains.map { |domain| TagManager.instance.normalize_domain(domain) }
account_scope = Account.none
domain_block_scope = DomainBlock.none
emoji_scope = CustomEmoji.none
scope = begin
if options[:limited_federation_mode]
Account.remote.where.not(domain: DomainAllow.pluck(:domain))
elsif !domains.empty?
if options[:by_uri]
domains.map { |domain| Account.remote.where(Account.arel_table[:uri].matches("https://#{domain}/%", false, true)) }.reduce(:or)
else
Account.remote.where(domain: domains)
end
# Sanity check on command arguments
if options[:limited_federation_mode] && !domains.empty?
say('DOMAIN parameter not supported with --limited-federation-mode', :red)
exit(1)
elsif domains.empty? && !options[:limited_federation_mode]
say('No domain(s) given', :red)
exit(1)
end
# Build scopes from command arguments
if options[:limited_federation_mode]
account_scope = Account.remote.where.not(domain: DomainAllow.select(:domain))
emoji_scope = CustomEmoji.remote.where.not(domain: DomainAllow.select(:domain))
else
# Handle wildcard subdomains
subdomain_patterns = domains.filter_map { |domain| "%.#{Account.sanitize_sql_like(domain[2..])}" if domain.start_with?('*.') }
domains = domains.filter { |domain| !domain.start_with?('*.') }
# Handle --include-subdomains
subdomain_patterns += domains.map { |domain| "%.#{Account.sanitize_sql_like(domain)}" } if options[:include_subdomains]
uri_patterns = (domains.map { |domain| Account.sanitize_sql_like(domain) } + subdomain_patterns).map { |pattern| "https://#{pattern}/%" }
if options[:purge_domain_blocks]
domain_block_scope = DomainBlock.where(domain: domains)
domain_block_scope = domain_block_scope.or(DomainBlock.where(DomainBlock.arel_table[:domain].matches_any(subdomain_patterns))) unless subdomain_patterns.empty?
end
if options[:by_uri]
account_scope = Account.remote.where(Account.arel_table[:uri].matches_any(uri_patterns, false, true))
emoji_scope = CustomEmoji.remote.where(CustomEmoji.arel_table[:uri].matches_any(uri_patterns, false, true))
else
say('No domain(s) given', :red)
exit(1)
account_scope = Account.remote.where(domain: domains)
account_scope = account_scope.or(Account.remote.where(Account.arel_table[:domain].matches_any(subdomain_patterns))) unless subdomain_patterns.empty?
emoji_scope = CustomEmoji.where(domain: domains)
emoji_scope = emoji_scope.or(CustomEmoji.remote.where(CustomEmoji.arel_table[:uri].matches_any(subdomain_patterns))) unless subdomain_patterns.empty?
end
end
processed, = parallelize_with_progress(scope) do |account|
# Actually perform the deletions
processed, = parallelize_with_progress(account_scope) do |account|
DeleteAccountService.new.call(account, reserve_username: false, skip_side_effects: true) unless options[:dry_run]
end
DomainBlock.where(domain: domains).destroy_all unless options[:dry_run]
say("Removed #{processed} accounts#{dry_run}", :green)
custom_emojis = CustomEmoji.where(domain: domains)
custom_emojis_count = custom_emojis.count
custom_emojis.destroy_all unless options[:dry_run]
if options[:purge_domain_blocks]
domain_block_count = domain_block_scope.count
domain_block_scope.in_batches.destroy_all unless options[:dry_run]
say("Removed #{domain_block_count} domain blocks#{dry_run}", :green)
end
custom_emojis_count = emoji_scope.count
emoji_scope.in_batches.destroy_all unless options[:dry_run]
Instance.refresh unless options[:dry_run]
say("Removed #{custom_emojis_count} custom emojis", :green)
say("Removed #{custom_emojis_count} custom emojis#{dry_run}", :green)
end
option :concurrency, type: :numeric, default: 50, aliases: [:c]
@ -97,7 +134,7 @@ module Mastodon
failed = Concurrent::AtomicFixnum.new(0)
start_at = Time.now.to_f
seed = start ? [start] : Instance.pluck(:domain)
blocked_domains = Regexp.new('\\.?' + DomainBlock.where(severity: 1).pluck(:domain).join('|') + '$')
blocked_domains = /\.?(#{DomainBlock.where(severity: 1).pluck(:domain).map { |domain| Regexp.escape(domain) }.join('|')})$/
progress = create_progress_bar
pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0)

78
lib/mastodon/media_cli.rb
View file

@ -14,35 +14,78 @@ module Mastodon
end
option :days, type: :numeric, default: 7, aliases: [:d]
option :prune_profiles, type: :boolean, default: false
option :remove_headers, type: :boolean, default: false
option :include_follows, type: :boolean, default: false
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :verbose, type: :boolean, default: false, aliases: [:v]
option :dry_run, type: :boolean, default: false
desc 'remove', 'Remove remote media files'
desc 'remove', 'Remove remote media files, headers or avatars'
long_desc <<-DESC
Removes locally cached copies of media attachments from other servers.
Removes locally cached copies of media attachments (and optionally profile
headers and avatars) from other servers. By default, only media attachements
are removed.
The --days option specifies how old media attachments have to be before
they are removed. It defaults to 7 days.
they are removed. In case of avatars and headers, it specifies how old
the last webfinger request and update to the user has to be before they
are pruned. It defaults to 7 days.
If --prune-profiles is specified, only avatars and headers are removed.
If --remove-headers is specified, only headers are removed.
If --include-follows is specified along with --prune-profiles or
--remove-headers, all non-local profiles will be pruned irrespective of
follow status. By default, only accounts that are not followed by or
following anyone locally are pruned.
DESC
# rubocop:disable Metrics/PerceivedComplexity
def remove
time_ago = options[:days].days.ago
dry_run = options[:dry_run] ? '(DRY RUN)' : ''
if options[:prune_profiles] && options[:remove_headers]
say('--prune-profiles and --remove-headers should not be specified simultaneously', :red, true)
exit(1)
end
if options[:include_follows] && !(options[:prune_profiles] || options[:remove_headers])
say('--include-follows can only be used with --prune-profiles or --remove-headers', :red, true)
exit(1)
end
time_ago = options[:days].days.ago
dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where('created_at < ?', time_ago)) do |media_attachment|
next if media_attachment.file.blank?
if options[:prune_profiles] || options[:remove_headers]
processed, aggregate = parallelize_with_progress(Account.remote.where({ last_webfingered_at: ..time_ago, updated_at: ..time_ago })) do |account|
next if !options[:include_follows] && Follow.where(account: account).or(Follow.where(target_account: account)).exists?
next if account.avatar.blank? && account.header.blank?
next if options[:remove_headers] && account.header.blank?
size = (media_attachment.file_file_size || 0) + (media_attachment.thumbnail_file_size || 0)
size = (account.header_file_size || 0)
size += (account.avatar_file_size || 0) if options[:prune_profiles]
unless options[:dry_run]
media_attachment.file.destroy
media_attachment.thumbnail.destroy
media_attachment.save
unless options[:dry_run]
account.header.destroy
account.avatar.destroy if options[:prune_profiles]
account.save!
end
size
end
size
say("Visited #{processed} accounts and removed profile media totaling #{number_to_human_size(aggregate)}#{dry_run}", :green, true)
end
say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)}) #{dry_run}", :green, true)
unless options[:prune_profiles] || options[:remove_headers]
processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where(created_at: ..time_ago)) do |media_attachment|
next if media_attachment.file.blank?
size = (media_attachment.file_file_size || 0) + (media_attachment.thumbnail_file_size || 0)
unless options[:dry_run]
media_attachment.file.destroy
media_attachment.thumbnail.destroy
media_attachment.save
end
size
end
say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true)
end
end
option :start_after
@ -183,6 +226,7 @@ module Mastodon
say("Removed #{removed} orphans (approx. #{number_to_human_size(reclaimed_bytes)})#{dry_run}", :green, true)
end
# rubocop:enable Metrics/PerceivedComplexity
option :account, type: :string
option :domain, type: :string
@ -269,7 +313,7 @@ module Mastodon
def lookup(url)
path = Addressable::URI.parse(url).path
path_segments = path.split('/')[2..-1]
path_segments = path.split('/')[2..]
path_segments.delete('cache')
unless [7, 10].include?(path_segments.size)

2
lib/mastodon/premailer_webpack_strategy.rb
View file

@ -13,7 +13,7 @@ module PremailerWebpackStrategy
HTTP.get(url).to_s
else
url = url[1..-1] if url.start_with?('/')
File.read(Rails.root.join('public', url))
File.read(Rails.public_path.join(url))
end
css.gsub(/url\(\//, "url(#{asset_host}/")

1
lib/mastodon/redis_config.rb
View file

@ -37,6 +37,7 @@ REDIS_CACHE_PARAMS = {
namespace: cache_namespace,
pool_size: Sidekiq.server? ? Sidekiq.options[:concurrency] : Integer(ENV['MAX_THREADS'] || 5),
pool_timeout: 5,
connect_timeout: 5,
}.freeze
REDIS_SIDEKIQ_PARAMS = {

4
lib/mastodon/statuses_cli.rb
View file

@ -93,7 +93,7 @@ module Mastodon
c.table_name = 'statuses_to_be_deleted'
end
Object.const_set('StatusToBeDeleted', klass)
Object.const_set(:StatusToBeDeleted, klass)
scope = StatusToBeDeleted
processed = 0
@ -175,7 +175,7 @@ module Mastodon
c.table_name = 'conversations_to_be_deleted'
end
Object.const_set('ConversationsToBeDeleted', klass)
Object.const_set(:ConversationsToBeDeleted, klass)
scope = ConversationsToBeDeleted
processed = 0

6
lib/mastodon/version.rb
View file

@ -9,11 +9,11 @@ module Mastodon
end
def minor
0
1
end
def patch
10
8
end
def flags
@ -21,7 +21,7 @@ module Mastodon
end
def suffix
'-cw1'
'-cw0'
end
def to_a