From 406adfca275909111153dfde91626a849fed5a1f Mon Sep 17 00:00:00 2001 From: ThibG Date: Fri, 18 Dec 2020 23:31:14 +0100 Subject: [PATCH] Backport fixes to 3.2 (#15360) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix 2FA/sign-in token sessions being valid after password change (#14802) If someone tries logging in to an account and is prompted for a 2FA code or sign-in token, even if the account's password or e-mail is updated in the meantime, the session will show the prompt and allow the login process to complete with a valid 2FA code or sign-in token * Fix Move handler not being triggered when failing to fetch target (#15107) When failing to fetch the target account, the ProcessingWorker fails as expected, but since it hasn't cleared the `move_in_progress` flag, the next attempt at processing skips the `Move` activity altogether. This commit changes it to clear the flag when encountering any unexpected error on fetching the target account. This is likely to occur because, of, e.g., a timeout, when many instances query the same actor at the same time. * Fix slow distinct queries where grouped queries are faster (#15287) About 2x speed-up on inboxes query * Fix possible inconsistencies in tag search (#14906) Do not downcase the queried tag before passing it to postgres when searching: - tags are not downcased on creation - `arel_table[:name].lower.matches(pattern)` generates an ILIKE anyway - if Postgres and Rails happen to use different case-folding rules, downcasing before query but not before insertion may mean that some tags with some casings are not searchable * Fix updating account counters when account_stat is not yet created (#15108) * Fix account processing failing because of large collections (#15027) Fixes #15025 * Fix downloading remote media files when server returns empty filename (#14867) Fixes #14817 * Fix webfinger redirect handling in ResolveAccountService (#15187) * Fix webfinger redirect handling in ResolveAccountService ResolveAccountService#process_webfinger! handled a one-step webfinger redirection, but only accepting the result if it matched the exact URI passed as input, defeating the point of a redirection check. Instead, use the same logic as in `ActivityPub::FetchRemoteAccountService`, updating the resulting `acct:` URI with the result of the first webfinger query. * Add tests * Remove dependency on unused and unmaintained http_parser.rb gem (#14574) It seems that years ago, the “http” gem dependend on the “http_parser.rb” gem (it now depends on the “http-parser” gem), and, still years ago, we pulled it from git in order to benefit from a bugfix that wasn't released yet (#7467). * Add tootctl maintenance fix-duplicates (#14860, #15201, #15264, #15349, #15359) * Fix old migration script not being able to run if it fails midway (#15361) * Fix old migration script not being able to run if it fails midway Improve the robustness of a migration script likely to fail because of database corruption so it can run again once database corruptions are fixed. * Display a specific error message in case of index corruption Co-authored-by: Eugen Rochko Co-authored-by: Claire Co-authored-by: Eugen Rochko Co-authored-by: Claire --- Gemfile | 1 - Gemfile.lock | 9 - app/controllers/accounts_controller.rb | 2 +- app/controllers/admin/statuses_controller.rb | 2 +- app/controllers/api/base_controller.rb | 2 +- app/controllers/auth/sessions_controller.rb | 22 +- .../sign_in_token_authentication_concern.rb | 16 +- .../two_factor_authentication_concern.rb | 16 +- .../concerns/user_tracking_concern.rb | 7 +- app/lib/activitypub/activity/move.rb | 3 + app/models/account.rb | 2 +- app/models/account_stat.rb | 25 +- app/models/form/account_batch.rb | 2 +- app/models/tag.rb | 2 +- app/models/user.rb | 25 +- .../activitypub/process_account_service.rb | 2 +- app/services/resolve_account_service.rb | 23 +- ...3_add_fixed_lowercase_index_to_accounts.rb | 26 +- lib/cli.rb | 4 + lib/mastodon/maintenance_cli.rb | 618 ++++++++++++++++++ lib/paperclip/response_with_limit_adapter.rb | 2 +- .../auth/sessions_controller_spec.rb | 12 +- spec/models/account_spec.rb | 23 + spec/services/resolve_account_service_spec.rb | 52 +- 24 files changed, 821 insertions(+), 77 deletions(-) create mode 100644 lib/mastodon/maintenance_cli.rb diff --git a/Gemfile b/Gemfile index 0891371be..5ac1c79c1 100644 --- a/Gemfile +++ b/Gemfile @@ -60,7 +60,6 @@ gem 'health_check', git: 'https://github.com/ianheggie/health_check', ref: '0b79 gem 'htmlentities', '~> 4.3' gem 'http', '~> 4.4' gem 'http_accept_language', '~> 2.1' -gem 'http_parser.rb', '~> 0.6', git: 'https://github.com/tmm1/http_parser.rb', ref: '54b17ba8c7d8d20a16dfc65d1775241833219cf2', submodules: true gem 'httplog', '~> 1.4.3' gem 'idn-ruby', require: 'idn' gem 'kaminari', '~> 1.2' diff --git a/Gemfile.lock b/Gemfile.lock index 62c20bf07..c3216870d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -13,14 +13,6 @@ GIT specs: posix-spawn (0.3.13) -GIT - remote: https://github.com/tmm1/http_parser.rb - revision: 54b17ba8c7d8d20a16dfc65d1775241833219cf2 - ref: 54b17ba8c7d8d20a16dfc65d1775241833219cf2 - submodules: true - specs: - http_parser.rb (0.6.1) - GIT remote: https://github.com/witgo/nilsimsa revision: fd184883048b922b176939f851338d0a4971a532 @@ -709,7 +701,6 @@ DEPENDENCIES htmlentities (~> 4.3) http (~> 4.4) http_accept_language (~> 2.1) - http_parser.rb (~> 0.6)! httplog (~> 1.4.3) i18n-tasks (~> 0.9) idn-ruby diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index f5e82692e..b42f711f0 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -82,7 +82,7 @@ class AccountsController < ApplicationController end def account_media_status_ids - @account.media_attachments.attached.reorder(nil).select(:status_id).distinct + @account.media_attachments.attached.reorder(nil).select(:status_id).group(:status_id) end def no_replies_scope diff --git a/app/controllers/admin/statuses_controller.rb b/app/controllers/admin/statuses_controller.rb index 650195034..d7c192f0d 100644 --- a/app/controllers/admin/statuses_controller.rb +++ b/app/controllers/admin/statuses_controller.rb @@ -14,7 +14,7 @@ module Admin @statuses = @account.statuses.where(visibility: [:public, :unlisted]) if params[:media] - account_media_status_ids = @account.media_attachments.attached.reorder(nil).select(:status_id).distinct + account_media_status_ids = @account.media_attachments.attached.reorder(nil).select(:status_id).group(:status_id) @statuses.merge!(Status.where(id: account_media_status_ids)) end diff --git a/app/controllers/api/base_controller.rb b/app/controllers/api/base_controller.rb index 045e7dd26..055cd416d 100644 --- a/app/controllers/api/base_controller.rb +++ b/app/controllers/api/base_controller.rb @@ -102,7 +102,7 @@ class Api::BaseController < ApplicationController elsif !current_user.approved? render json: { error: 'Your login is currently pending approval' }, status: 403 else - set_user_activity + update_user_sign_in end end diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb index 1fd755334..783338450 100644 --- a/app/controllers/auth/sessions_controller.rb +++ b/app/controllers/auth/sessions_controller.rb @@ -7,6 +7,7 @@ class Auth::SessionsController < Devise::SessionsController skip_before_action :require_no_authentication, only: [:create] skip_before_action :require_functional! + skip_before_action :update_user_sign_in include TwoFactorAuthenticationConcern include SignInTokenAuthenticationConcern @@ -24,6 +25,7 @@ class Auth::SessionsController < Devise::SessionsController def create super do |resource| + resource.update_sign_in!(request, new_sign_in: true) remember_me(resource) flash.delete(:notice) end @@ -41,7 +43,7 @@ class Auth::SessionsController < Devise::SessionsController def find_user if session[:attempt_user_id] - User.find(session[:attempt_user_id]) + User.find_by(id: session[:attempt_user_id]) else user = User.authenticate_with_ldap(user_params) if Devise.ldap_authentication user ||= User.authenticate_with_pam(user_params) if Devise.pam_authentication @@ -74,6 +76,7 @@ class Auth::SessionsController < Devise::SessionsController def require_no_authentication super + # Delete flash message that isn't entirely useful and may be confusing in # most cases because /web doesn't display/clear flash messages. flash.delete(:alert) if flash[:alert] == I18n.t('devise.failure.already_authenticated') @@ -91,13 +94,30 @@ class Auth::SessionsController < Devise::SessionsController def home_paths(resource) paths = [about_path] + if single_user_mode? && resource.is_a?(User) paths << short_account_path(username: resource.account) end + paths end def continue_after? truthy_param?(:continue) end + + def restart_session + clear_attempt_from_session + redirect_to new_user_session_path, alert: I18n.t('devise.failure.timeout') + end + + def set_attempt_session(user) + session[:attempt_user_id] = user.id + session[:attempt_user_updated_at] = user.updated_at.to_s + end + + def clear_attempt_from_session + session.delete(:attempt_user_id) + session.delete(:attempt_user_updated_at) + end end diff --git a/app/controllers/concerns/sign_in_token_authentication_concern.rb b/app/controllers/concerns/sign_in_token_authentication_concern.rb index 91f813acc..3c95a4afd 100644 --- a/app/controllers/concerns/sign_in_token_authentication_concern.rb +++ b/app/controllers/concerns/sign_in_token_authentication_concern.rb @@ -18,7 +18,9 @@ module SignInTokenAuthenticationConcern def authenticate_with_sign_in_token user = self.resource = find_user - if user_params[:sign_in_token_attempt].present? && session[:attempt_user_id] + if user.present? && session[:attempt_user_id].present? && session[:attempt_user_updated_at] != user.updated_at.to_s + restart_session + elsif user_params.key?(:sign_in_token_attempt) && session[:attempt_user_id] authenticate_with_sign_in_token_attempt(user) elsif user.present? && user.external_or_valid_password?(user_params[:password]) prompt_for_sign_in_token(user) @@ -27,7 +29,7 @@ module SignInTokenAuthenticationConcern def authenticate_with_sign_in_token_attempt(user) if valid_sign_in_token_attempt?(user) - session.delete(:attempt_user_id) + clear_attempt_from_session remember_me(user) sign_in(user) else @@ -42,10 +44,10 @@ module SignInTokenAuthenticationConcern UserMailer.sign_in_token(user, request.remote_ip, request.user_agent, Time.now.utc.to_s).deliver_later! end - set_locale do - session[:attempt_user_id] = user.id - @body_classes = 'lighter' - render :sign_in_token - end + set_attempt_session(user) + + @body_classes = 'lighter' + + set_locale { render :sign_in_token } end end diff --git a/app/controllers/concerns/two_factor_authentication_concern.rb b/app/controllers/concerns/two_factor_authentication_concern.rb index daafe56f4..f54a3c829 100644 --- a/app/controllers/concerns/two_factor_authentication_concern.rb +++ b/app/controllers/concerns/two_factor_authentication_concern.rb @@ -21,7 +21,9 @@ module TwoFactorAuthenticationConcern def authenticate_with_two_factor user = self.resource = find_user - if user_params[:otp_attempt].present? && session[:attempt_user_id] + if user.present? && session[:attempt_user_id].present? && session[:attempt_user_updated_at] != user.updated_at.to_s + restart_session + elsif user_params[:otp_attempt].present? && session[:attempt_user_id] authenticate_with_two_factor_attempt(user) elsif user.present? && user.external_or_valid_password?(user_params[:password]) prompt_for_two_factor(user) @@ -30,7 +32,7 @@ module TwoFactorAuthenticationConcern def authenticate_with_two_factor_attempt(user) if valid_otp_attempt?(user) - session.delete(:attempt_user_id) + clear_attempt_from_session remember_me(user) sign_in(user) else @@ -40,10 +42,10 @@ module TwoFactorAuthenticationConcern end def prompt_for_two_factor(user) - set_locale do - session[:attempt_user_id] = user.id - @body_classes = 'lighter' - render :two_factor - end + set_attempt_session(user) + + @body_classes = 'lighter' + + set_locale { render :two_factor } end end diff --git a/app/controllers/concerns/user_tracking_concern.rb b/app/controllers/concerns/user_tracking_concern.rb index be10705fc..efda37fae 100644 --- a/app/controllers/concerns/user_tracking_concern.rb +++ b/app/controllers/concerns/user_tracking_concern.rb @@ -6,14 +6,13 @@ module UserTrackingConcern UPDATE_SIGN_IN_HOURS = 24 included do - before_action :set_user_activity + before_action :update_user_sign_in end private - def set_user_activity - return unless user_needs_sign_in_update? - current_user.update_tracked_fields!(request) + def update_user_sign_in + current_user.update_sign_in!(request) if user_needs_sign_in_update? end def user_needs_sign_in_update? diff --git a/app/lib/activitypub/activity/move.rb b/app/lib/activitypub/activity/move.rb index 2103f503f..7e073f64d 100644 --- a/app/lib/activitypub/activity/move.rb +++ b/app/lib/activitypub/activity/move.rb @@ -20,6 +20,9 @@ class ActivityPub::Activity::Move < ActivityPub::Activity # Initiate a re-follow for each follower MoveWorker.perform_async(origin_account.id, target_account.id) + rescue + unmark_as_processing! + raise end private diff --git a/app/models/account.rb b/app/models/account.rb index 6b7ebda9e..9288c7fe8 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -415,7 +415,7 @@ class Account < ApplicationRecord end def inboxes - urls = reorder(nil).where(protocol: :activitypub).pluck(Arel.sql("distinct coalesce(nullif(accounts.shared_inbox_url, ''), accounts.inbox_url)")) + urls = reorder(nil).where(protocol: :activitypub).group(:preferred_inbox_url).pluck(Arel.sql("coalesce(nullif(accounts.shared_inbox_url, ''), accounts.inbox_url) AS preferred_inbox_url")) DeliveryFailureTracker.without_unavailable(urls) end diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb index c84e4217c..e70b54d79 100644 --- a/app/models/account_stat.rb +++ b/app/models/account_stat.rb @@ -21,26 +21,26 @@ class AccountStat < ApplicationRecord def increment_count!(key) update(attributes_for_increment(key)) - rescue ActiveRecord::StaleObjectError + rescue ActiveRecord::StaleObjectError, ActiveRecord::RecordNotUnique begin reload_with_id rescue ActiveRecord::RecordNotFound - # Nothing to do - else - retry + return end + + retry end def decrement_count!(key) - update(key => [public_send(key) - 1, 0].max) - rescue ActiveRecord::StaleObjectError + update(attributes_for_decrement(key)) + rescue ActiveRecord::StaleObjectError, ActiveRecord::RecordNotUnique begin reload_with_id rescue ActiveRecord::RecordNotFound - # Nothing to do - else - retry + return end + + retry end private @@ -51,8 +51,13 @@ class AccountStat < ApplicationRecord attrs end + def attributes_for_decrement(key) + attrs = { key => [public_send(key) - 1, 0].max } + attrs + end + def reload_with_id - self.id = find_by!(account: account).id if new_record? + self.id = self.class.find_by!(account: account).id if new_record? reload end end diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb index 0b285fde9..fc6ab5c90 100644 --- a/app/models/form/account_batch.rb +++ b/app/models/form/account_batch.rb @@ -43,7 +43,7 @@ class Form::AccountBatch end def account_domains - accounts.pluck(Arel.sql('distinct domain')).compact + accounts.group(:domain).pluck(:domain).compact end def accounts diff --git a/app/models/tag.rb b/app/models/tag.rb index bce76fc16..0870afc41 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -126,7 +126,7 @@ class Tag < ApplicationRecord end def search_for(term, limit = 5, offset = 0, options = {}) - normalized_term = normalize(term.strip).mb_chars.downcase.to_s + normalized_term = normalize(term.strip) pattern = sanitize_sql_like(normalized_term) + '%' query = Tag.listable.where(arel_table[:name].lower.matches(pattern)) query = query.where(arel_table[:name].lower.eq(normalized_term).or(arel_table[:reviewed_at].not_eq(nil))) if options[:exclude_unreviewed] diff --git a/app/models/user.rb b/app/models/user.rb index 306e2d435..0efeed9f7 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -61,7 +61,7 @@ class User < ApplicationRecord devise :two_factor_backupable, otp_number_of_backup_codes: 10 - devise :registerable, :recoverable, :rememberable, :trackable, :validatable, + devise :registerable, :recoverable, :rememberable, :validatable, :confirmable include Omniauthable @@ -161,6 +161,24 @@ class User < ApplicationRecord prepare_new_user! if new_user && approved? end + def update_sign_in!(request, new_sign_in: false) + old_current, new_current = current_sign_in_at, Time.now.utc + self.last_sign_in_at = old_current || new_current + self.current_sign_in_at = new_current + + old_current, new_current = current_sign_in_ip, request.remote_ip + self.last_sign_in_ip = old_current || new_current + self.current_sign_in_ip = new_current + + if new_sign_in + self.sign_in_count ||= 0 + self.sign_in_count += 1 + end + + save(validate: false) unless new_record? + prepare_returning_user! + end + def pending? !approved? end @@ -192,11 +210,6 @@ class User < ApplicationRecord prepare_new_user! end - def update_tracked_fields!(request) - super - prepare_returning_user! - end - def disable_two_factor! self.otp_required_for_login = false otp_backup_codes&.clear diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 85b915ec6..9f95f1950 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -196,7 +196,7 @@ class ActivityPub::ProcessAccountService < BaseService total_items = collection.is_a?(Hash) && collection['totalItems'].present? && collection['totalItems'].is_a?(Numeric) ? collection['totalItems'] : nil has_first_page = collection.is_a?(Hash) && collection['first'].present? @collections[type] = [total_items, has_first_page] - rescue HTTP::Error, OpenSSL::SSL::SSLError + rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::LengthValidationError @collections[type] = [nil, nil] end diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index 3f7bb7cc5..eee1de51a 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -31,6 +31,7 @@ class ResolveAccountService < BaseService # At this point we are in need of a Webfinger query, which may # yield us a different username/domain through a redirect process_webfinger!(@uri) + @domain = nil if TagManager.instance.local_domain?(@domain) # Because the username/domain pair may be different than what # we already checked, we need to check if we've already got @@ -75,21 +76,27 @@ class ResolveAccountService < BaseService @uri = [@username, @domain].compact.join('@') end - def process_webfinger!(uri, redirected = false) + def process_webfinger!(uri) @webfinger = webfinger!("acct:#{uri}") - confirmed_username, confirmed_domain = @webfinger.subject.gsub(/\Aacct:/, '').split('@') + confirmed_username, confirmed_domain = split_acct(@webfinger.subject) if confirmed_username.casecmp(@username).zero? && confirmed_domain.casecmp(@domain).zero? @username = confirmed_username @domain = confirmed_domain - @uri = uri - elsif !redirected - return process_webfinger!("#{confirmed_username}@#{confirmed_domain}", true) - else - raise WebfingerRedirectError, "The URI #{uri} tries to hijack #{@username}@#{@domain}" + return end - @domain = nil if TagManager.instance.local_domain?(@domain) + # Account doesn't match, so it may have been redirected + @webfinger = webfinger!("acct:#{confirmed_username}@#{confirmed_domain}") + @username, @domain = split_acct(@webfinger.subject) + + unless confirmed_username.casecmp(@username).zero? && confirmed_domain.casecmp(@domain).zero? + raise WebfingerRedirectError, "The URI #{uri} tries to hijack #{@username}@#{@domain}" + end + end + + def split_acct(acct) + acct.gsub(/\Aacct:/, '').split('@') end def process_account! diff --git a/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb b/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb index c5688681f..c3aa8e33c 100644 --- a/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb +++ b/db/migrate/20200620164023_add_fixed_lowercase_index_to_accounts.rb @@ -1,10 +1,30 @@ class AddFixedLowercaseIndexToAccounts < ActiveRecord::Migration[5.2] disable_ddl_transaction! + class CorruptionError < StandardError + def cause + nil + end + + def backtrace + [] + end + end + def up - rename_index :accounts, 'index_accounts_on_username_and_domain_lower', 'old_index_accounts_on_username_and_domain_lower' unless index_name_exists?(:accounts, 'old_index_accounts_on_username_and_domain_lower') - add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true, algorithm: :concurrently - remove_index :accounts, name: 'old_index_accounts_on_username_and_domain_lower' + if index_name_exists?(:accounts, 'old_index_accounts_on_username_and_domain_lower') && index_name_exists?(:accounts, 'index_accounts_on_username_and_domain_lower') + remove_index :accounts, name: 'index_accounts_on_username_and_domain_lower' + elsif index_name_exists?(:accounts, 'index_accounts_on_username_and_domain_lower') + rename_index :accounts, 'index_accounts_on_username_and_domain_lower', 'old_index_accounts_on_username_and_domain_lower' + end + + begin + add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true, algorithm: :concurrently + rescue ActiveRecord::RecordNotUnique + raise CorruptionError, 'Migration failed because of index corruption, see https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/#fixing' + end + + remove_index :accounts, name: 'old_index_accounts_on_username_and_domain_lower' if index_name_exists?(:accounts, 'old_index_accounts_on_username_and_domain_lower') end def down diff --git a/lib/cli.rb b/lib/cli.rb index 9162144cc..38df8abcc 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -13,6 +13,7 @@ require_relative 'mastodon/preview_cards_cli' require_relative 'mastodon/cache_cli' require_relative 'mastodon/upgrade_cli' require_relative 'mastodon/email_domain_blocks_cli' +require_relative 'mastodon/maintenance_cli' require_relative 'mastodon/version' module Mastodon @@ -57,6 +58,9 @@ module Mastodon desc 'email_domain_blocks SUBCOMMAND ...ARGS', 'Manage e-mail domain blocks' subcommand 'email_domain_blocks', Mastodon::EmailDomainBlocksCLI + desc 'maintenance SUBCOMMAND ...ARGS', 'Various maintenance utilities' + subcommand 'maintenance', Mastodon::MaintenanceCLI + option :dry_run, type: :boolean desc 'self-destruct', 'Erase the server from the federation' long_desc <<~LONG_DESC diff --git a/lib/mastodon/maintenance_cli.rb b/lib/mastodon/maintenance_cli.rb new file mode 100644 index 000000000..822051ceb --- /dev/null +++ b/lib/mastodon/maintenance_cli.rb @@ -0,0 +1,618 @@ +# frozen_string_literal: true + +require 'tty-prompt' +require_relative '../../config/boot' +require_relative '../../config/environment' +require_relative 'cli_helper' + +module Mastodon + class MaintenanceCLI < Thor + include CLIHelper + + def self.exit_on_failure? + true + end + + MIN_SUPPORTED_VERSION = 2019_10_01_213028 + MAX_SUPPORTED_VERSION = 2020_12_18_054746 + + # Stubs to enjoy ActiveRecord queries while not depending on a particular + # version of the code/database + + class Status < ApplicationRecord; end + class StatusPin < ApplicationRecord; end + class Poll < ApplicationRecord; end + class Report < ApplicationRecord; end + class Tombstone < ApplicationRecord; end + class Favourite < ApplicationRecord; end + class Follow < ApplicationRecord; end + class FollowRequest < ApplicationRecord; end + class Block < ApplicationRecord; end + class Mute < ApplicationRecord; end + class AccountIdentityProof < ApplicationRecord; end + class AccountModerationNote < ApplicationRecord; end + class AccountPin < ApplicationRecord; end + class ListAccount < ApplicationRecord; end + class PollVote < ApplicationRecord; end + class Mention < ApplicationRecord; end + class AccountDomainBlock < ApplicationRecord; end + class AnnouncementReaction < ApplicationRecord; end + class FeaturedTag < ApplicationRecord; end + class CustomEmoji < ApplicationRecord; end + class CustomEmojiCategory < ApplicationRecord; end + class Bookmark < ApplicationRecord; end + class WebauthnCredential < ApplicationRecord; end + + class PreviewCard < ApplicationRecord + self.inheritance_column = false + end + + class MediaAttachment < ApplicationRecord + self.inheritance_column = nil + end + + class AccountStat < ApplicationRecord + belongs_to :account, inverse_of: :account_stat + end + + # Dummy class, to make migration possible across version changes + class Account < ApplicationRecord + has_one :user, inverse_of: :account + has_one :account_stat, inverse_of: :account + + scope :local, -> { where(domain: nil) } + + def local? + domain.nil? + end + + def acct + local? ? username : "#{username}@#{domain}" + end + + # This is a duplicate of the AccountMerging concern because we need it to + # be independent from code version. + def merge_with!(other_account) + # Since it's the same remote resource, the remote resource likely + # already believes we are following/blocking, so it's safe to + # re-attribute the relationships too. However, during the presence + # of the index bug users could have *also* followed the reference + # account already, therefore mass update will not work and we need + # to check for (and skip past) uniqueness errors + + owned_classes = [ + Status, StatusPin, MediaAttachment, Poll, Report, Tombstone, Favourite, + Follow, FollowRequest, Block, Mute, AccountIdentityProof, + AccountModerationNote, AccountPin, AccountStat, ListAccount, + PollVote, Mention + ] + owned_classes << AccountDeletionRequest if ActiveRecord::Base.connection.table_exists?(:account_deletion_requests) + owned_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes) + + owned_classes.each do |klass| + klass.where(account_id: other_account.id).find_each do |record| + begin + record.update_attribute(:account_id, id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + + target_classes = [Follow, FollowRequest, Block, Mute, AccountModerationNote, AccountPin] + target_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes) + + target_classes.each do |klass| + klass.where(target_account_id: other_account.id).find_each do |record| + begin + record.update_attribute(:target_account_id, id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + end + end + + class User < ApplicationRecord + belongs_to :account, inverse_of: :user + end + + desc 'fix-duplicates', 'Fix duplicates in database and rebuild indexes' + long_desc <<~LONG_DESC + Delete or merge duplicate accounts, statuses, emojis, etc. and rebuild indexes. + + This is useful if your database indexes are corrupted because of issues such as https://wiki.postgresql.org/wiki/Locale_data_changes + + Mastodon has to be stopped to run this task, which will take a long time and may be destructive. + LONG_DESC + def fix_duplicates + @prompt = TTY::Prompt.new + + if ActiveRecord::Migrator.current_version < MIN_SUPPORTED_VERSION + @prompt.warn 'Your version of the database schema is too old and is not supported by this script.' + @prompt.warn 'Please update to at least Mastodon 3.0.0 before running this script.' + exit(1) + elsif ActiveRecord::Migrator.current_version > MAX_SUPPORTED_VERSION + @prompt.warn 'Your version of the database schema is more recent than this script, this may cause unexpected errors.' + exit(1) unless @prompt.yes?('Continue anyway?') + end + + @prompt.warn 'This task will take a long time to run and is potentially destructive.' + @prompt.warn 'Please make sure to stop Mastodon and have a backup.' + exit(1) unless @prompt.yes?('Continue?') + + deduplicate_accounts! + deduplicate_users! + deduplicate_account_domain_blocks! + deduplicate_account_identity_proofs! + deduplicate_announcement_reactions! + deduplicate_conversations! + deduplicate_custom_emojis! + deduplicate_custom_emoji_categories! + deduplicate_domain_allows! + deduplicate_domain_blocks! + deduplicate_unavailable_domains! + deduplicate_email_domain_blocks! + deduplicate_media_attachments! + deduplicate_preview_cards! + deduplicate_statuses! + deduplicate_tags! + deduplicate_webauthn_credentials! + + Rails.cache.clear + + @prompt.say 'Finished!' + end + + private + + def deduplicate_accounts! + remove_index_if_exists!(:accounts, 'index_accounts_on_username_and_domain_lower') + + @prompt.say 'Deduplicating accounts… for local accounts, you will be asked to chose which account to keep unchanged.' + + find_duplicate_accounts.each do |row| + accounts = Account.where(id: row['ids'].split(',')).to_a + + if accounts.first.local? + deduplicate_local_accounts!(accounts) + else + deduplicate_remote_accounts!(accounts) + end + end + + @prompt.say 'Restoring index_accounts_on_username_and_domain_lower…' + if ActiveRecord::Migrator.current_version < 20200620164023 + ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true + else + ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true + end + end + + def deduplicate_users! + remove_index_if_exists!(:users, 'index_users_on_confirmation_token') + remove_index_if_exists!(:users, 'index_users_on_email') + remove_index_if_exists!(:users, 'index_users_on_remember_token') + remove_index_if_exists!(:users, 'index_users_on_reset_password_token') + + @prompt.say 'Deduplicating user records…' + + # Deduplicating email + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row| + users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse + ref_user = users.shift + @prompt.warn "Multiple users registered with e-mail address #{ref_user.email}." + @prompt.warn "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}" + @prompt.warn 'Please reach out to them and set another address with `tootctl account modify` or delete them.' + + i = 0 + users.each do |user| + user.update!(email: "#{i} " + user.email) + end + end + + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row| + users = User.where(id: row['ids'].split(',')).sort_by(&:created_at).reverse.drop(1) + @prompt.warn "Unsetting confirmation token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}" + + users.each do |user| + user.update!(confirmation_token: nil) + end + end + + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row| + users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1) + @prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}" + + users.each do |user| + user.update!(remember_token: nil) + end + end + + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE reset_password_token IS NOT NULL GROUP BY reset_password_token HAVING count(*) > 1").each do |row| + users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1) + @prompt.warn "Unsetting password reset token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}" + + users.each do |user| + user.update!(reset_password_token: nil) + end + end + + @prompt.say 'Restoring users indexes…' + ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true + ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true + ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true + ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true + end + + def deduplicate_account_domain_blocks! + remove_index_if_exists!(:account_domain_blocks, 'index_account_domain_blocks_on_account_id_and_domain') + + @prompt.say 'Removing duplicate account domain blocks…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_domain_blocks GROUP BY account_id, domain HAVING count(*) > 1").each do |row| + AccountDomainBlock.where(id: row['ids'].split(',').drop(1)).delete_all + end + + @prompt.say 'Restoring account domain blocks indexes…' + ActiveRecord::Base.connection.add_index :account_domain_blocks, ['account_id', 'domain'], name: 'index_account_domain_blocks_on_account_id_and_domain', unique: true + end + + def deduplicate_account_identity_proofs! + remove_index_if_exists!(:account_identity_proofs, 'index_account_proofs_on_account_and_provider_and_username') + + @prompt.say 'Removing duplicate account identity proofs…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM account_identity_proofs GROUP BY account_id, provider, provider_username HAVING count(*) > 1").each do |row| + AccountIdentityProof.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring account identity proofs indexes…' + ActiveRecord::Base.connection.add_index :account_identity_proofs, ['account_id', 'provider', 'provider_username'], name: 'index_account_proofs_on_account_and_provider_and_username', unique: true + end + + def deduplicate_announcement_reactions! + return unless ActiveRecord::Base.connection.table_exists?(:announcement_reactions) + + remove_index_if_exists!(:announcement_reactions, 'index_announcement_reactions_on_account_id_and_announcement_id') + + @prompt.say 'Removing duplicate account identity proofs…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM announcement_reactions GROUP BY account_id, announcement_id, name HAVING count(*) > 1").each do |row| + AnnouncementReaction.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring announcement_reactions indexes…' + ActiveRecord::Base.connection.add_index :announcement_reactions, ['account_id', 'announcement_id', 'name'], name: 'index_announcement_reactions_on_account_id_and_announcement_id', unique: true + end + + def deduplicate_conversations! + remove_index_if_exists!(:conversations, 'index_conversations_on_uri') + + @prompt.say 'Deduplicating conversations…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM conversations WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row| + conversations = Conversation.where(id: row['ids'].split(',')).sort_by(&:id).reverse + + ref_conversation = conversations.shift + + conversations.each do |other| + merge_conversations!(ref_conversation, other) + other.destroy + end + end + + @prompt.say 'Restoring conversations indexes…' + ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true + end + + def deduplicate_custom_emojis! + remove_index_if_exists!(:custom_emojis, 'index_custom_emojis_on_shortcode_and_domain') + + @prompt.say 'Deduplicating custom_emojis…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emojis GROUP BY shortcode, domain HAVING count(*) > 1").each do |row| + emojis = CustomEmoji.where(id: row['ids'].split(',')).sort_by(&:id).reverse + + ref_emoji = emojis.shift + + emojis.each do |other| + merge_custom_emojis!(ref_emoji, other) + other.destroy + end + end + + @prompt.say 'Restoring custom_emojis indexes…' + ActiveRecord::Base.connection.add_index :custom_emojis, ['shortcode', 'domain'], name: 'index_custom_emojis_on_shortcode_and_domain', unique: true + end + + def deduplicate_custom_emoji_categories! + remove_index_if_exists!(:custom_emoji_categories, 'index_custom_emoji_categories_on_name') + + @prompt.say 'Deduplicating custom_emoji_categories…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM custom_emoji_categories GROUP BY name HAVING count(*) > 1").each do |row| + categories = CustomEmojiCategory.where(id: row['ids'].split(',')).sort_by(&:id).reverse + + ref_category = categories.shift + + categories.each do |other| + merge_custom_emoji_categories!(ref_category, other) + other.destroy + end + end + + @prompt.say 'Restoring custom_emoji_categories indexes…' + ActiveRecord::Base.connection.add_index :custom_emoji_categories, ['name'], name: 'index_custom_emoji_categories_on_name', unique: true + end + + def deduplicate_domain_allows! + remove_index_if_exists!(:domain_allows, 'index_domain_allows_on_domain') + + @prompt.say 'Deduplicating domain_allows…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_allows GROUP BY domain HAVING count(*) > 1").each do |row| + DomainAllow.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring domain_allows indexes…' + ActiveRecord::Base.connection.add_index :domain_allows, ['domain'], name: 'index_domain_allows_on_domain', unique: true + end + + def deduplicate_domain_blocks! + remove_index_if_exists!(:domain_blocks, 'index_domain_blocks_on_domain') + + @prompt.say 'Deduplicating domain_allows…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row| + domain_blocks = DomainBlock.where(id: row['ids'].split(',')).by_severity.reverse.to_a + + reject_media = domain_blocks.any?(&:reject_media?) + reject_reports = domain_blocks.any?(&:reject_reports?) + + reference_block = domain_blocks.shift + + private_comment = domain_blocks.reduce(reference_block.private_comment.presence) { |a, b| a || b.private_comment.presence } + public_comment = domain_blocks.reduce(reference_block.public_comment.presence) { |a, b| a || b.public_comment.presence } + + reference_block.update!(reject_media: reject_media, reject_reports: reject_reports, private_comment: private_comment, public_comment: public_comment) + + domain_blocks.each(&:destroy) + end + + @prompt.say 'Restoring domain_blocks indexes…' + ActiveRecord::Base.connection.add_index :domain_blocks, ['domain'], name: 'index_domain_blocks_on_domain', unique: true + end + + def deduplicate_unavailable_domains! + return unless ActiveRecord::Base.connection.table_exists?(:unavailable_domains) + + remove_index_if_exists!(:unavailable_domains, 'index_unavailable_domains_on_domain') + + @prompt.say 'Deduplicating unavailable_domains…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM unavailable_domains GROUP BY domain HAVING count(*) > 1").each do |row| + UnavailableDomain.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring domain_allows indexes…' + ActiveRecord::Base.connection.add_index :unavailable_domains, ['domain'], name: 'index_unavailable_domains_on_domain', unique: true + end + + def deduplicate_email_domain_blocks! + remove_index_if_exists!(:email_domain_blocks, 'index_email_domain_blocks_on_domain') + + @prompt.say 'Deduplicating email_domain_blocks…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM email_domain_blocks GROUP BY domain HAVING count(*) > 1").each do |row| + domain_blocks = EmailDomainBlock.where(id: row['ids'].split(',')).sort_by { |b| b.parent.nil? ? 1 : 0 }.to_a + domain_blocks.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring email_domain_blocks indexes…' + ActiveRecord::Base.connection.add_index :email_domain_blocks, ['domain'], name: 'index_email_domain_blocks_on_domain', unique: true + end + + def deduplicate_media_attachments! + remove_index_if_exists!(:media_attachments, 'index_media_attachments_on_shortcode') + + @prompt.say 'Deduplicating media_attachments…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM media_attachments WHERE shortcode IS NOT NULL GROUP BY shortcode HAVING count(*) > 1").each do |row| + MediaAttachment.where(id: row['ids'].split(',').drop(1)).update_all(shortcode: nil) + end + + @prompt.say 'Restoring media_attachments indexes…' + ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true + end + + def deduplicate_preview_cards! + remove_index_if_exists!(:preview_cards, 'index_preview_cards_on_url') + + @prompt.say 'Deduplicating preview_cards…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM preview_cards GROUP BY url HAVING count(*) > 1").each do |row| + PreviewCard.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring preview_cards indexes…' + ActiveRecord::Base.connection.add_index :preview_cards, ['url'], name: 'index_preview_cards_on_url', unique: true + end + + def deduplicate_statuses! + remove_index_if_exists!(:statuses, 'index_statuses_on_uri') + + @prompt.say 'Deduplicating statuses…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM statuses WHERE uri IS NOT NULL GROUP BY uri HAVING count(*) > 1").each do |row| + statuses = Status.where(id: row['ids'].split(',')).sort_by(&:id) + ref_status = statuses.shift + statuses.each do |status| + merge_statuses!(ref_status, status) if status.account_id == ref_status.account_id + status.destroy + end + end + + @prompt.say 'Restoring statuses indexes…' + ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true + end + + def deduplicate_tags! + remove_index_if_exists!(:tags, 'index_tags_on_name_lower') + + @prompt.say 'Deduplicating tags…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM tags GROUP BY lower((name)::text) HAVING count(*) > 1").each do |row| + tags = Tag.where(id: row['ids'].split(',')).sort_by { |t| [t.usable?, t.trendable?, t.listable?].count(false) } + ref_tag = tags.shift + tags.each do |tag| + merge_tags!(ref_tag, tag) + tag.destroy + end + end + + @prompt.say 'Restoring tags indexes…' + ActiveRecord::Base.connection.add_index :tags, 'lower((name)::text)', name: 'index_tags_on_name_lower', unique: true + end + + def deduplicate_webauthn_credentials! + return unless ActiveRecord::Base.connection.table_exists?(:webauthn_credentials) + + remove_index_if_exists!(:webauthn_credentials, 'index_webauthn_credentials_on_external_id') + + @prompt.say 'Deduplicating webauthn_credentials…' + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webauthn_credentials GROUP BY external_id HAVING count(*) > 1").each do |row| + WebauthnCredential.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + end + + @prompt.say 'Restoring webauthn_credentials indexes…' + ActiveRecord::Base.connection.add_index :webauthn_credentials, ['external_id'], name: 'index_webauthn_credentials_on_external_id', unique: true + end + + def deduplicate_local_accounts!(accounts) + accounts = accounts.sort_by(&:id).reverse + + @prompt.warn "Multiple local accounts were found for username '#{accounts.first.username}'." + @prompt.warn 'All those accounts are distinct accounts but only the most recently-created one is fully-functionnal.' + + accounts.each_with_index do |account, idx| + @prompt.say '%2d. %s: created at: %s; updated at: %s; last logged in at: %s; statuses: %5d; last status at: %s' % [idx, account.username, account.created_at, account.updated_at, account.user&.last_sign_in_at&.to_s || 'N/A', account.account_stat&.statuses_count || 0, account.account_stat&.last_status_at || 'N/A'] + end + + @prompt.say 'Please chose the one to keep unchanged, other ones will be automatically renamed.' + + ref_id = @prompt.ask('Account to keep unchanged:') do |q| + q.required true + q.default 0 + q.convert :int + end + + accounts.delete_at(ref_id) + + i = 0 + accounts.each do |account| + i += 1 + username = account.username + "_#{i}" + + while Account.local.exists?(username: username) + i += 1 + username = account.username + "_#{i}" + end + + account.update!(username: username) + end + end + + def deduplicate_remote_accounts!(accounts) + accounts = accounts.sort_by(&:updated_at).reverse + + reference_account = accounts.shift + + accounts.each do |other_account| + if other_account.public_key == reference_account.public_key + # The accounts definitely point to the same resource, so + # it's safe to re-attribute content and relationships + reference_account.merge_with!(other_account) + end + + other_account.destroy + end + end + + def merge_conversations!(main_conv, duplicate_conv) + owned_classes = [ConversationMute, AccountConversation] + owned_classes.each do |klass| + klass.where(conversation_id: duplicate_conv.id).find_each do |record| + begin + record.update_attribute(:account_id, main_conv.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + end + + def merge_custom_emojis!(main_emoji, duplicate_emoji) + owned_classes = [AnnouncementReaction] + owned_classes.each do |klass| + klass.where(custom_emoji_id: duplicate_emoji.id).update_all(custom_emoji_id: main_emoji.id) + end + end + + def merge_custom_emoji_categories!(main_category, duplicate_category) + owned_classes = [CustomEmoji] + owned_classes.each do |klass| + klass.where(category_id: duplicate_category.id).update_all(category_id: main_category.id) + end + end + + def merge_statuses!(main_status, duplicate_status) + owned_classes = [Favourite, Mention, Poll] + owned_classes << Bookmark if ActiveRecord::Base.connection.table_exists?(:bookmarks) + owned_classes.each do |klass| + klass.where(status_id: duplicate_status.id).find_each do |record| + begin + record.update_attribute(:status_id, main_status.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + + StatusPin.where(account_id: main_status.account_id, status_id: duplicate_status.id).find_each do |record| + begin + record.update_attribute(:status_id, main_status.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + + Status.where(in_reply_to_id: duplicate_status.id).find_each do |record| + begin + record.update_attribute(:in_reply_to_id, main_status.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + + Status.where(reblog_of_id: duplicate_status.id).find_each do |record| + begin + record.update_attribute(:reblog_of_id, main_status.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + + def merge_tags!(main_tag, duplicate_tag) + [FeaturedTag].each do |klass| + klass.where(tag_id: duplicate_tag.id).find_each do |record| + begin + record.update_attribute(:tag_id, main_tag.id) + rescue ActiveRecord::RecordNotUnique + next + end + end + end + end + + def find_duplicate_accounts + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM accounts GROUP BY lower(username), COALESCE(lower(domain), '') HAVING count(*) > 1") + end + + def remove_index_if_exists!(table, name) + ActiveRecord::Base.connection.remove_index(table, name: name) + rescue ArgumentError + nil + rescue ActiveRecord::StatementInvalid + nil + end + end +end diff --git a/lib/paperclip/response_with_limit_adapter.rb b/lib/paperclip/response_with_limit_adapter.rb index 8711b1349..17a2abd25 100644 --- a/lib/paperclip/response_with_limit_adapter.rb +++ b/lib/paperclip/response_with_limit_adapter.rb @@ -16,7 +16,7 @@ module Paperclip private def cache_current_values - @original_filename = filename_from_content_disposition || filename_from_path || 'data' + @original_filename = filename_from_content_disposition.presence || filename_from_path.presence || 'data' @size = @target.response.content_length @tempfile = copy_to_tempfile(@target) @content_type = ContentTypeDetector.new(@tempfile.path).detect diff --git a/spec/controllers/auth/sessions_controller_spec.rb b/spec/controllers/auth/sessions_controller_spec.rb index c387842cd..4560d64a8 100644 --- a/spec/controllers/auth/sessions_controller_spec.rb +++ b/spec/controllers/auth/sessions_controller_spec.rb @@ -215,7 +215,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'using a valid OTP' do before do - post :create, params: { user: { otp_attempt: user.current_otp } }, session: { attempt_user_id: user.id } + post :create, params: { user: { otp_attempt: user.current_otp } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'redirects to home' do @@ -230,7 +230,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'when the server has an decryption error' do before do allow_any_instance_of(User).to receive(:validate_and_consume_otp!).and_raise(OpenSSL::Cipher::CipherError) - post :create, params: { user: { otp_attempt: user.current_otp } }, session: { attempt_user_id: user.id } + post :create, params: { user: { otp_attempt: user.current_otp } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'shows a login error' do @@ -244,7 +244,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'using a valid recovery code' do before do - post :create, params: { user: { otp_attempt: recovery_codes.first } }, session: { attempt_user_id: user.id } + post :create, params: { user: { otp_attempt: recovery_codes.first } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'redirects to home' do @@ -258,7 +258,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'using an invalid OTP' do before do - post :create, params: { user: { otp_attempt: 'wrongotp' } }, session: { attempt_user_id: user.id } + post :create, params: { user: { otp_attempt: 'wrongotp' } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'shows a login error' do @@ -302,7 +302,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'using a valid sign in token' do before do user.generate_sign_in_token && user.save - post :create, params: { user: { sign_in_token_attempt: user.sign_in_token } }, session: { attempt_user_id: user.id } + post :create, params: { user: { sign_in_token_attempt: user.sign_in_token } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'redirects to home' do @@ -316,7 +316,7 @@ RSpec.describe Auth::SessionsController, type: :controller do context 'using an invalid sign in token' do before do - post :create, params: { user: { sign_in_token_attempt: 'wrongotp' } }, session: { attempt_user_id: user.id } + post :create, params: { user: { sign_in_token_attempt: 'wrongotp' } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s } end it 'shows a login error' do diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb index 98d29e6f3..75f628076 100644 --- a/spec/models/account_spec.rb +++ b/spec/models/account_spec.rb @@ -817,4 +817,27 @@ RSpec.describe Account, type: :model do include_examples 'AccountAvatar', :account include_examples 'AccountHeader', :account + + describe '#increment_count!' do + subject { Fabricate(:account) } + + it 'increments the count in multi-threaded an environment when account_stat is not yet initialized' do + subject + + increment_by = 15 + wait_for_start = true + + threads = Array.new(increment_by) do + Thread.new do + true while wait_for_start + Account.find(subject.id).increment_count!(:followers_count) + end + end + + wait_for_start = false + threads.each(&:join) + + expect(subject.reload.followers_count).to eq 15 + end + end end diff --git a/spec/services/resolve_account_service_spec.rb b/spec/services/resolve_account_service_spec.rb index cea942e39..6e4a0d9fe 100644 --- a/spec/services/resolve_account_service_spec.rb +++ b/spec/services/resolve_account_service_spec.rb @@ -4,23 +4,61 @@ RSpec.describe ResolveAccountService, type: :service do subject { described_class.new } before do - stub_request(:get, "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt')) - stub_request(:get, "https://example.com/.well-known/webfinger?resource=acct:catsrgr8@example.com").to_return(status: 404) stub_request(:get, "https://example.com/.well-known/host-meta").to_return(status: 404) stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt')) - stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:catsrgr8@quitter.no").to_return(status: 404) stub_request(:get, "https://ap.example.com/.well-known/webfinger?resource=acct:foo@ap.example.com").to_return(request_fixture('activitypub-webfinger.txt')) stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor.txt')) stub_request(:get, "https://ap.example.com/users/foo.atom").to_return(request_fixture('activitypub-feed.txt')) stub_request(:get, %r{https://ap.example.com/users/foo/\w+}).to_return(status: 404) end - it 'raises error if no such user can be resolved via webfinger' do - expect(subject.call('catsrgr8@quitter.no')).to be_nil + context 'when there is an LRDD endpoint but no resolvable account' do + before do + stub_request(:get, "https://quitter.no/.well-known/host-meta").to_return(request_fixture('.host-meta.txt')) + stub_request(:get, "https://quitter.no/.well-known/webfinger?resource=acct:catsrgr8@quitter.no").to_return(status: 404) + end + + it 'returns nil' do + expect(subject.call('catsrgr8@quitter.no')).to be_nil + end end - it 'raises error if the domain does not have webfinger' do - expect(subject.call('catsrgr8@example.com')).to be_nil + context 'when there is no LRDD endpoint nor resolvable account' do + before do + stub_request(:get, "https://example.com/.well-known/webfinger?resource=acct:catsrgr8@example.com").to_return(status: 404) + end + + it 'returns nil' do + expect(subject.call('catsrgr8@example.com')).to be_nil + end + end + + context 'with a legitimate webfinger redirection' do + before do + webfinger = { subject: 'acct:foo@ap.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo' }] } + stub_request(:get, 'https://redirected.example.com/.well-known/webfinger?resource=acct:Foo@redirected.example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' }) + end + + it 'returns new remote account' do + account = subject.call('Foo@redirected.example.com') + + expect(account.activitypub?).to eq true + expect(account.acct).to eq 'foo@ap.example.com' + expect(account.inbox_url).to eq 'https://ap.example.com/users/foo/inbox' + end + end + + context 'with too many webfinger redirections' do + before do + webfinger = { subject: 'acct:foo@evil.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo' }] } + stub_request(:get, 'https://redirected.example.com/.well-known/webfinger?resource=acct:Foo@redirected.example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' }) + webfinger2 = { subject: 'acct:foo@ap.example.com', links: [{ rel: 'self', href: 'https://ap.example.com/users/foo' }] } + stub_request(:get, 'https://evil.example.com/.well-known/webfinger?resource=acct:foo@evil.example.com').to_return(body: Oj.dump(webfinger2), headers: { 'Content-Type': 'application/jrd+json' }) + end + + it 'returns nil' do + expect(subject.call('Foo@redirected.example.com')).to be_nil + end end context 'with an ActivityPub account' do