From eea2654236966fee69c4f18217a6a62b1c66d2da Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 13 Nov 2023 17:58:00 +0100
Subject: [PATCH 001/117] Fix format-dependent redirects being cached
 regardless of requested format (#27634)

---
 config/routes.rb            | 23 +++++++++++++++++++----
 spec/requests/cache_spec.rb | 16 +++++++++++-----
 2 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/config/routes.rb b/config/routes.rb
index 5de8562a8..7e2f1aabe 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -3,6 +3,18 @@
 require 'sidekiq_unique_jobs/web'
 require 'sidekiq-scheduler/web'
 
+class RedirectWithVary < ActionDispatch::Routing::PathRedirect
+  def serve(...)
+    super.tap do |_, headers, _|
+      headers['Vary'] = 'Origin, Accept'
+    end
+  end
+end
+
+def redirect_with_vary(path)
+  RedirectWithVary.new(301, path)
+end
+
 Rails.application.routes.draw do
   # Paths of routes on the web app that to not require to be indexed or
   # have alternative format representations requiring separate controllers
@@ -90,10 +102,13 @@ Rails.application.routes.draw do
     confirmations:      'auth/confirmations',
   }
 
-  get '/users/:username', to: redirect('/@%{username}'), constraints: lambda { |req| req.format.nil? || req.format.html? }
-  get '/users/:username/following', to: redirect('/@%{username}/following'), constraints: lambda { |req| req.format.nil? || req.format.html? }
-  get '/users/:username/followers', to: redirect('/@%{username}/followers'), constraints: lambda { |req| req.format.nil? || req.format.html? }
-  get '/users/:username/statuses/:id', to: redirect('/@%{username}/%{id}'), constraints: lambda { |req| req.format.nil? || req.format.html? }
+  # rubocop:disable Style/FormatStringToken - those do not go through the usual formatting functions and are not safe to correct
+  get '/users/:username', to: redirect_with_vary('/@%{username}'), constraints: lambda { |req| req.format.nil? || req.format.html? }
+  get '/users/:username/following', to: redirect_with_vary('/@%{username}/following'), constraints: lambda { |req| req.format.nil? || req.format.html? }
+  get '/users/:username/followers', to: redirect_with_vary('/@%{username}/followers'), constraints: lambda { |req| req.format.nil? || req.format.html? }
+  get '/users/:username/statuses/:id', to: redirect_with_vary('/@%{username}/%{id}'), constraints: lambda { |req| req.format.nil? || req.format.html? }
+  # rubocop:enable Style/FormatStringToken
+
   get '/authorize_follow', to: redirect { |_, request| "/authorize_interaction?#{request.params.to_query}" }
 
   resources :accounts, path: 'users', only: [:show], param: :username do
diff --git a/spec/requests/cache_spec.rb b/spec/requests/cache_spec.rb
index c391c8b3d..d40895fc3 100644
--- a/spec/requests/cache_spec.rb
+++ b/spec/requests/cache_spec.rb
@@ -124,7 +124,7 @@ describe 'Caching behavior' do
       expect(response.cookies).to be_empty
     end
 
-    it 'sets public cache control' do
+    it 'sets public cache control', :aggregate_failures do
       # expect(response.cache_control[:max_age]&.to_i).to be_positive
       expect(response.cache_control[:public]).to be_truthy
       expect(response.cache_control[:private]).to be_falsy
@@ -141,11 +141,8 @@ describe 'Caching behavior' do
   end
 
   shared_examples 'non-cacheable error' do
-    it 'does not return HTTP success' do
+    it 'does not return HTTP success and does not have cache headers', :aggregate_failures do
       expect(response).to_not have_http_status(200)
-    end
-
-    it 'does not have cache headers' do
       expect(response.cache_control[:public]).to be_falsy
     end
   end
@@ -182,6 +179,15 @@ describe 'Caching behavior' do
   end
 
   context 'when anonymously accessed' do
+    describe '/users/alice' do
+      it 'redirects with proper cache header', :aggregate_failures do
+        get '/users/alice'
+
+        expect(response).to redirect_to('/@alice')
+        expect(response.headers['Vary']&.split(',')&.map { |x| x.strip.downcase }).to include('accept')
+      end
+    end
+
     TestEndpoints::ALWAYS_CACHED.each do |endpoint|
       describe endpoint do
         before { get endpoint }

From ef149674f093e97ffbeb12910e2ebdd668bed7c2 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 23 Oct 2023 14:27:07 +0200
Subject: [PATCH 002/117] Change Content-Security-Policy to be tighter on media
 paths (#26889)

---
 config/initializers/content_security_policy.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/config/initializers/content_security_policy.rb b/config/initializers/content_security_policy.rb
index 6ce84a6e4..85a328b44 100644
--- a/config/initializers/content_security_policy.rb
+++ b/config/initializers/content_security_policy.rb
@@ -5,7 +5,11 @@
 # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy
 
 def host_to_url(str)
-  "http#{Rails.configuration.x.use_https ? 's' : ''}://#{str.split('/').first}" if str.present?
+  return if str.blank?
+
+  uri = Addressable::URI.parse("http#{Rails.configuration.x.use_https ? 's' : ''}://#{str}")
+  uri.path += '/' unless uri.path.blank? || uri.path.end_with?('/')
+  uri.to_s
 end
 
 base_host = Rails.configuration.x.web_domain

From 156d32689b3d084a7fe9f0aa6a8788cd4e831f17 Mon Sep 17 00:00:00 2001
From: gunchleoc <fios@foramnagaidhlig.net>
Date: Thu, 28 Sep 2023 09:13:44 +0100
Subject: [PATCH 003/117] Only strip country code when language not listed in
 SUPPORTED_LOCALES (#27099)

---
 app/helpers/languages_helper.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/helpers/languages_helper.rb b/app/helpers/languages_helper.rb
index a8c66552c..c42c4c23e 100644
--- a/app/helpers/languages_helper.rb
+++ b/app/helpers/languages_helper.rb
@@ -254,6 +254,7 @@ module LanguagesHelper
 
   def valid_locale_or_nil(str)
     return if str.blank?
+    return str if valid_locale?(str)
 
     code, = str.to_s.split(/[_-]/) # Strip out the region from e.g. en_US or ja-JP
 

From aa69ca74ed113e4f0526522931f24db55de90b9a Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 6 Oct 2023 17:46:04 +0200
Subject: [PATCH 004/117] Fix incorrect serialization of regional languages in
 `contentMap` (#27207)

---
 app/lib/activitypub/case_transform.rb               |  2 ++
 .../serializers/activitypub/note_serializer_spec.rb | 13 ++++++++++---
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/app/lib/activitypub/case_transform.rb b/app/lib/activitypub/case_transform.rb
index da2c5eb8b..bf5de7221 100644
--- a/app/lib/activitypub/case_transform.rb
+++ b/app/lib/activitypub/case_transform.rb
@@ -14,6 +14,8 @@ module ActivityPub::CaseTransform
       when String
         camel_lower_cache[value] ||= if value.start_with?('_:')
                                        "_:#{value.delete_prefix('_:').underscore.camelize(:lower)}"
+                                     elsif LanguagesHelper::ISO_639_1_REGIONAL.key?(value.to_sym)
+                                       value
                                      else
                                        value.underscore.camelize(:lower)
                                      end
diff --git a/spec/serializers/activitypub/note_serializer_spec.rb b/spec/serializers/activitypub/note_serializer_spec.rb
index 4b2b8ec87..31ee31f13 100644
--- a/spec/serializers/activitypub/note_serializer_spec.rb
+++ b/spec/serializers/activitypub/note_serializer_spec.rb
@@ -7,7 +7,7 @@ describe ActivityPub::NoteSerializer do
 
   let!(:account) { Fabricate(:account) }
   let!(:other) { Fabricate(:account) }
-  let!(:parent) { Fabricate(:status, account: account, visibility: :public) }
+  let!(:parent) { Fabricate(:status, account: account, visibility: :public, language: 'zh-TW') }
   let!(:reply_by_account_first) { Fabricate(:status, account: account, thread: parent, visibility: :public) }
   let!(:reply_by_account_next) { Fabricate(:status, account: account, thread: parent, visibility: :public) }
   let!(:reply_by_other_first) { Fabricate(:status, account: other, thread: parent, visibility: :public) }
@@ -18,8 +18,15 @@ describe ActivityPub::NoteSerializer do
     @serialization = ActiveModelSerializers::SerializableResource.new(parent, serializer: described_class, adapter: ActivityPub::Adapter)
   end
 
-  it 'has a Note type' do
-    expect(subject['type']).to eql('Note')
+  it 'has the expected shape' do
+    expect(subject).to include({
+      '@context' => include('https://www.w3.org/ns/activitystreams'),
+      'type' => 'Note',
+      'attributedTo' => ActivityPub::TagManager.instance.uri_for(account),
+      'contentMap' => include({
+        'zh-TW' => a_kind_of(String),
+      }),
+    })
   end
 
   it 'has a replies collection' do

From cdedae6d63ff6ac21dd8810896d39f016fd8ec68 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 23 Oct 2023 14:19:38 +0200
Subject: [PATCH 005/117] Fix some link anchors being recognized as hashtags
 (#27271)

---
 app/models/tag.rb       |  2 +-
 spec/models/tag_spec.rb | 26 +++++++++++++++-----------
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 672d80c8b..413f6f500 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -35,7 +35,7 @@ class Tag < ApplicationRecord
   HASHTAG_LAST_SEQUENCE = '([[:word:]_]*[[:alpha:]][[:word:]_]*)'
   HASHTAG_NAME_PAT = "#{HASHTAG_FIRST_SEQUENCE}|#{HASHTAG_LAST_SEQUENCE}"
 
-  HASHTAG_RE = %r{(?:^|[^/)\w])#(#{HASHTAG_NAME_PAT})}i
+  HASHTAG_RE = %r{(?<![=/)[:word]])#(#{HASHTAG_NAME_PAT})}i
   HASHTAG_NAME_RE = /\A(#{HASHTAG_NAME_PAT})\z/i
   HASHTAG_INVALID_CHARS_RE = /[^[:alnum:]#{HASHTAG_SEPARATORS}]/
 
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 4d6e5c380..80b9c861f 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -32,44 +32,48 @@ RSpec.describe Tag do
       expect(subject.match('https://en.wikipedia.org/wiki/Ghostbusters_(song)#Lawsuit')).to be_nil
     end
 
+    it 'does not match URLs with hashtag-like anchors after an empty query parameter' do
+      expect(subject.match('https://en.wikipedia.org/wiki/Ghostbusters_(song)?foo=#Lawsuit')).to be_nil
+    end
+
     it 'matches #ａｅｓｔｈｅｔｉｃ' do
-      expect(subject.match('this is #ａｅｓｔｈｅｔｉｃ').to_s).to eq ' #ａｅｓｔｈｅｔｉｃ'
+      expect(subject.match('this is #ａｅｓｔｈｅｔｉｃ').to_s).to eq '#ａｅｓｔｈｅｔｉｃ'
     end
 
     it 'matches digits at the start' do
-      expect(subject.match('hello #3d').to_s).to eq ' #3d'
+      expect(subject.match('hello #3d').to_s).to eq '#3d'
     end
 
     it 'matches digits in the middle' do
-      expect(subject.match('hello #l33ts35k').to_s).to eq ' #l33ts35k'
+      expect(subject.match('hello #l33ts35k').to_s).to eq '#l33ts35k'
     end
 
     it 'matches digits at the end' do
-      expect(subject.match('hello #world2016').to_s).to eq ' #world2016'
+      expect(subject.match('hello #world2016').to_s).to eq '#world2016'
     end
 
     it 'matches underscores at the beginning' do
-      expect(subject.match('hello #_test').to_s).to eq ' #_test'
+      expect(subject.match('hello #_test').to_s).to eq '#_test'
     end
 
     it 'matches underscores at the end' do
-      expect(subject.match('hello #test_').to_s).to eq ' #test_'
+      expect(subject.match('hello #test_').to_s).to eq '#test_'
     end
 
     it 'matches underscores in the middle' do
-      expect(subject.match('hello #one_two_three').to_s).to eq ' #one_two_three'
+      expect(subject.match('hello #one_two_three').to_s).to eq '#one_two_three'
     end
 
     it 'matches middle dots' do
-      expect(subject.match('hello #one·two·three').to_s).to eq ' #one·two·three'
+      expect(subject.match('hello #one·two·three').to_s).to eq '#one·two·three'
     end
 
     it 'matches ・unicode in ぼっち・ざ・ろっく correctly' do
-      expect(subject.match('testing #ぼっち・ざ・ろっく').to_s).to eq ' #ぼっち・ざ・ろっく'
+      expect(subject.match('testing #ぼっち・ざ・ろっく').to_s).to eq '#ぼっち・ざ・ろっく'
     end
 
     it 'matches ZWNJ' do
-      expect(subject.match('just add #نرم‌افزار and').to_s).to eq ' #نرم‌افزار'
+      expect(subject.match('just add #نرم‌افزار and').to_s).to eq '#نرم‌افزار'
     end
 
     it 'does not match middle dots at the start' do
@@ -77,7 +81,7 @@ RSpec.describe Tag do
     end
 
     it 'does not match middle dots at the end' do
-      expect(subject.match('hello #one·two·three·').to_s).to eq ' #one·two·three'
+      expect(subject.match('hello #one·two·three·').to_s).to eq '#one·two·three'
     end
 
     it 'does not match purely-numeric hashtags' do

From 8be33d4316c7ea828f8b10361d5189fde79c3295 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?KMY=EF=BC=88=E9=9B=AA=E3=81=82=E3=81=99=E3=81=8B=EF=BC=89?=
 <tt@kmycode.net>
Date: Fri, 13 Oct 2023 18:15:47 +0900
Subject: [PATCH 006/117] Fix when unfollow a tag, my post also disappears from
 the home timeline (#27391)

---
 app/lib/feed_manager.rb       |  1 +
 spec/lib/feed_manager_spec.rb | 38 +++++++++++++++++++++++++++++++++++
 2 files changed, 39 insertions(+)

diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 26e5b78e8..8b7f20811 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -192,6 +192,7 @@ class FeedManager
     # also tagged with another followed hashtag or from a followed user
     scope = from_tag.statuses
                     .where(id: timeline_status_ids)
+                    .where.not(account: into_account)
                     .where.not(account: into_account.following)
                     .tagged_with_none(TagFollow.where(account: into_account).pluck(:tag_id))
 
diff --git a/spec/lib/feed_manager_spec.rb b/spec/lib/feed_manager_spec.rb
index 25edaada6..f4dd42f84 100644
--- a/spec/lib/feed_manager_spec.rb
+++ b/spec/lib/feed_manager_spec.rb
@@ -525,6 +525,44 @@ RSpec.describe FeedManager do
     end
   end
 
+  describe '#unmerge_tag_from_home' do
+    let(:receiver) { Fabricate(:account) }
+    let(:tag) { Fabricate(:tag) }
+
+    it 'leaves a tagged status' do
+      status = Fabricate(:status)
+      status.tags << tag
+      described_class.instance.push_to_home(receiver, status)
+
+      described_class.instance.unmerge_tag_from_home(tag, receiver)
+
+      expect(redis.zrange("feed:home:#{receiver.id}", 0, -1)).to_not include(status.id.to_s)
+    end
+
+    it 'remains a tagged status written by receiver\'s followee' do
+      followee = Fabricate(:account)
+      receiver.follow!(followee)
+
+      status = Fabricate(:status, account: followee)
+      status.tags << tag
+      described_class.instance.push_to_home(receiver, status)
+
+      described_class.instance.unmerge_tag_from_home(tag, receiver)
+
+      expect(redis.zrange("feed:home:#{receiver.id}", 0, -1)).to include(status.id.to_s)
+    end
+
+    it 'remains a tagged status written by receiver' do
+      status = Fabricate(:status, account: receiver)
+      status.tags << tag
+      described_class.instance.push_to_home(receiver, status)
+
+      described_class.instance.unmerge_tag_from_home(tag, receiver)
+
+      expect(redis.zrange("feed:home:#{receiver.id}", 0, -1)).to include(status.id.to_s)
+    end
+  end
+
   describe '#clear_from_home' do
     let(:account) { Fabricate(:account) }
     let(:followed_account) { Fabricate(:account) }

From 13205b54fd69f8cce1975aff8e784aa2f24e82ff Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 16 Oct 2023 15:24:14 +0200
Subject: [PATCH 007/117] Fix handling of `inLanguage` attribute in preview
 card processing (#27423)

---
 app/lib/link_details_extractor.rb       |  3 ++-
 spec/lib/link_details_extractor_spec.rb | 10 ++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb
index b95ec8051..a96612cab 100644
--- a/app/lib/link_details_extractor.rb
+++ b/app/lib/link_details_extractor.rb
@@ -36,7 +36,8 @@ class LinkDetailsExtractor
     end
 
     def language
-      json['inLanguage']
+      lang = json['inLanguage']
+      lang.is_a?(Hash) ? (lang['alternateName'] || lang['name']) : lang
     end
 
     def type
diff --git a/spec/lib/link_details_extractor_spec.rb b/spec/lib/link_details_extractor_spec.rb
index 599bc4e6d..8c485cef2 100644
--- a/spec/lib/link_details_extractor_spec.rb
+++ b/spec/lib/link_details_extractor_spec.rb
@@ -82,6 +82,10 @@ RSpec.describe LinkDetailsExtractor do
           'name' => 'Pet News',
           'url' => 'https://example.com',
         },
+        'inLanguage' => {
+          name: 'English',
+          alternateName: 'en',
+        },
       }.to_json
     end
 
@@ -115,6 +119,12 @@ RSpec.describe LinkDetailsExtractor do
           expect(subject.provider_name).to eq 'Pet News'
         end
       end
+
+      describe '#language' do
+        it 'returns the language from structured data' do
+          expect(subject.language).to eq 'en'
+        end
+      end
     end
 
     context 'when is wrapped in CDATA tags' do

From 700ae1f9183a01afe574687a91afb6e7a99825fc Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 26 Oct 2023 19:03:31 +0200
Subject: [PATCH 008/117] Fix report processing notice not mentioning the
 report number when performing a custom action (#27442)

---
 app/controllers/admin/account_actions_controller.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/controllers/admin/account_actions_controller.rb b/app/controllers/admin/account_actions_controller.rb
index e89404b60..e674bf55a 100644
--- a/app/controllers/admin/account_actions_controller.rb
+++ b/app/controllers/admin/account_actions_controller.rb
@@ -21,7 +21,7 @@ module Admin
       account_action.save!
 
       if account_action.with_report?
-        redirect_to admin_reports_path, notice: I18n.t('admin.reports.processed_msg', id: params[:report_id])
+        redirect_to admin_reports_path, notice: I18n.t('admin.reports.processed_msg', id: resource_params[:report_id])
       else
         redirect_to admin_account_path(@account.id)
       end

From bece853e3cc45a7b8b8b19b78b6f772c65e80b2a Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 27 Oct 2023 10:35:21 +0200
Subject: [PATCH 009/117] Fix error and incorrect URLs in
 `/api/v1/accounts/:id/featured_tags` for remote accounts (#27459)

---
 .../rest/featured_tag_serializer.rb           |  4 +-
 config/routes.rb                              |  2 +-
 .../accounts/featured_tags_controller_spec.rb | 23 ---------
 .../api/v1/accounts/featured_tags_spec.rb     | 50 +++++++++++++++++++
 4 files changed, 54 insertions(+), 25 deletions(-)
 delete mode 100644 spec/controllers/api/v1/accounts/featured_tags_controller_spec.rb
 create mode 100644 spec/requests/api/v1/accounts/featured_tags_spec.rb

diff --git a/app/serializers/rest/featured_tag_serializer.rb b/app/serializers/rest/featured_tag_serializer.rb
index c4b35ab03..c1ff4602a 100644
--- a/app/serializers/rest/featured_tag_serializer.rb
+++ b/app/serializers/rest/featured_tag_serializer.rb
@@ -10,7 +10,9 @@ class REST::FeaturedTagSerializer < ActiveModel::Serializer
   end
 
   def url
-    short_account_tag_url(object.account, object.tag)
+    # The path is hardcoded because we have to deal with both local and
+    # remote users, which are different routes
+    account_with_domain_url(object.account, "tagged/#{object.tag.to_param}")
   end
 
   def name
diff --git a/config/routes.rb b/config/routes.rb
index 7e2f1aabe..379262f4c 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -149,7 +149,7 @@ Rails.application.routes.draw do
     get '/@:account_username/:id/embed', to: 'statuses#embed', as: :embed_short_account_status
   end
 
-  get '/@:username_with_domain/(*any)', to: 'home#index', constraints: { username_with_domain: %r{([^/])+?} }, format: false
+  get '/@:username_with_domain/(*any)', to: 'home#index', constraints: { username_with_domain: %r{([^/])+?} }, as: :account_with_domain, format: false
   get '/settings', to: redirect('/settings/profile')
 
   draw(:settings)
diff --git a/spec/controllers/api/v1/accounts/featured_tags_controller_spec.rb b/spec/controllers/api/v1/accounts/featured_tags_controller_spec.rb
deleted file mode 100644
index 53ac1e2a7..000000000
--- a/spec/controllers/api/v1/accounts/featured_tags_controller_spec.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe Api::V1::Accounts::FeaturedTagsController do
-  render_views
-
-  let(:user)    { Fabricate(:user) }
-  let(:token)   { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:accounts') }
-  let(:account) { Fabricate(:account) }
-
-  before do
-    allow(controller).to receive(:doorkeeper_token) { token }
-  end
-
-  describe 'GET #index' do
-    it 'returns http success' do
-      get :index, params: { account_id: account.id, limit: 2 }
-
-      expect(response).to have_http_status(200)
-    end
-  end
-end
diff --git a/spec/requests/api/v1/accounts/featured_tags_spec.rb b/spec/requests/api/v1/accounts/featured_tags_spec.rb
new file mode 100644
index 000000000..bae7d448b
--- /dev/null
+++ b/spec/requests/api/v1/accounts/featured_tags_spec.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe 'account featured tags API' do
+  let(:user)     { Fabricate(:user) }
+  let(:token)    { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) }
+  let(:scopes)   { 'read:accounts' }
+  let(:headers)  { { 'Authorization' => "Bearer #{token.token}" } }
+  let(:account)  { Fabricate(:account) }
+
+  describe 'GET /api/v1/accounts/:id/featured_tags' do
+    subject do
+      get "/api/v1/accounts/#{account.id}/featured_tags", headers: headers
+    end
+
+    before do
+      account.featured_tags.create!(name: 'foo')
+      account.featured_tags.create!(name: 'bar')
+    end
+
+    it 'returns the expected tags', :aggregate_failures do
+      subject
+
+      expect(response).to have_http_status(200)
+      expect(body_as_json).to contain_exactly(a_hash_including({
+        name: 'bar',
+        url: "https://cb6e6126.ngrok.io/@#{account.username}/tagged/bar",
+      }), a_hash_including({
+        name: 'foo',
+        url: "https://cb6e6126.ngrok.io/@#{account.username}/tagged/foo",
+      }))
+    end
+
+    context 'when the account is remote' do
+      it 'returns the expected tags', :aggregate_failures do
+        subject
+
+        expect(response).to have_http_status(200)
+        expect(body_as_json).to contain_exactly(a_hash_including({
+          name: 'bar',
+          url: "https://cb6e6126.ngrok.io/@#{account.pretty_acct}/tagged/bar",
+        }), a_hash_including({
+          name: 'foo',
+          url: "https://cb6e6126.ngrok.io/@#{account.pretty_acct}/tagged/foo",
+        }))
+      end
+    end
+  end
+end

From c66ade7de80e72d1dfdd1833c04cb3be47754f71 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 20 Oct 2023 10:45:46 +0200
Subject: [PATCH 010/117] Fix processing LDSigned activities from actors with
 unknown public keys (#27474)

---
 app/lib/activitypub/linked_data_signature.rb  |  6 ++--
 .../activitypub/linked_data_signature_spec.rb | 34 +++++++++++++++++++
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/app/lib/activitypub/linked_data_signature.rb b/app/lib/activitypub/linked_data_signature.rb
index ea59879f3..faea63e8f 100644
--- a/app/lib/activitypub/linked_data_signature.rb
+++ b/app/lib/activitypub/linked_data_signature.rb
@@ -18,8 +18,8 @@ class ActivityPub::LinkedDataSignature
 
     return unless type == 'RsaSignature2017'
 
-    creator   = ActivityPub::TagManager.instance.uri_to_actor(creator_uri)
-    creator ||= ActivityPub::FetchRemoteKeyService.new.call(creator_uri, id: false)
+    creator = ActivityPub::TagManager.instance.uri_to_actor(creator_uri)
+    creator = ActivityPub::FetchRemoteKeyService.new.call(creator_uri, id: false) if creator&.public_key.blank?
 
     return if creator.nil?
 
@@ -28,6 +28,8 @@ class ActivityPub::LinkedDataSignature
     to_be_verified = options_hash + document_hash
 
     creator if creator.keypair.public_key.verify(OpenSSL::Digest.new('SHA256'), Base64.decode64(signature), to_be_verified)
+  rescue OpenSSL::PKey::RSAError
+    false
   end
 
   def sign!(creator, sign_with: nil)
diff --git a/spec/lib/activitypub/linked_data_signature_spec.rb b/spec/lib/activitypub/linked_data_signature_spec.rb
index 6a6ad1a70..03d0c5a87 100644
--- a/spec/lib/activitypub/linked_data_signature_spec.rb
+++ b/spec/lib/activitypub/linked_data_signature_spec.rb
@@ -38,6 +38,40 @@ RSpec.describe ActivityPub::LinkedDataSignature do
       end
     end
 
+    context 'when local account record is missing a public key' do
+      let(:raw_signature) do
+        {
+          'creator' => 'http://example.com/alice',
+          'created' => '2017-09-23T20:21:34Z',
+        }
+      end
+
+      let(:signature) { raw_signature.merge('type' => 'RsaSignature2017', 'signatureValue' => sign(sender, raw_signature, raw_json)) }
+
+      let(:service_stub) { instance_double(ActivityPub::FetchRemoteKeyService) }
+
+      before do
+        # Ensure signature is computed with the old key
+        signature
+
+        # Unset key
+        old_key = sender.public_key
+        sender.update!(private_key: '', public_key: '')
+
+        allow(ActivityPub::FetchRemoteKeyService).to receive(:new).and_return(service_stub)
+
+        allow(service_stub).to receive(:call).with('http://example.com/alice', id: false) do
+          sender.update!(public_key: old_key)
+          sender
+        end
+      end
+
+      it 'fetches key and returns creator' do
+        expect(subject.verify_actor!).to eq sender
+        expect(service_stub).to have_received(:call).with('http://example.com/alice', id: false).once
+      end
+    end
+
     context 'when signature is missing' do
       let(:signature) { nil }
 

From d5bc10b711dede8c84087de9240eea85c14ab7b0 Mon Sep 17 00:00:00 2001
From: Renaud Chaput <renchap@gmail.com>
Date: Thu, 19 Oct 2023 13:22:44 +0200
Subject: [PATCH 011/117] The `class` props should be `className` (#27462)

---
 .../mastodon/features/ui/components/link_footer.jsx           | 2 +-
 .../mastodon/features/ui/components/navigation_panel.jsx      | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/javascript/mastodon/features/ui/components/link_footer.jsx b/app/javascript/mastodon/features/ui/components/link_footer.jsx
index 9585df2ec..6b1555243 100644
--- a/app/javascript/mastodon/features/ui/components/link_footer.jsx
+++ b/app/javascript/mastodon/features/ui/components/link_footer.jsx
@@ -100,7 +100,7 @@ class LinkFooter extends PureComponent {
           {DividingCircle}
           <a href={source_url} rel='noopener noreferrer' target='_blank'><FormattedMessage id='footer.source_code' defaultMessage='View source code' /></a>
           {DividingCircle}
-          <span class='version'>v{version}</span>
+          <span className='version'>v{version}</span>
         </p>
       </div>
     );
diff --git a/app/javascript/mastodon/features/ui/components/navigation_panel.jsx b/app/javascript/mastodon/features/ui/components/navigation_panel.jsx
index 8006ca89a..22eee79c0 100644
--- a/app/javascript/mastodon/features/ui/components/navigation_panel.jsx
+++ b/app/javascript/mastodon/features/ui/components/navigation_panel.jsx
@@ -59,10 +59,10 @@ class NavigationPanel extends Component {
           <Link to='/' className='column-link column-link--logo'><WordmarkLogo /></Link>
 
           {transientSingleColumn ? (
-            <div class='switch-to-advanced'>
+            <div className='switch-to-advanced'>
               {intl.formatMessage(messages.openedInClassicInterface)}
               {" "}
-              <a href={`/deck${location.pathname}`} class='switch-to-advanced__toggle'>
+              <a href={`/deck${location.pathname}`} className='switch-to-advanced__toggle'>
                 {intl.formatMessage(messages.advancedInterface)}
               </a>
             </div>

From 4fc252354624d5b0fbafd86251204cc19edc525d Mon Sep 17 00:00:00 2001
From: Renaud Chaput <renchap@gmail.com>
Date: Thu, 19 Oct 2023 19:36:08 +0200
Subject: [PATCH 012/117] Do not display the navigation banner in the logo
 container (#27476)

---
 .../ui/components/navigation_panel.jsx        | 30 +++++++++++--------
 .../styles/mastodon/components.scss           |  1 +
 2 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/app/javascript/mastodon/features/ui/components/navigation_panel.jsx b/app/javascript/mastodon/features/ui/components/navigation_panel.jsx
index 22eee79c0..b16eb5e17 100644
--- a/app/javascript/mastodon/features/ui/components/navigation_panel.jsx
+++ b/app/javascript/mastodon/features/ui/components/navigation_panel.jsx
@@ -53,24 +53,30 @@ class NavigationPanel extends Component {
     const { intl } = this.props;
     const { signedIn, disabledAccountId } = this.context.identity;
 
+    let banner = undefined;
+
+    if(transientSingleColumn)
+      banner = (<div className='switch-to-advanced'>
+        {intl.formatMessage(messages.openedInClassicInterface)}
+        {" "}
+        <a href={`/deck${location.pathname}`} className='switch-to-advanced__toggle'>
+          {intl.formatMessage(messages.advancedInterface)}
+        </a>
+      </div>);
+
     return (
       <div className='navigation-panel'>
         <div className='navigation-panel__logo'>
           <Link to='/' className='column-link column-link--logo'><WordmarkLogo /></Link>
-
-          {transientSingleColumn ? (
-            <div className='switch-to-advanced'>
-              {intl.formatMessage(messages.openedInClassicInterface)}
-              {" "}
-              <a href={`/deck${location.pathname}`} className='switch-to-advanced__toggle'>
-                {intl.formatMessage(messages.advancedInterface)}
-              </a>
-            </div>
-          ) : (
-            <hr />
-          )}
+          {!banner && <hr />}
         </div>
 
+        {banner &&
+          <div class='navigation-panel__banner'>
+            {banner}
+          </div>
+        }
+
         {signedIn && (
           <>
             <ColumnLink transparent to='/home' icon='home' text={intl.formatMessage(messages.home)} />
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index 567397a07..8c88b7dd6 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -2466,6 +2466,7 @@ $ui-header-height: 55px;
 
     .navigation-panel__sign-in-banner,
     .navigation-panel__logo,
+    .navigation-panel__banner,
     .getting-started__trends {
       display: none;
     }

From 4c3870647414f4cbd05373b0767d79e89a0db9a2 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 25 Oct 2023 15:55:57 +0200
Subject: [PATCH 013/117] Fix batch attachment deletion when using OpenStack
 Swift (#27554)

---
 app/lib/attachment_batch.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/lib/attachment_batch.rb b/app/lib/attachment_batch.rb
index b75938bdd..78bd59316 100644
--- a/app/lib/attachment_batch.rb
+++ b/app/lib/attachment_batch.rb
@@ -75,7 +75,7 @@ class AttachmentBatch
             end
           when :fog
             logger.debug { "Deleting #{attachment.path(style)}" }
-            attachment.directory.files.new(key: attachment.path(style)).destroy
+            attachment.send(:directory).files.new(key: attachment.path(style)).destroy
           when :azure
             logger.debug { "Deleting #{attachment.path(style)}" }
             attachment.destroy

From de86e822f4f51d7e20a59d48b2c7e66431e374fe Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 26 Oct 2023 15:09:48 +0200
Subject: [PATCH 014/117] Fix error when trying to delete already-deleted file
 with OpenStack Swift (#27569)

---
 app/lib/attachment_batch.rb | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/app/lib/attachment_batch.rb b/app/lib/attachment_batch.rb
index 78bd59316..13a9da828 100644
--- a/app/lib/attachment_batch.rb
+++ b/app/lib/attachment_batch.rb
@@ -75,7 +75,12 @@ class AttachmentBatch
             end
           when :fog
             logger.debug { "Deleting #{attachment.path(style)}" }
-            attachment.send(:directory).files.new(key: attachment.path(style)).destroy
+
+            begin
+              attachment.send(:directory).files.new(key: attachment.path(style)).destroy
+            rescue Fog::Storage::OpenStack::NotFound
+              # Ignore failure to delete a file that has already been deleted
+            end
           when :azure
             logger.debug { "Deleting #{attachment.path(style)}" }
             attachment.destroy

From e6f4c91c5c7a28b5185f31c0dc5b58c242f4270f Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 27 Oct 2023 16:04:51 +0200
Subject: [PATCH 015/117] Fix hashtag matching pattern matching some URLs
 (#27584)

---
 app/models/tag.rb       | 2 +-
 spec/models/tag_spec.rb | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 413f6f500..8fab98fb5 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -35,7 +35,7 @@ class Tag < ApplicationRecord
   HASHTAG_LAST_SEQUENCE = '([[:word:]_]*[[:alpha:]][[:word:]_]*)'
   HASHTAG_NAME_PAT = "#{HASHTAG_FIRST_SEQUENCE}|#{HASHTAG_LAST_SEQUENCE}"
 
-  HASHTAG_RE = %r{(?<![=/)[:word]])#(#{HASHTAG_NAME_PAT})}i
+  HASHTAG_RE = %r{(?<![=/)\w])#(#{HASHTAG_NAME_PAT})}i
   HASHTAG_NAME_RE = /\A(#{HASHTAG_NAME_PAT})\z/i
   HASHTAG_INVALID_CHARS_RE = /[^[:alnum:]#{HASHTAG_SEPARATORS}]/
 
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 80b9c861f..6177b7a25 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -32,6 +32,10 @@ RSpec.describe Tag do
       expect(subject.match('https://en.wikipedia.org/wiki/Ghostbusters_(song)#Lawsuit')).to be_nil
     end
 
+    it 'does not match URLs with hashtag-like anchors after a numeral' do
+      expect(subject.match('https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111895#c4')).to be_nil
+    end
+
     it 'does not match URLs with hashtag-like anchors after an empty query parameter' do
       expect(subject.match('https://en.wikipedia.org/wiki/Ghostbusters_(song)?foo=#Lawsuit')).to be_nil
     end

From 81d7cfd54467477e408b94a5a7cea67b1b9b7518 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 30 Oct 2023 23:32:25 +0100
Subject: [PATCH 016/117] Fix posts from force-sensitized accounts being able
 to trend (#27620)

---
 app/models/trends/statuses.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/models/trends/statuses.rb b/app/models/trends/statuses.rb
index 427fa8e79..c47fb8427 100644
--- a/app/models/trends/statuses.rb
+++ b/app/models/trends/statuses.rb
@@ -106,7 +106,7 @@ class Trends::Statuses < Trends::Base
   private
 
   def eligible?(status)
-    status.public_visibility? && status.account.discoverable? && !status.account.silenced? && status.spoiler_text.blank? && !status.sensitive? && !status.reply? && valid_locale?(status.language)
+    status.public_visibility? && status.account.discoverable? && !status.account.silenced? && !status.account.sensitized? && status.spoiler_text.blank? && !status.sensitive? && !status.reply? && valid_locale?(status.language)
   end
 
   def calculate_scores(statuses, at_time)

From 54a07731d14ed3809375b31e2ce86e347afed271 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 2 Nov 2023 15:58:37 +0100
Subject: [PATCH 017/117] Fix posts from threads received out-of-order
 sometimes not being inserted into timelines (#27653)

---
 app/services/fan_out_on_write_service.rb     |   8 +-
 app/workers/thread_resolve_worker.rb         |   9 +-
 spec/lib/activitypub/activity/create_spec.rb | 103 +++++++++++++++++++
 3 files changed, 116 insertions(+), 4 deletions(-)

diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb
index 2554756a5..f2a79c9fc 100644
--- a/app/services/fan_out_on_write_service.rb
+++ b/app/services/fan_out_on_write_service.rb
@@ -8,6 +8,7 @@ class FanOutOnWriteService < BaseService
   # @param [Hash] options
   # @option options [Boolean] update
   # @option options [Array<Integer>] silenced_account_ids
+  # @option options [Boolean] skip_notifications
   def call(status, options = {})
     @status    = status
     @account   = status.account
@@ -37,8 +38,11 @@ class FanOutOnWriteService < BaseService
 
   def fan_out_to_local_recipients!
     deliver_to_self!
-    notify_mentioned_accounts!
-    notify_about_update! if update?
+
+    unless @options[:skip_notifications]
+      notify_mentioned_accounts!
+      notify_about_update! if update?
+    end
 
     case @status.visibility.to_sym
     when :public, :unlisted, :private
diff --git a/app/workers/thread_resolve_worker.rb b/app/workers/thread_resolve_worker.rb
index 3206c45f6..d4cefb3fd 100644
--- a/app/workers/thread_resolve_worker.rb
+++ b/app/workers/thread_resolve_worker.rb
@@ -7,13 +7,18 @@ class ThreadResolveWorker
   sidekiq_options queue: 'pull', retry: 3
 
   def perform(child_status_id, parent_url, options = {})
-    child_status  = Status.find(child_status_id)
-    parent_status = FetchRemoteStatusService.new.call(parent_url, **options.deep_symbolize_keys)
+    child_status = Status.find(child_status_id)
+    return if child_status.in_reply_to_id.present?
+
+    parent_status = ActivityPub::TagManager.instance.uri_to_resource(parent_url, Status)
+    parent_status ||= FetchRemoteStatusService.new.call(parent_url, **options.deep_symbolize_keys)
 
     return if parent_status.nil?
 
     child_status.thread = parent_status
     child_status.save!
+
+    DistributionWorker.perform_async(child_status_id, { 'skip_notifications' => true }) if child_status.within_realtime_window?
   rescue ActiveRecord::RecordNotFound
     true
   end
diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb
index f6c24754c..8425f2127 100644
--- a/spec/lib/activitypub/activity/create_spec.rb
+++ b/spec/lib/activitypub/activity/create_spec.rb
@@ -23,6 +23,109 @@ RSpec.describe ActivityPub::Activity::Create do
     stub_request(:get, 'http://example.com/emojib.png').to_return(body: attachment_fixture('emojo.png'), headers: { 'Content-Type' => 'application/octet-stream' })
   end
 
+  describe 'processing posts received out of order' do
+    let(:follower) { Fabricate(:account, username: 'bob') }
+
+    let(:object_json) do
+      {
+        id: [ActivityPub::TagManager.instance.uri_for(sender), 'post1'].join('/'),
+        type: 'Note',
+        to: [
+          'https://www.w3.org/ns/activitystreams#Public',
+          ActivityPub::TagManager.instance.uri_for(follower),
+        ],
+        content: '@bob lorem ipsum',
+        published: 1.hour.ago.utc.iso8601,
+        updated: 1.hour.ago.utc.iso8601,
+        tag: {
+          type: 'Mention',
+          href: ActivityPub::TagManager.instance.uri_for(follower),
+        },
+      }
+    end
+
+    let(:reply_json) do
+      {
+        id: [ActivityPub::TagManager.instance.uri_for(sender), 'reply'].join('/'),
+        type: 'Note',
+        inReplyTo: object_json[:id],
+        to: [
+          'https://www.w3.org/ns/activitystreams#Public',
+          ActivityPub::TagManager.instance.uri_for(follower),
+        ],
+        content: '@bob lorem ipsum',
+        published: Time.now.utc.iso8601,
+        updated: Time.now.utc.iso8601,
+        tag: {
+          type: 'Mention',
+          href: ActivityPub::TagManager.instance.uri_for(follower),
+        },
+      }
+    end
+
+    def activity_for_object(json)
+      {
+        '@context': 'https://www.w3.org/ns/activitystreams',
+        id: [json[:id], 'activity'].join('/'),
+        type: 'Create',
+        actor: ActivityPub::TagManager.instance.uri_for(sender),
+        object: json,
+      }.with_indifferent_access
+    end
+
+    before do
+      follower.follow!(sender)
+    end
+
+    around do |example|
+      Sidekiq::Testing.fake! do
+        example.run
+        Sidekiq::Worker.clear_all
+      end
+    end
+
+    it 'correctly processes posts and inserts them in timelines', :aggregate_failures do
+      # Simulate a temporary failure preventing from fetching the parent post
+      stub_request(:get, object_json[:id]).to_return(status: 500)
+
+      # When receiving the reply…
+      described_class.new(activity_for_object(reply_json), sender, delivery: true).perform
+
+      # NOTE: Refering explicitly to the workers is a bit awkward
+      DistributionWorker.drain
+      FeedInsertWorker.drain
+
+      # …it creates a status with an unknown parent
+      reply = Status.find_by(uri: reply_json[:id])
+      expect(reply.reply?).to be true
+      expect(reply.in_reply_to_id).to be_nil
+
+      # …and creates a notification
+      expect(LocalNotificationWorker.jobs.size).to eq 1
+
+      # …but does not insert it into timelines
+      expect(redis.zscore(FeedManager.instance.key(:home, follower.id), reply.id)).to be_nil
+
+      # When receiving the parent…
+      described_class.new(activity_for_object(object_json), sender, delivery: true).perform
+
+      Sidekiq::Worker.drain_all
+
+      # …it creates a status and insert it into timelines
+      parent = Status.find_by(uri: object_json[:id])
+      expect(parent.reply?).to be false
+      expect(parent.in_reply_to_id).to be_nil
+      expect(reply.reload.in_reply_to_id).to eq parent.id
+
+      # Check that the both statuses have been inserted into the home feed
+      expect(redis.zscore(FeedManager.instance.key(:home, follower.id), parent.id)).to be_within(0.1).of(parent.id.to_f)
+      expect(redis.zscore(FeedManager.instance.key(:home, follower.id), reply.id)).to be_within(0.1).of(reply.id.to_f)
+
+      # Creates two notifications
+      expect(Notification.count).to eq 2
+    end
+  end
+
   describe '#perform' do
     context 'when fetching' do
       subject { described_class.new(json, sender) }

From 1076a6cd62a80f1e331980e7c81174fa1a67f435 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 6 Nov 2023 10:28:14 +0100
Subject: [PATCH 018/117] Fix incoming status creation date not being
 restricted to standard ISO8601 (#27655)

---
 app/lib/activitypub/parser/status_parser.rb  |  3 +-
 spec/lib/activitypub/activity/create_spec.rb | 48 ++++++++++++++++++--
 2 files changed, 45 insertions(+), 6 deletions(-)

diff --git a/app/lib/activitypub/parser/status_parser.rb b/app/lib/activitypub/parser/status_parser.rb
index 3ba154d01..45f5fc5bf 100644
--- a/app/lib/activitypub/parser/status_parser.rb
+++ b/app/lib/activitypub/parser/status_parser.rb
@@ -53,7 +53,8 @@ class ActivityPub::Parser::StatusParser
   end
 
   def created_at
-    @object['published']&.to_datetime
+    datetime = @object['published']&.to_datetime
+    datetime if datetime.present? && (0..9999).cover?(datetime.year)
   rescue ArgumentError
     nil
   end
diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb
index 8425f2127..7594efd59 100644
--- a/spec/lib/activitypub/activity/create_spec.rb
+++ b/spec/lib/activitypub/activity/create_spec.rb
@@ -134,6 +134,46 @@ RSpec.describe ActivityPub::Activity::Create do
         subject.perform
       end
 
+      context 'when object publication date is below ISO8601 range' do
+        let(:object_json) do
+          {
+            id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join,
+            type: 'Note',
+            content: 'Lorem ipsum',
+            published: '-0977-11-03T08:31:22Z',
+          }
+        end
+
+        it 'creates status with a valid creation date', :aggregate_failures do
+          status = sender.statuses.first
+
+          expect(status).to_not be_nil
+          expect(status.text).to eq 'Lorem ipsum'
+
+          expect(status.created_at).to be_within(30).of(Time.now.utc)
+        end
+      end
+
+      context 'when object publication date is above ISO8601 range' do
+        let(:object_json) do
+          {
+            id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join,
+            type: 'Note',
+            content: 'Lorem ipsum',
+            published: '10000-11-03T08:31:22Z',
+          }
+        end
+
+        it 'creates status with a valid creation date', :aggregate_failures do
+          status = sender.statuses.first
+
+          expect(status).to_not be_nil
+          expect(status.text).to eq 'Lorem ipsum'
+
+          expect(status.created_at).to be_within(30).of(Time.now.utc)
+        end
+      end
+
       context 'when object has been edited' do
         let(:object_json) do
           {
@@ -145,18 +185,16 @@ RSpec.describe ActivityPub::Activity::Create do
           }
         end
 
-        it 'creates status' do
+        it 'creates status with appropriate creation and edition dates', :aggregate_failures do
           status = sender.statuses.first
 
           expect(status).to_not be_nil
           expect(status.text).to eq 'Lorem ipsum'
-        end
 
-        it 'marks status as edited' do
-          status = sender.statuses.first
+          expect(status.created_at).to eq '2022-01-22T15:00:00Z'.to_datetime
 
-          expect(status).to_not be_nil
           expect(status.edited?).to be true
+          expect(status.edited_at).to eq '2022-01-22T16:00:00Z'.to_datetime
         end
       end
 

From 8d02e58ff42d28f62a354c1ce8282924d54c455f Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 16 Nov 2023 14:43:02 +0100
Subject: [PATCH 019/117] Fix upper border radius of onboarding columns
 (#27890)

---
 app/javascript/styles/mastodon/components.scss | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index 8c88b7dd6..fbad5e2db 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -2239,8 +2239,7 @@ $ui-header-height: 55px;
 
   > .scrollable {
     background: $ui-base-color;
-    border-bottom-left-radius: 4px;
-    border-bottom-right-radius: 4px;
+    border-radius: 0 0 4px 4px;
   }
 }
 

From 252ea2fc67a8e95532c1887fc9f2842ddae189f3 Mon Sep 17 00:00:00 2001
From: Jonathan de Jong <jonathandejong02@gmail.com>
Date: Fri, 27 Oct 2023 16:55:00 +0200
Subject: [PATCH 020/117] Have `Follow` activities bypass availability (#27586)

Co-authored-by: Claire <claire.github-309c@sitedethib.com>
---
 app/services/follow_service.rb             | 2 +-
 app/workers/activitypub/delivery_worker.rb | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb
index feea40e3c..1aa0241fe 100644
--- a/app/services/follow_service.rb
+++ b/app/services/follow_service.rb
@@ -71,7 +71,7 @@ class FollowService < BaseService
     if @target_account.local?
       LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name, 'follow_request')
     elsif @target_account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url)
+      ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url, { 'bypass_availability' => true })
     end
 
     follow_request
diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb
index 7c1c14766..376c237a9 100644
--- a/app/workers/activitypub/delivery_worker.rb
+++ b/app/workers/activitypub/delivery_worker.rb
@@ -23,9 +23,10 @@ class ActivityPub::DeliveryWorker
   HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze
 
   def perform(json, source_account_id, inbox_url, options = {})
-    return unless DeliveryFailureTracker.available?(inbox_url)
-
     @options        = options.with_indifferent_access
+
+    return unless @options[:bypass_availability] || DeliveryFailureTracker.available?(inbox_url)
+
     @json           = json
     @source_account = Account.find(source_account_id)
     @inbox_url      = inbox_url

From e11100d7824258cafbd08b56945f340720a8029a Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 27 Nov 2023 14:25:54 +0100
Subject: [PATCH 021/117] Clamp dates when serializing to Elasticsearch API
 (#28081)

---
 app/chewy/accounts_index.rb                     |  4 +++-
 app/chewy/concerns/datetime_clamping_concern.rb | 14 ++++++++++++++
 app/chewy/public_statuses_index.rb              |  4 +++-
 app/chewy/statuses_index.rb                     |  4 +++-
 app/chewy/tags_index.rb                         |  4 +++-
 5 files changed, 26 insertions(+), 4 deletions(-)
 create mode 100644 app/chewy/concerns/datetime_clamping_concern.rb

diff --git a/app/chewy/accounts_index.rb b/app/chewy/accounts_index.rb
index 00db257ac..59f2f991f 100644
--- a/app/chewy/accounts_index.rb
+++ b/app/chewy/accounts_index.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class AccountsIndex < Chewy::Index
+  include DatetimeClampingConcern
+
   settings index: index_preset(refresh_interval: '30s'), analysis: {
     filter: {
       english_stop: {
@@ -60,7 +62,7 @@ class AccountsIndex < Chewy::Index
     field(:following_count, type: 'long')
     field(:followers_count, type: 'long')
     field(:properties, type: 'keyword', value: ->(account) { account.searchable_properties })
-    field(:last_status_at, type: 'date', value: ->(account) { account.last_status_at || account.created_at })
+    field(:last_status_at, type: 'date', value: ->(account) { clamp_date(account.last_status_at || account.created_at) })
     field(:display_name, type: 'text', analyzer: 'verbatim') { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
     field(:username, type: 'text', analyzer: 'verbatim', value: ->(account) { [account.username, account.domain].compact.join('@') }) { field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'verbatim' }
     field(:text, type: 'text', analyzer: 'verbatim', value: ->(account) { account.searchable_text }) { field :stemmed, type: 'text', analyzer: 'natural' }
diff --git a/app/chewy/concerns/datetime_clamping_concern.rb b/app/chewy/concerns/datetime_clamping_concern.rb
new file mode 100644
index 000000000..7f176b6e5
--- /dev/null
+++ b/app/chewy/concerns/datetime_clamping_concern.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module DatetimeClampingConcern
+  extend ActiveSupport::Concern
+
+  MIN_ISO8601_DATETIME = '0000-01-01T00:00:00Z'.to_datetime.freeze
+  MAX_ISO8601_DATETIME = '9999-12-31T23:59:59Z'.to_datetime.freeze
+
+  class_methods do
+    def clamp_date(datetime)
+      datetime.clamp(MIN_ISO8601_DATETIME, MAX_ISO8601_DATETIME)
+    end
+  end
+end
diff --git a/app/chewy/public_statuses_index.rb b/app/chewy/public_statuses_index.rb
index 4be204d4a..076f72e52 100644
--- a/app/chewy/public_statuses_index.rb
+++ b/app/chewy/public_statuses_index.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class PublicStatusesIndex < Chewy::Index
+  include DatetimeClampingConcern
+
   settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: {
     filter: {
       english_stop: {
@@ -62,6 +64,6 @@ class PublicStatusesIndex < Chewy::Index
     field(:tags, type: 'text', analyzer: 'hashtag', value: ->(status) { status.tags.map(&:display_name) })
     field(:language, type: 'keyword')
     field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties })
-    field(:created_at, type: 'date')
+    field(:created_at, type: 'date', value: ->(status) { clamp_date(status.created_at) })
   end
 end
diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb
index 6b25dc9df..c717de66f 100644
--- a/app/chewy/statuses_index.rb
+++ b/app/chewy/statuses_index.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class StatusesIndex < Chewy::Index
+  include DatetimeClampingConcern
+
   settings index: index_preset(refresh_interval: '30s', number_of_shards: 5), analysis: {
     filter: {
       english_stop: {
@@ -60,6 +62,6 @@ class StatusesIndex < Chewy::Index
     field(:searchable_by, type: 'long', value: ->(status) { status.searchable_by })
     field(:language, type: 'keyword')
     field(:properties, type: 'keyword', value: ->(status) { status.searchable_properties })
-    field(:created_at, type: 'date')
+    field(:created_at, type: 'date', value: ->(status) { clamp_date(status.created_at) })
   end
 end
diff --git a/app/chewy/tags_index.rb b/app/chewy/tags_index.rb
index 5b6349a96..c99218a47 100644
--- a/app/chewy/tags_index.rb
+++ b/app/chewy/tags_index.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class TagsIndex < Chewy::Index
+  include DatetimeClampingConcern
+
   settings index: index_preset(refresh_interval: '30s'), analysis: {
     analyzer: {
       content: {
@@ -42,6 +44,6 @@ class TagsIndex < Chewy::Index
     field(:name, type: 'text', analyzer: 'content', value: :display_name) { field(:edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content') }
     field(:reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? })
     field(:usage, type: 'long', value: ->(tag, crutches) { tag.history.aggregate(crutches.time_period).accounts })
-    field(:last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at })
+    field(:last_status_at, type: 'date', value: ->(tag) { clamp_date(tag.last_status_at || tag.created_at) })
   end
 end

From 09115731d617c556135ac465421645ed54e92ca9 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 24 Nov 2023 10:31:28 +0100
Subject: [PATCH 022/117] Change GIF max matrix size error to explicitly
 mention GIF files (#27927)

---
 app/models/concerns/attachmentable.rb | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb
index c0ee1bdce..4cdbdeb47 100644
--- a/app/models/concerns/attachmentable.rb
+++ b/app/models/concerns/attachmentable.rb
@@ -52,9 +52,13 @@ module Attachmentable
     return if attachment.blank? || !/image.*/.match?(attachment.content_type) || attachment.queued_for_write[:original].blank?
 
     width, height = FastImage.size(attachment.queued_for_write[:original].path)
-    matrix_limit  = attachment.content_type == 'image/gif' ? GIF_MATRIX_LIMIT : MAX_MATRIX_LIMIT
+    return unless width.present? && height.present?
 
-    raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported" if width.present? && height.present? && (width * height > matrix_limit)
+    if attachment.content_type == 'image/gif' && width * height > GIF_MATRIX_LIMIT
+      raise Mastodon::DimensionsValidationError, "#{width}x#{height} GIF files are not supported"
+    elsif width * height > MAX_MATRIX_LIMIT
+      raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported"
+    end
   end
 
   def appropriate_extension(attachment)

From 7b9496322fd726a70c6fec1eee77c8cd04f0fd9e Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 30 Nov 2023 12:45:54 +0100
Subject: [PATCH 023/117] Change dismissed banners to be stored server-side
 (#27055)

---
 .../components/dismissable_banner.tsx         | 25 ++++++++++++++++---
 app/javascript/mastodon/reducers/settings.js  |  9 +++++++
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/app/javascript/mastodon/components/dismissable_banner.tsx b/app/javascript/mastodon/components/dismissable_banner.tsx
index 04a28e3cb..698469140 100644
--- a/app/javascript/mastodon/components/dismissable_banner.tsx
+++ b/app/javascript/mastodon/components/dismissable_banner.tsx
@@ -1,9 +1,16 @@
+/* eslint-disable @typescript-eslint/no-unsafe-call,
+                  @typescript-eslint/no-unsafe-return,
+                  @typescript-eslint/no-unsafe-assignment,
+                  @typescript-eslint/no-unsafe-member-access
+                  -- the settings store is not yet typed */
 import type { PropsWithChildren } from 'react';
-import { useCallback, useState } from 'react';
+import { useCallback, useState, useEffect } from 'react';
 
 import { defineMessages, useIntl } from 'react-intl';
 
+import { changeSetting } from 'mastodon/actions/settings';
 import { bannerSettings } from 'mastodon/settings';
+import { useAppSelector, useAppDispatch } from 'mastodon/store';
 
 import { IconButton } from './icon_button';
 
@@ -19,13 +26,25 @@ export const DismissableBanner: React.FC<PropsWithChildren<Props>> = ({
   id,
   children,
 }) => {
-  const [visible, setVisible] = useState(!bannerSettings.get(id));
+  const dismissed = useAppSelector((state) =>
+    state.settings.getIn(['dismissed_banners', id], false),
+  );
+  const dispatch = useAppDispatch();
+
+  const [visible, setVisible] = useState(!bannerSettings.get(id) && !dismissed);
   const intl = useIntl();
 
   const handleDismiss = useCallback(() => {
     setVisible(false);
     bannerSettings.set(id, true);
-  }, [id]);
+    dispatch(changeSetting(['dismissed_banners', id], true));
+  }, [id, dispatch]);
+
+  useEffect(() => {
+    if (!visible && !dismissed) {
+      dispatch(changeSetting(['dismissed_banners', id], true));
+    }
+  }, [id, dispatch, visible, dismissed]);
 
   if (!visible) {
     return null;
diff --git a/app/javascript/mastodon/reducers/settings.js b/app/javascript/mastodon/reducers/settings.js
index 07d1bda0f..a605ecbb8 100644
--- a/app/javascript/mastodon/reducers/settings.js
+++ b/app/javascript/mastodon/reducers/settings.js
@@ -100,6 +100,15 @@ const initialState = ImmutableMap({
       body: '',
     }),
   }),
+
+  dismissed_banners: ImmutableMap({
+    'public_timeline': false,
+    'community_timeline': false,
+    'home.explore_prompt': false,
+    'explore/links': false,
+    'explore/statuses': false,
+    'explore/tags': false,
+  }),
 });
 
 const defaultColumns = fromJS([

From 4b8fe9df73cf27db68f3c92178e37eff9deb3bd3 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 27 Nov 2023 15:00:52 +0100
Subject: [PATCH 024/117] Bump version to v4.2.2

---
 CHANGELOG.md            | 27 +++++++++++++++++++++++++++
 SECURITY.md             |  2 +-
 docker-compose.yml      |  6 +++---
 lib/mastodon/version.rb |  2 +-
 4 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f9303f011..54bc2b403 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,33 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.2.2] - 2023-12-04
+
+### Changed
+
+- Change dismissed banners to be stored server-side ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27055))
+- Change GIF max matrix size error to explicitly mention GIF files ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27927))
+- Change `Follow` activities delivery to bypass availability check ([ShadowJonathan](https://github.com/mastodon/mastodon/pull/27586))
+- Change single-column navigation notice to be displayed outside of the logo container ([renchap](https://github.com/mastodon/mastodon/pull/27462), [renchap](https://github.com/mastodon/mastodon/pull/27476))
+- Change Content-Security-Policy to be tighter on media paths ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/26889))
+- Change post language code to include country code when relevant ([gunchleoc](https://github.com/mastodon/mastodon/pull/27099), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/27207))
+
+### Fixed
+
+- Fix upper border radius of onboarding columns ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27890))
+- Fix incoming status creation date not being restricted to standard ISO8601 ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27655), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/28081))
+- Fix some posts from threads received out-of-order sometimes not being inserted into timelines ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27653))
+- Fix posts from force-sensitized accounts being able to trend ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27620))
+- Fix error when trying to delete already-deleted file with OpenStack Swift ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27569))
+- Fix batch attachment deletion when using OpenStack Swift ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27554))
+- Fix processing LDSigned activities from actors with unknown public keys ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27474))
+- Fix error and incorrect URLs in `/api/v1/accounts/:id/featured_tags` for remote accounts ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27459))
+- Fix report processing notice not mentioning the report number when performing a custom action ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27442))
+- Fix handling of `inLanguage` attribute in preview card processing ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27423))
+- Fix own posts being removed from home timeline when unfollowing a used hashtag ([kmycode](https://github.com/mastodon/mastodon/pull/27391))
+- Fix some link anchors being recognized as hashtags ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27271), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/27584))
+- Fix format-dependent redirects being cached regardless of requested format ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27634))
+
 ## [4.2.1] - 2023-10-10
 
 ### Added
diff --git a/SECURITY.md b/SECURITY.md
index 3e13377db..954ff73a2 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -17,6 +17,6 @@ A "vulnerability in Mastodon" is a vulnerability in the code distributed through
 | ------- | ---------------- |
 | 4.2.x   | Yes              |
 | 4.1.x   | Yes              |
-| 4.0.x   | Until 2023-10-31 |
+| 4.0.x   | No               |
 | 3.5.x   | Until 2023-12-31 |
 | < 3.5   | No               |
diff --git a/docker-compose.yml b/docker-compose.yml
index 1a180b089..efb8e528e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -56,7 +56,7 @@ services:
 
   web:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.1
+    image: ghcr.io/mastodon/mastodon:v4.2.2
     restart: always
     env_file: .env.production
     command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@@ -77,7 +77,7 @@ services:
 
   streaming:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.1
+    image: ghcr.io/mastodon/mastodon:v4.2.2
     restart: always
     env_file: .env.production
     command: node ./streaming
@@ -95,7 +95,7 @@ services:
 
   sidekiq:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.1
+    image: ghcr.io/mastodon/mastodon:v4.2.2
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq
diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb
index 00538d07c..7d5578d82 100644
--- a/lib/mastodon/version.rb
+++ b/lib/mastodon/version.rb
@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      1
+      2
     end
 
     def default_prerelease

From 71b60b09f494ddd4670128cf252b6372a293a51e Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Tue, 5 Dec 2023 14:15:33 +0100
Subject: [PATCH 025/117] Update dependency json-ld to v3.3.1

---
 Gemfile.lock | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index a003cd18d..76925a2bf 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -148,6 +148,7 @@ GEM
       net-http-persistent (~> 4.0)
       nokogiri (~> 1, >= 1.10.8)
     base64 (0.1.1)
+    bcp47_spec (0.2.1)
     bcrypt (3.1.18)
     better_errors (2.10.1)
       erubi (>= 1.0.0)
@@ -377,19 +378,19 @@ GEM
     ipaddress (0.8.3)
     jmespath (1.6.2)
     json (2.6.3)
-    json-canonicalization (0.3.2)
+    json-canonicalization (1.0.0)
     json-jwt (1.15.3)
       activesupport (>= 4.2)
       aes_key_wrap
       bindata
       httpclient
-    json-ld (3.2.5)
+    json-ld (3.3.1)
       htmlentities (~> 4.3)
-      json-canonicalization (~> 0.3, >= 0.3.2)
+      json-canonicalization (~> 1.0)
       link_header (~> 0.0, >= 0.0.8)
       multi_json (~> 1.15)
       rack (>= 2.2, < 4)
-      rdf (~> 3.2, >= 3.2.10)
+      rdf (~> 3.3)
     json-ld-preloaded (3.2.2)
       json-ld (~> 3.2)
       rdf (~> 3.2)
@@ -593,7 +594,8 @@ GEM
       zeitwerk (~> 2.5)
     rainbow (3.1.1)
     rake (13.0.6)
-    rdf (3.2.11)
+    rdf (3.3.1)
+      bcp47_spec (~> 0.2)
       link_header (~> 0.0, >= 0.0.8)
     rdf-normalize (0.6.1)
       rdf (~> 3.2)

From 90371a4fc4da90c692e632b3231d804e310de916 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Tue, 5 Dec 2023 14:21:18 +0100
Subject: [PATCH 026/117] Bump version to v4.2.3

---
 CHANGELOG.md            | 6 ++++++
 docker-compose.yml      | 6 +++---
 lib/mastodon/version.rb | 2 +-
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 54bc2b403..5f970519e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.2.3] - 2023-12-05
+
+### Fixed
+
+- Fix dependency on `json-canonicalization` version that has been made unavailable since last release
+
 ## [4.2.2] - 2023-12-04
 
 ### Changed
diff --git a/docker-compose.yml b/docker-compose.yml
index efb8e528e..dda71cb09 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -56,7 +56,7 @@ services:
 
   web:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.2
+    image: ghcr.io/mastodon/mastodon:v4.2.3
     restart: always
     env_file: .env.production
     command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@@ -77,7 +77,7 @@ services:
 
   streaming:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.2
+    image: ghcr.io/mastodon/mastodon:v4.2.3
     restart: always
     env_file: .env.production
     command: node ./streaming
@@ -95,7 +95,7 @@ services:
 
   sidekiq:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.2
+    image: ghcr.io/mastodon/mastodon:v4.2.3
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq
diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb
index 7d5578d82..2e68df474 100644
--- a/lib/mastodon/version.rb
+++ b/lib/mastodon/version.rb
@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      2
+      3
     end
 
     def default_prerelease

From d7875adad242881876e316477b0610cbad466893 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Tue, 19 Dec 2023 11:27:37 +0100
Subject: [PATCH 027/117] Fix call to inefficient `delete_matched` cache method
 in domain blocks (#28367)

---
 .../api/v1/accounts/notes_controller.rb       |  2 +-
 .../api/v1/accounts/pins_controller.rb        |  2 +-
 .../v1/accounts/relationships_controller.rb   |  5 +-
 app/controllers/api/v1/accounts_controller.rb |  2 +-
 .../api/v1/follow_requests_controller.rb      |  4 +-
 app/controllers/relationships_controller.rb   |  2 +-
 app/models/account_domain_block.rb            | 10 +--
 app/models/concerns/account_interactions.rb   |  6 --
 app/models/concerns/relationship_cacheable.rb |  4 +-
 .../account_relationships_presenter.rb        | 63 ++++++++++++++-----
 .../account_relationships_presenter_spec.rb   | 61 +++++++++++++-----
 11 files changed, 104 insertions(+), 57 deletions(-)

diff --git a/app/controllers/api/v1/accounts/notes_controller.rb b/app/controllers/api/v1/accounts/notes_controller.rb
index 032e807d1..6d115631a 100644
--- a/app/controllers/api/v1/accounts/notes_controller.rb
+++ b/app/controllers/api/v1/accounts/notes_controller.rb
@@ -25,6 +25,6 @@ class Api::V1::Accounts::NotesController < Api::BaseController
   end
 
   def relationships_presenter
-    AccountRelationshipsPresenter.new([@account.id], current_user.account_id)
+    AccountRelationshipsPresenter.new([@account], current_user.account_id)
   end
 end
diff --git a/app/controllers/api/v1/accounts/pins_controller.rb b/app/controllers/api/v1/accounts/pins_controller.rb
index 73f845c61..0eb13c048 100644
--- a/app/controllers/api/v1/accounts/pins_controller.rb
+++ b/app/controllers/api/v1/accounts/pins_controller.rb
@@ -25,6 +25,6 @@ class Api::V1::Accounts::PinsController < Api::BaseController
   end
 
   def relationships_presenter
-    AccountRelationshipsPresenter.new([@account.id], current_user.account_id)
+    AccountRelationshipsPresenter.new([@account], current_user.account_id)
   end
 end
diff --git a/app/controllers/api/v1/accounts/relationships_controller.rb b/app/controllers/api/v1/accounts/relationships_controller.rb
index 503f85c97..038d6700f 100644
--- a/app/controllers/api/v1/accounts/relationships_controller.rb
+++ b/app/controllers/api/v1/accounts/relationships_controller.rb
@@ -5,11 +5,10 @@ class Api::V1::Accounts::RelationshipsController < Api::BaseController
   before_action :require_user!
 
   def index
-    accounts = Account.without_suspended.where(id: account_ids).select('id')
+    @accounts = Account.without_suspended.where(id: account_ids).select(:id, :domain).to_a
     # .where doesn't guarantee that our results are in the same order
     # we requested them, so return the "right" order to the requestor.
-    @accounts = accounts.index_by(&:id).values_at(*account_ids).compact
-    render json: @accounts, each_serializer: REST::RelationshipSerializer, relationships: relationships
+    render json: @accounts.index_by(&:id).values_at(*account_ids).compact, each_serializer: REST::RelationshipSerializer, relationships: relationships
   end
 
   private
diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb
index ddb94d5ca..321a57ac5 100644
--- a/app/controllers/api/v1/accounts_controller.rb
+++ b/app/controllers/api/v1/accounts_controller.rb
@@ -86,7 +86,7 @@ class Api::V1::AccountsController < Api::BaseController
   end
 
   def relationships(**options)
-    AccountRelationshipsPresenter.new([@account.id], current_user.account_id, **options)
+    AccountRelationshipsPresenter.new([@account], current_user.account_id, **options)
   end
 
   def account_params
diff --git a/app/controllers/api/v1/follow_requests_controller.rb b/app/controllers/api/v1/follow_requests_controller.rb
index 7c197ce6b..ee717ebbc 100644
--- a/app/controllers/api/v1/follow_requests_controller.rb
+++ b/app/controllers/api/v1/follow_requests_controller.rb
@@ -25,11 +25,11 @@ class Api::V1::FollowRequestsController < Api::BaseController
   private
 
   def account
-    Account.find(params[:id])
+    @account ||= Account.find(params[:id])
   end
 
   def relationships(**options)
-    AccountRelationshipsPresenter.new([params[:id]], current_user.account_id, **options)
+    AccountRelationshipsPresenter.new([account], current_user.account_id, **options)
   end
 
   def load_accounts
diff --git a/app/controllers/relationships_controller.rb b/app/controllers/relationships_controller.rb
index e87b5a656..dd794f319 100644
--- a/app/controllers/relationships_controller.rb
+++ b/app/controllers/relationships_controller.rb
@@ -33,7 +33,7 @@ class RelationshipsController < ApplicationController
   end
 
   def set_relationships
-    @relationships = AccountRelationshipsPresenter.new(@accounts.pluck(:id), current_user.account_id)
+    @relationships = AccountRelationshipsPresenter.new(@accounts, current_user.account_id)
   end
 
   def form_account_batch_params
diff --git a/app/models/account_domain_block.rb b/app/models/account_domain_block.rb
index af1e6a68d..db2e37184 100644
--- a/app/models/account_domain_block.rb
+++ b/app/models/account_domain_block.rb
@@ -18,16 +18,12 @@ class AccountDomainBlock < ApplicationRecord
   belongs_to :account
   validates :domain, presence: true, uniqueness: { scope: :account_id }, domain: true
 
-  after_commit :remove_blocking_cache
-  after_commit :remove_relationship_cache
+  after_commit :invalidate_domain_blocking_cache
 
   private
 
-  def remove_blocking_cache
+  def invalidate_domain_blocking_cache
     Rails.cache.delete("exclude_domains_for:#{account_id}")
-  end
-
-  def remove_relationship_cache
-    Rails.cache.delete_matched("relationship:#{account_id}:*")
+    Rails.cache.delete(['exclude_domains', account_id, domain])
   end
 end
diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb
index 3c64ebd9f..2de15031f 100644
--- a/app/models/concerns/account_interactions.rb
+++ b/app/models/concerns/account_interactions.rb
@@ -60,12 +60,6 @@ module AccountInteractions
       end
     end
 
-    def domain_blocking_map(target_account_ids, account_id)
-      accounts_map    = Account.where(id: target_account_ids).select('id, domain').each_with_object({}) { |a, h| h[a.id] = a.domain }
-      blocked_domains = domain_blocking_map_by_domain(accounts_map.values.compact, account_id)
-      accounts_map.reduce({}) { |h, (id, domain)| h.merge(id => blocked_domains[domain]) }
-    end
-
     def domain_blocking_map_by_domain(target_domains, account_id)
       follow_mapping(AccountDomainBlock.where(account_id: account_id, domain: target_domains), :domain)
     end
diff --git a/app/models/concerns/relationship_cacheable.rb b/app/models/concerns/relationship_cacheable.rb
index 0d9359f7e..c32a8d62c 100644
--- a/app/models/concerns/relationship_cacheable.rb
+++ b/app/models/concerns/relationship_cacheable.rb
@@ -10,7 +10,7 @@ module RelationshipCacheable
   private
 
   def remove_relationship_cache
-    Rails.cache.delete("relationship:#{account_id}:#{target_account_id}")
-    Rails.cache.delete("relationship:#{target_account_id}:#{account_id}")
+    Rails.cache.delete(['relationship', account_id, target_account_id])
+    Rails.cache.delete(['relationship', target_account_id, account_id])
   end
 end
diff --git a/app/presenters/account_relationships_presenter.rb b/app/presenters/account_relationships_presenter.rb
index 5d2b5435d..8482ef54d 100644
--- a/app/presenters/account_relationships_presenter.rb
+++ b/app/presenters/account_relationships_presenter.rb
@@ -5,8 +5,9 @@ class AccountRelationshipsPresenter
               :muting, :requested, :requested_by, :domain_blocking,
               :endorsed, :account_note
 
-  def initialize(account_ids, current_account_id, **options)
-    @account_ids        = account_ids.map { |a| a.is_a?(Account) ? a.id : a.to_i }
+  def initialize(accounts, current_account_id, **options)
+    @accounts = accounts.to_a
+    @account_ids        = @accounts.pluck(:id)
     @current_account_id = current_account_id
 
     @following       = cached[:following].merge(Account.following_map(@uncached_account_ids, @current_account_id))
@@ -16,10 +17,11 @@ class AccountRelationshipsPresenter
     @muting          = cached[:muting].merge(Account.muting_map(@uncached_account_ids, @current_account_id))
     @requested       = cached[:requested].merge(Account.requested_map(@uncached_account_ids, @current_account_id))
     @requested_by    = cached[:requested_by].merge(Account.requested_by_map(@uncached_account_ids, @current_account_id))
-    @domain_blocking = cached[:domain_blocking].merge(Account.domain_blocking_map(@uncached_account_ids, @current_account_id))
     @endorsed        = cached[:endorsed].merge(Account.endorsed_map(@uncached_account_ids, @current_account_id))
     @account_note    = cached[:account_note].merge(Account.account_note_map(@uncached_account_ids, @current_account_id))
 
+    @domain_blocking = domain_blocking_map
+
     cache_uncached!
 
     @following.merge!(options[:following_map] || {})
@@ -36,6 +38,31 @@ class AccountRelationshipsPresenter
 
   private
 
+  def domain_blocking_map
+    target_domains = @accounts.pluck(:domain).compact.uniq
+    blocks_by_domain = {}
+
+    # Fetch from cache
+    cache_keys = target_domains.map { |domain| domain_cache_key(domain) }
+    Rails.cache.read_multi(*cache_keys).each do |key, blocking|
+      blocks_by_domain[key.last] = blocking
+    end
+
+    uncached_domains = target_domains - blocks_by_domain.keys
+
+    # Read uncached values from database
+    AccountDomainBlock.where(account_id: @current_account_id, domain: uncached_domains).pluck(:domain).each do |domain|
+      blocks_by_domain[domain] = true
+    end
+
+    # Write database reads to cache
+    to_cache = uncached_domains.to_h { |domain| [domain_cache_key(domain), blocks_by_domain[domain]] }
+    Rails.cache.write_multi(to_cache, expires_in: 1.day)
+
+    # Return formatted value
+    @accounts.each_with_object({}) { |account, h| h[account.id] = blocks_by_domain[account.domain] }
+  end
+
   def cached
     return @cached if defined?(@cached)
 
@@ -47,28 +74,23 @@ class AccountRelationshipsPresenter
       muting: {},
       requested: {},
       requested_by: {},
-      domain_blocking: {},
       endorsed: {},
       account_note: {},
     }
 
-    @uncached_account_ids = []
+    @uncached_account_ids = @account_ids.uniq
 
-    @account_ids.each do |account_id|
-      maps_for_account = Rails.cache.read("relationship:#{@current_account_id}:#{account_id}")
-
-      if maps_for_account.is_a?(Hash)
-        @cached.deep_merge!(maps_for_account)
-      else
-        @uncached_account_ids << account_id
-      end
+    cache_ids = @account_ids.map { |account_id| relationship_cache_key(account_id) }
+    Rails.cache.read_multi(*cache_ids).each do |key, maps_for_account|
+      @cached.deep_merge!(maps_for_account)
+      @uncached_account_ids.delete(key.last)
     end
 
     @cached
   end
 
   def cache_uncached!
-    @uncached_account_ids.each do |account_id|
+    to_cache = @uncached_account_ids.to_h do |account_id|
       maps_for_account = {
         following: { account_id => following[account_id] },
         followed_by: { account_id => followed_by[account_id] },
@@ -77,12 +99,21 @@ class AccountRelationshipsPresenter
         muting: { account_id => muting[account_id] },
         requested: { account_id => requested[account_id] },
         requested_by: { account_id => requested_by[account_id] },
-        domain_blocking: { account_id => domain_blocking[account_id] },
         endorsed: { account_id => endorsed[account_id] },
         account_note: { account_id => account_note[account_id] },
       }
 
-      Rails.cache.write("relationship:#{@current_account_id}:#{account_id}", maps_for_account, expires_in: 1.day)
+      [relationship_cache_key(account_id), maps_for_account]
     end
+
+    Rails.cache.write_multi(to_cache, expires_in: 1.day)
+  end
+
+  def domain_cache_key(domain)
+    ['exclude_domains', @current_account_id, domain]
+  end
+
+  def relationship_cache_key(account_id)
+    ['relationship', @current_account_id, account_id]
   end
 end
diff --git a/spec/presenters/account_relationships_presenter_spec.rb b/spec/presenters/account_relationships_presenter_spec.rb
index 5c2ba54e0..282cae4f0 100644
--- a/spec/presenters/account_relationships_presenter_spec.rb
+++ b/spec/presenters/account_relationships_presenter_spec.rb
@@ -5,30 +5,57 @@ require 'rails_helper'
 RSpec.describe AccountRelationshipsPresenter do
   describe '.initialize' do
     before do
-      allow(Account).to receive(:following_map).with(account_ids, current_account_id).and_return(default_map)
-      allow(Account).to receive(:followed_by_map).with(account_ids, current_account_id).and_return(default_map)
-      allow(Account).to receive(:blocking_map).with(account_ids, current_account_id).and_return(default_map)
-      allow(Account).to receive(:muting_map).with(account_ids, current_account_id).and_return(default_map)
-      allow(Account).to receive(:requested_map).with(account_ids, current_account_id).and_return(default_map)
-      allow(Account).to receive(:requested_by_map).with(account_ids, current_account_id).and_return(default_map)
-      allow(Account).to receive(:domain_blocking_map).with(account_ids, current_account_id).and_return(default_map)
+      allow(Account).to receive(:following_map).with(accounts.pluck(:id), current_account_id).and_return(default_map)
+      allow(Account).to receive(:followed_by_map).with(accounts.pluck(:id), current_account_id).and_return(default_map)
+      allow(Account).to receive(:blocking_map).with(accounts.pluck(:id), current_account_id).and_return(default_map)
+      allow(Account).to receive(:muting_map).with(accounts.pluck(:id), current_account_id).and_return(default_map)
+      allow(Account).to receive(:requested_map).with(accounts.pluck(:id), current_account_id).and_return(default_map)
+      allow(Account).to receive(:requested_by_map).with(accounts.pluck(:id), current_account_id).and_return(default_map)
     end
 
-    let(:presenter)          { described_class.new(account_ids, current_account_id, **options) }
+    let(:presenter)          { described_class.new(accounts, current_account_id, **options) }
     let(:current_account_id) { Fabricate(:account).id }
-    let(:account_ids)        { [Fabricate(:account).id] }
-    let(:default_map)        { { 1 => true } }
+    let(:accounts)           { [Fabricate(:account)] }
+    let(:default_map)        { { accounts[0].id => true } }
 
     context 'when options are not set' do
       let(:options) { {} }
 
       it 'sets default maps' do
-        expect(presenter.following).to       eq default_map
-        expect(presenter.followed_by).to     eq default_map
-        expect(presenter.blocking).to        eq default_map
-        expect(presenter.muting).to          eq default_map
-        expect(presenter.requested).to       eq default_map
-        expect(presenter.domain_blocking).to eq default_map
+        expect(presenter).to have_attributes(
+          following: default_map,
+          followed_by: default_map,
+          blocking: default_map,
+          muting: default_map,
+          requested: default_map,
+          domain_blocking: { accounts[0].id => nil }
+        )
+      end
+    end
+
+    context 'with a warm cache' do
+      let(:options) { {} }
+
+      before do
+        described_class.new(accounts, current_account_id, **options)
+
+        allow(Account).to receive(:following_map).with([], current_account_id).and_return({})
+        allow(Account).to receive(:followed_by_map).with([], current_account_id).and_return({})
+        allow(Account).to receive(:blocking_map).with([], current_account_id).and_return({})
+        allow(Account).to receive(:muting_map).with([], current_account_id).and_return({})
+        allow(Account).to receive(:requested_map).with([], current_account_id).and_return({})
+        allow(Account).to receive(:requested_by_map).with([], current_account_id).and_return({})
+      end
+
+      it 'sets returns expected values' do
+        expect(presenter).to have_attributes(
+          following: default_map,
+          followed_by: default_map,
+          blocking: default_map,
+          muting: default_map,
+          requested: default_map,
+          domain_blocking: { accounts[0].id => nil }
+        )
       end
     end
 
@@ -84,7 +111,7 @@ RSpec.describe AccountRelationshipsPresenter do
       let(:options) { { domain_blocking_map: { 7 => true } } }
 
       it 'sets @domain_blocking merged with default_map and options[:domain_blocking_map]' do
-        expect(presenter.domain_blocking).to eq default_map.merge(options[:domain_blocking_map])
+        expect(presenter.domain_blocking).to eq({ accounts[0].id => nil }.merge(options[:domain_blocking_map]))
       end
     end
   end

From 279be076794c21304aee2a1c1469a6f39b8fbe50 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Tue, 12 Dec 2023 09:29:46 +0100
Subject: [PATCH 028/117] Fix `LinkCrawlWorker` error when encountering empty
 OEmbed response (#28268)

---
 app/services/fetch_oembed_service.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/services/fetch_oembed_service.rb b/app/services/fetch_oembed_service.rb
index 1ae592238..dc84b16b6 100644
--- a/app/services/fetch_oembed_service.rb
+++ b/app/services/fetch_oembed_service.rb
@@ -100,7 +100,7 @@ class FetchOEmbedService
   end
 
   def validate(oembed)
-    oembed if oembed[:version].to_s == '1.0' && oembed[:type].present?
+    oembed if oembed.present? && oembed[:version].to_s == '1.0' && oembed[:type].present?
   end
 
   def html

From 7abc61887f30d7f1d1145e5be104a098b4b14f87 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Tue, 12 Dec 2023 15:31:37 +0100
Subject: [PATCH 029/117] Fix empty column explainer getting cropped under
 certain conditions (#28337)

---
 app/javascript/styles/mastodon/components.scss | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index fbad5e2db..7d152fbf7 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -4493,11 +4493,6 @@ a.status-card {
   align-items: center;
   justify-content: center;
 
-  @supports (display: grid) {
-    // hack to fix Chrome <57
-    contain: strict;
-  }
-
   & > span {
     max-width: 500px;
   }

From a12b7551cfa4a95ea8ca7ab1ae1a3d96d1a3ad24 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 13 Dec 2023 08:47:32 +0100
Subject: [PATCH 030/117] Fix N+1s because of association preloaders not
 actually getting called (#28339)

---
 app/lib/inline_renderer.rb                    | 4 ++--
 app/models/announcement.rb                    | 4 ++--
 app/models/concerns/account_search.rb         | 4 ++--
 app/models/notification.rb                    | 2 +-
 app/serializers/initial_state_serializer.rb   | 4 ++--
 app/services/account_search_service.rb        | 2 +-
 app/services/batched_remove_status_service.rb | 4 ++--
 7 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/app/lib/inline_renderer.rb b/app/lib/inline_renderer.rb
index eda3da2c2..0aebb13fc 100644
--- a/app/lib/inline_renderer.rb
+++ b/app/lib/inline_renderer.rb
@@ -37,13 +37,13 @@ class InlineRenderer
   private
 
   def preload_associations_for_status
-    ActiveRecord::Associations::Preloader.new(records: @object, associations: {
+    ActiveRecord::Associations::Preloader.new(records: [@object], associations: {
       active_mentions: :account,
 
       reblog: {
         active_mentions: :account,
       },
-    })
+    }).call
   end
 
   def current_user
diff --git a/app/models/announcement.rb b/app/models/announcement.rb
index c5d6dd62e..2cd7c1d5e 100644
--- a/app/models/announcement.rb
+++ b/app/models/announcement.rb
@@ -78,9 +78,9 @@ class Announcement < ApplicationRecord
       else
         scope.select("name, custom_emoji_id, count(*) as count, exists(select 1 from announcement_reactions r where r.account_id = #{account.id} and r.announcement_id = announcement_reactions.announcement_id and r.name = announcement_reactions.name) as me")
       end
-    end
+    end.to_a
 
-    ActiveRecord::Associations::Preloader.new(records: records, associations: :custom_emoji)
+    ActiveRecord::Associations::Preloader.new(records: records, associations: :custom_emoji).call
     records
   end
 
diff --git a/app/models/concerns/account_search.rb b/app/models/concerns/account_search.rb
index 9f7720f11..5bd624025 100644
--- a/app/models/concerns/account_search.rb
+++ b/app/models/concerns/account_search.rb
@@ -122,7 +122,7 @@ module AccountSearch
       tsquery = generate_query_for_search(terms)
 
       find_by_sql([BASIC_SEARCH_SQL, { limit: limit, offset: offset, tsquery: tsquery }]).tap do |records|
-        ActiveRecord::Associations::Preloader.new(records: records, associations: :account_stat)
+        ActiveRecord::Associations::Preloader.new(records: records, associations: [:account_stat, { user: :role }]).call
       end
     end
 
@@ -131,7 +131,7 @@ module AccountSearch
       sql_template = following ? ADVANCED_SEARCH_WITH_FOLLOWING : ADVANCED_SEARCH_WITHOUT_FOLLOWING
 
       find_by_sql([sql_template, { id: account.id, limit: limit, offset: offset, tsquery: tsquery }]).tap do |records|
-        ActiveRecord::Associations::Preloader.new(records: records, associations: :account_stat)
+        ActiveRecord::Associations::Preloader.new(records: records, associations: [:account_stat, { user: :role }]).call
       end
     end
 
diff --git a/app/models/notification.rb b/app/models/notification.rb
index 60f834a63..54212d675 100644
--- a/app/models/notification.rb
+++ b/app/models/notification.rb
@@ -111,7 +111,7 @@ class Notification < ApplicationRecord
 
         # Instead of using the usual `includes`, manually preload each type.
         # If polymorphic associations are loaded with the usual `includes`, other types of associations will be loaded more.
-        ActiveRecord::Associations::Preloader.new(records: grouped_notifications, associations: associations)
+        ActiveRecord::Associations::Preloader.new(records: grouped_notifications, associations: associations).call
       end
 
       unique_target_statuses = notifications.filter_map(&:target_status).uniq
diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb
index b707d6fcb..d1c03a413 100644
--- a/app/serializers/initial_state_serializer.rb
+++ b/app/serializers/initial_state_serializer.rb
@@ -86,8 +86,8 @@ class InitialStateSerializer < ActiveModel::Serializer
 
     ActiveRecord::Associations::Preloader.new(
       records: [object.current_account, object.admin, object.owner, object.disabled_account, object.moved_to_account].compact,
-      associations: [:account_stat, :user, { moved_to_account: [:account_stat, :user] }]
-    )
+      associations: [:account_stat, { user: :role, moved_to_account: [:account_stat, { user: :role }] }]
+    ).call
 
     store[object.current_account.id.to_s]  = ActiveModelSerializers::SerializableResource.new(object.current_account, serializer: REST::AccountSerializer) if object.current_account
     store[object.admin.id.to_s]            = ActiveModelSerializers::SerializableResource.new(object.admin, serializer: REST::AccountSerializer) if object.admin
diff --git a/app/services/account_search_service.rb b/app/services/account_search_service.rb
index b437ff475..7b85b09d8 100644
--- a/app/services/account_search_service.rb
+++ b/app/services/account_search_service.rb
@@ -218,7 +218,7 @@ class AccountSearchService < BaseService
 
     records = query_builder.build.limit(limit_for_non_exact_results).offset(offset).objects.compact
 
-    ActiveRecord::Associations::Preloader.new(records: records, associations: :account_stat)
+    ActiveRecord::Associations::Preloader.new(records: records, associations: [:account_stat, { user: :role }]).call
 
     records
   rescue Faraday::ConnectionFailed, Parslet::ParseFailed
diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb
index c54cc1d35..de4ee16e9 100644
--- a/app/services/batched_remove_status_service.rb
+++ b/app/services/batched_remove_status_service.rb
@@ -11,7 +11,7 @@ class BatchedRemoveStatusService < BaseService
     ActiveRecord::Associations::Preloader.new(
       records: statuses,
       associations: options[:skip_side_effects] ? :reblogs : [:account, :tags, reblogs: :account]
-    )
+    ).call
 
     statuses_and_reblogs = statuses.flat_map { |status| [status] + status.reblogs }
 
@@ -23,7 +23,7 @@ class BatchedRemoveStatusService < BaseService
     ActiveRecord::Associations::Preloader.new(
       records: statuses_with_account_conversations,
       associations: [mentions: :account]
-    )
+    ).call
 
     statuses_with_account_conversations.each(&:unlink_from_conversations!)
 

From 0a01bc01d2fc8967f75cd24ab3347d13a9d7f211 Mon Sep 17 00:00:00 2001
From: MitarashiDango <mitarashi_dango@mail.matcha-soft.com>
Date: Sat, 13 Jan 2024 00:58:28 +0900
Subject: [PATCH 031/117] Fix Undo Announce activity is not sent, when not
 followed by the reblogged post author (#18482)

Co-authored-by: Claire <claire.github-309c@sitedethib.com>
---
 app/lib/status_reach_finder.rb              | 34 ++++++++++-----------
 app/services/reblog_service.rb              |  6 +---
 spec/lib/activitypub/tag_manager_spec.rb    |  8 +++++
 spec/services/reblog_service_spec.rb        |  4 ---
 spec/services/remove_status_service_spec.rb | 18 +++++++++++
 5 files changed, 44 insertions(+), 26 deletions(-)

diff --git a/app/lib/status_reach_finder.rb b/app/lib/status_reach_finder.rb
index 36fb0e80f..17e42e3ec 100644
--- a/app/lib/status_reach_finder.rb
+++ b/app/lib/status_reach_finder.rb
@@ -16,28 +16,28 @@ class StatusReachFinder
   private
 
   def reached_account_inboxes
+    Account.where(id: reached_account_ids).inboxes
+  end
+
+  def reached_account_ids
     # When the status is a reblog, there are no interactions with it
     # directly, we assume all interactions are with the original one
 
     if @status.reblog?
-      []
+      [reblog_of_account_id]
     else
-      Account.where(id: reached_account_ids).inboxes
-    end
-  end
-
-  def reached_account_ids
-    [
-      replied_to_account_id,
-      reblog_of_account_id,
-      mentioned_account_ids,
-      reblogs_account_ids,
-      favourites_account_ids,
-      replies_account_ids,
-    ].tap do |arr|
-      arr.flatten!
-      arr.compact!
-      arr.uniq!
+      [
+        replied_to_account_id,
+        reblog_of_account_id,
+        mentioned_account_ids,
+        reblogs_account_ids,
+        favourites_account_ids,
+        replies_account_ids,
+      ].tap do |arr|
+        arr.flatten!
+        arr.compact!
+        arr.uniq!
+      end
     end
   end
 
diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb
index 6ec094474..c02674612 100644
--- a/app/services/reblog_service.rb
+++ b/app/services/reblog_service.rb
@@ -43,11 +43,7 @@ class ReblogService < BaseService
   def create_notification(reblog)
     reblogged_status = reblog.reblog
 
-    if reblogged_status.account.local?
-      LocalNotificationWorker.perform_async(reblogged_status.account_id, reblog.id, reblog.class.name, 'reblog')
-    elsif reblogged_status.account.activitypub? && !reblogged_status.account.following?(reblog.account)
-      ActivityPub::DeliveryWorker.perform_async(build_json(reblog), reblog.account_id, reblogged_status.account.inbox_url)
-    end
+    LocalNotificationWorker.perform_async(reblogged_status.account_id, reblog.id, reblog.class.name, 'reblog') if reblogged_status.account.local?
   end
 
   def bump_potential_friendship(account, reblog)
diff --git a/spec/lib/activitypub/tag_manager_spec.rb b/spec/lib/activitypub/tag_manager_spec.rb
index 2bff125a6..55e9b4bb5 100644
--- a/spec/lib/activitypub/tag_manager_spec.rb
+++ b/spec/lib/activitypub/tag_manager_spec.rb
@@ -112,6 +112,14 @@ RSpec.describe ActivityPub::TagManager do
       expect(subject.cc(status)).to include(subject.uri_for(foo))
       expect(subject.cc(status)).to_not include(subject.uri_for(alice))
     end
+
+    it 'returns poster of reblogged post, if reblog' do
+      bob    = Fabricate(:account, username: 'bob', domain: 'example.com', inbox_url: 'http://example.com/bob')
+      alice  = Fabricate(:account, username: 'alice')
+      status = Fabricate(:status, visibility: :public, account: bob)
+      reblog = Fabricate(:status, visibility: :public, account: alice, reblog: status)
+      expect(subject.cc(reblog)).to include(subject.uri_for(bob))
+    end
   end
 
   describe '#local_uri?' do
diff --git a/spec/services/reblog_service_spec.rb b/spec/services/reblog_service_spec.rb
index 7b85e37ed..357b315af 100644
--- a/spec/services/reblog_service_spec.rb
+++ b/spec/services/reblog_service_spec.rb
@@ -86,9 +86,5 @@ RSpec.describe ReblogService, type: :service do
     it 'distributes to followers' do
       expect(ActivityPub::DistributionWorker).to have_received(:perform_async)
     end
-
-    it 'sends an announce activity to the author' do
-      expect(a_request(:post, bob.inbox_url)).to have_been_made.once
-    end
   end
 end
diff --git a/spec/services/remove_status_service_spec.rb b/spec/services/remove_status_service_spec.rb
index c19b4fac1..7e81f3f6a 100644
--- a/spec/services/remove_status_service_spec.rb
+++ b/spec/services/remove_status_service_spec.rb
@@ -110,4 +110,22 @@ RSpec.describe RemoveStatusService, type: :service do
              )).to have_been_made.once
     end
   end
+
+  context 'when removed status is a reblog of a non-follower' do
+    let!(:original_status) { Fabricate(:status, account: bill, text: 'Hello ThisIsASecret', visibility: :public) }
+    let!(:status) { ReblogService.new.call(alice, original_status) }
+
+    it 'sends Undo activity to followers' do
+      subject.call(status)
+      expect(a_request(:post, bill.inbox_url).with(
+               body: hash_including({
+                 'type' => 'Undo',
+                 'object' => hash_including({
+                   'type' => 'Announce',
+                   'object' => ActivityPub::TagManager.instance.uri_for(original_status),
+                 }),
+               })
+             )).to have_been_made.once
+    end
+  end
 end

From e8c575414233df9d8c1a44140f91aa8e03bdc161 Mon Sep 17 00:00:00 2001
From: gunchleoc <fios@foramnagaidhlig.net>
Date: Mon, 25 Sep 2023 12:59:37 +0200
Subject: [PATCH 032/117] Fix line wrapping of language selection button with
 long locale codes (#27100)

---
 app/javascript/styles/mastodon/components.scss | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index 7d152fbf7..5f773c1cf 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -284,6 +284,7 @@
   font-size: 11px;
   padding: 0 3px;
   line-height: 27px;
+  white-space: nowrap;
 
   &:hover,
   &:active,

From dfc8fcc6f0a266c72c1e37ee7b2982a5b9ee57f0 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 25 Sep 2023 15:06:57 +0200
Subject: [PATCH 033/117] Fix width of large text icon buttons (#27127)

---
 .../mastodon/features/compose/components/text_icon_button.jsx   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/javascript/mastodon/features/compose/components/text_icon_button.jsx b/app/javascript/mastodon/features/compose/components/text_icon_button.jsx
index 46b5d7fad..166d022b8 100644
--- a/app/javascript/mastodon/features/compose/components/text_icon_button.jsx
+++ b/app/javascript/mastodon/features/compose/components/text_icon_button.jsx
@@ -4,7 +4,7 @@ import { PureComponent } from 'react';
 const iconStyle = {
   height: null,
   lineHeight: '27px',
-  width: `${18 * 1.28571429}px`,
+  minWidth: `${18 * 1.28571429}px`,
 };
 
 export default class TextIconButton extends PureComponent {

From 8f2dac05672094f2b76944ec1935fd0338333131 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 20 Oct 2023 04:08:13 +0200
Subject: [PATCH 034/117] Fix missing background behind dismissable banner in
 web UI (#27479)

---
 .../mastodon/features/explore/statuses.jsx    | 32 ++++++++-----------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/app/javascript/mastodon/features/explore/statuses.jsx b/app/javascript/mastodon/features/explore/statuses.jsx
index f32a4a536..0d8d212b2 100644
--- a/app/javascript/mastodon/features/explore/statuses.jsx
+++ b/app/javascript/mastodon/features/explore/statuses.jsx
@@ -45,24 +45,20 @@ class Statuses extends PureComponent {
     const emptyMessage = <FormattedMessage id='empty_column.explore_statuses' defaultMessage='Nothing is trending right now. Check back later!' />;
 
     return (
-      <>
-        <DismissableBanner id='explore/statuses'>
-          <FormattedMessage id='dismissable_banner.explore_statuses' defaultMessage='These are posts from across the social web that are gaining traction today. Newer posts with more boosts and favorites are ranked higher.' />
-        </DismissableBanner>
-
-        <StatusList
-          trackScroll
-          timelineId='explore'
-          statusIds={statusIds}
-          scrollKey='explore-statuses'
-          hasMore={hasMore}
-          isLoading={isLoading}
-          onLoadMore={this.handleLoadMore}
-          emptyMessage={emptyMessage}
-          bindToDocument={!multiColumn}
-          withCounters
-        />
-      </>
+      <StatusList
+        trackScroll
+        prepend={<DismissableBanner id='explore/statuses'><FormattedMessage id='dismissable_banner.explore_statuses' defaultMessage='These are posts from across the social web that are gaining traction today. Newer posts with more boosts and favorites are ranked higher.' /></DismissableBanner>}
+        alwaysPrepend
+        timelineId='explore'
+        statusIds={statusIds}
+        scrollKey='explore-statuses'
+        hasMore={hasMore}
+        isLoading={isLoading}
+        onLoadMore={this.handleLoadMore}
+        emptyMessage={emptyMessage}
+        bindToDocument={!multiColumn}
+        withCounters
+      />
     );
   }
 

From 3ecc991f6328170a452c69839719bfe92911cbc4 Mon Sep 17 00:00:00 2001
From: Brian Holley <brian.holley@hotmail.com>
Date: Mon, 13 Nov 2023 16:39:54 -0800
Subject: [PATCH 035/117] Fix "Hide these posts from home" list setting not
 refreshing when switching lists (#27763)

---
 app/javascript/mastodon/features/list_timeline/index.jsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/javascript/mastodon/features/list_timeline/index.jsx b/app/javascript/mastodon/features/list_timeline/index.jsx
index a5acc416e..472c5ee8a 100644
--- a/app/javascript/mastodon/features/list_timeline/index.jsx
+++ b/app/javascript/mastodon/features/list_timeline/index.jsx
@@ -201,7 +201,7 @@ class ListTimeline extends PureComponent {
           </div>
 
           <div className='setting-toggle'>
-            <Toggle id={`list-${id}-exclusive`} defaultChecked={isExclusive} onChange={this.onExclusiveToggle} />
+            <Toggle id={`list-${id}-exclusive`} checked={isExclusive} onChange={this.onExclusiveToggle} />
             <label htmlFor={`list-${id}-exclusive`} className='setting-toggle__label'>
               <FormattedMessage id='lists.exclusive' defaultMessage='Hide these posts from home' />
             </label>

From 4d96d716c449d81eb1835b94329e00f316a48fa3 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 22 Nov 2023 12:38:07 +0100
Subject: [PATCH 036/117] Fix unsupported time zone or locale preventing
 sign-up (#28035)

Co-authored-by: Claire <claire.github-309c@sitedethib.com>
---
 app/models/user.rb                             | 14 ++++++++++----
 .../preferences/appearance_controller_spec.rb  |  6 ------
 spec/models/user_spec.rb                       | 18 ++++++++++++------
 3 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/app/models/user.rb b/app/models/user.rb
index fa445af81..309a00d1c 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -96,11 +96,9 @@ class User < ApplicationRecord
   accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? && !Setting.require_invite_text }
   validates :invite_request, presence: true, on: :create, if: :invite_text_required?
 
-  validates :locale, inclusion: I18n.available_locales.map(&:to_s), if: :locale?
   validates_with BlacklistedEmailValidator, if: -> { ENV['EMAIL_DOMAIN_LISTS_APPLY_AFTER_CONFIRMATION'] == 'true' || !confirmed? }
   validates_with EmailMxValidator, if: :validate_email_dns?
   validates :agreement, acceptance: { allow_nil: false, accept: [true, 'true', '1'] }, on: :create
-  validates :time_zone, inclusion: { in: ActiveSupport::TimeZone.all.map { |tz| tz.tzinfo.name } }, allow_blank: true
 
   # Honeypot/anti-spam fields
   attr_accessor :registration_form_time, :website, :confirm_password
@@ -124,6 +122,8 @@ class User < ApplicationRecord
 
   before_validation :sanitize_languages
   before_validation :sanitize_role
+  before_validation :sanitize_time_zone
+  before_validation :sanitize_locale
   before_create :set_approved
   after_commit :send_pending_devise_notifications
   after_create_commit :trigger_webhooks
@@ -451,9 +451,15 @@ class User < ApplicationRecord
   end
 
   def sanitize_role
-    return if role.nil?
+    self.role = nil if role.present? && role.everyone?
+  end
 
-    self.role = nil if role.everyone?
+  def sanitize_time_zone
+    self.time_zone = nil if time_zone.present? && ActiveSupport::TimeZone[time_zone].nil?
+  end
+
+  def sanitize_locale
+    self.locale = nil if locale.present? && I18n.available_locales.exclude?(locale.to_sym)
   end
 
   def prepare_new_user!
diff --git a/spec/controllers/settings/preferences/appearance_controller_spec.rb b/spec/controllers/settings/preferences/appearance_controller_spec.rb
index 9a98a4188..083bf4954 100644
--- a/spec/controllers/settings/preferences/appearance_controller_spec.rb
+++ b/spec/controllers/settings/preferences/appearance_controller_spec.rb
@@ -31,11 +31,5 @@ describe Settings::Preferences::AppearanceController do
 
       expect(response).to redirect_to(settings_preferences_appearance_path)
     end
-
-    it 'renders show on failure' do
-      put :update, params: { user: { locale: 'fake option' } }
-
-      expect(response).to render_template('preferences/appearance/show')
-    end
   end
 end
diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb
index bb61c02a6..0dad96cc8 100644
--- a/spec/models/user_spec.rb
+++ b/spec/models/user_spec.rb
@@ -27,12 +27,6 @@ RSpec.describe User do
       expect(user).to model_have_error_on_field(:account)
     end
 
-    it 'is invalid without a valid locale' do
-      user = Fabricate.build(:user, locale: 'toto')
-      user.valid?
-      expect(user).to model_have_error_on_field(:locale)
-    end
-
     it 'is invalid without a valid email' do
       user = Fabricate.build(:user, email: 'john@')
       user.valid?
@@ -45,6 +39,18 @@ RSpec.describe User do
       expect(user.valid?).to be true
     end
 
+    it 'cleans out invalid locale' do
+      user = Fabricate.build(:user, locale: 'toto')
+      expect(user.valid?).to be true
+      expect(user.locale).to be_nil
+    end
+
+    it 'cleans out invalid timezone' do
+      user = Fabricate.build(:user, time_zone: 'toto')
+      expect(user.valid?).to be true
+      expect(user.time_zone).to be_nil
+    end
+
     it 'cleans out empty string from languages' do
       user = Fabricate.build(:user, chosen_languages: [''])
       user.valid?

From c609b726cba4874597d203c80cce647cd0932fbc Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 7 Dec 2023 10:05:08 +0100
Subject: [PATCH 037/117] Fix error when processing link preview with an array
 as `inLanguage` (#28252)

---
 app/lib/link_details_extractor.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb
index a96612cab..bb031986d 100644
--- a/app/lib/link_details_extractor.rb
+++ b/app/lib/link_details_extractor.rb
@@ -37,6 +37,7 @@ class LinkDetailsExtractor
 
     def language
       lang = json['inLanguage']
+      lang = lang.first if lang.is_a?(Array)
       lang.is_a?(Hash) ? (lang['alternateName'] || lang['name']) : lang
     end
 

From 01caa18e5ba6eea8f102f31916ed47e5c781e862 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Tue, 2 Jan 2024 13:27:51 +0100
Subject: [PATCH 038/117] Fix streaming API redirection ignoring the port of
 `streaming_api_base_url` (#28558)

---
 app/controllers/api/v1/streaming_controller.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/controllers/api/v1/streaming_controller.rb b/app/controllers/api/v1/streaming_controller.rb
index 0cdd00d62..0cb6e856f 100644
--- a/app/controllers/api/v1/streaming_controller.rb
+++ b/app/controllers/api/v1/streaming_controller.rb
@@ -13,7 +13,9 @@ class Api::V1::StreamingController < Api::BaseController
 
   def streaming_api_url
     Addressable::URI.parse(request.url).tap do |uri|
-      uri.host = Addressable::URI.parse(Rails.configuration.x.streaming_api_base_url).host
+      base_url = Addressable::URI.parse(Rails.configuration.x.streaming_api_base_url)
+      uri.host = base_url.host
+      uri.port = base_url.port
     end.to_s
   end
 end

From c0a9db3611b3cda655409bbf28ec5531194cd20e Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 10 Jan 2024 16:05:46 +0100
Subject: [PATCH 039/117] Fix potential redirection loop of streaming endpoint
 (#28665)

---
 app/controllers/api/v1/streaming_controller.rb       | 7 ++++++-
 spec/controllers/api/v1/streaming_controller_spec.rb | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/app/controllers/api/v1/streaming_controller.rb b/app/controllers/api/v1/streaming_controller.rb
index 0cb6e856f..adb14676e 100644
--- a/app/controllers/api/v1/streaming_controller.rb
+++ b/app/controllers/api/v1/streaming_controller.rb
@@ -2,7 +2,7 @@
 
 class Api::V1::StreamingController < Api::BaseController
   def index
-    if Rails.configuration.x.streaming_api_base_url == request.host
+    if same_host?
       not_found
     else
       redirect_to streaming_api_url, status: 301, allow_other_host: true
@@ -11,6 +11,11 @@ class Api::V1::StreamingController < Api::BaseController
 
   private
 
+  def same_host?
+    base_url = Addressable::URI.parse(Rails.configuration.x.streaming_api_base_url)
+    request.host == base_url.host && request.port == (base_url.port || 80)
+  end
+
   def streaming_api_url
     Addressable::URI.parse(request.url).tap do |uri|
       base_url = Addressable::URI.parse(Rails.configuration.x.streaming_api_base_url)
diff --git a/spec/controllers/api/v1/streaming_controller_spec.rb b/spec/controllers/api/v1/streaming_controller_spec.rb
index 7014ed9b2..825bb1197 100644
--- a/spec/controllers/api/v1/streaming_controller_spec.rb
+++ b/spec/controllers/api/v1/streaming_controller_spec.rb
@@ -5,7 +5,7 @@ require 'rails_helper'
 describe Api::V1::StreamingController do
   around(:each) do |example|
     before = Rails.configuration.x.streaming_api_base_url
-    Rails.configuration.x.streaming_api_base_url = Rails.configuration.x.web_domain
+    Rails.configuration.x.streaming_api_base_url = "wss://#{Rails.configuration.x.web_domain}"
     example.run
     Rails.configuration.x.streaming_api_base_url = before
   end

From 1998c561b2ffe7200781778f281fcc0890ebf6df Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 22 Dec 2023 19:56:22 +0100
Subject: [PATCH 040/117] Convert signature verification specs to request specs
 (#28443)

---
 .../concerns/signature_verification_spec.rb   | 305 ----------------
 spec/requests/signature_verification_spec.rb  | 332 ++++++++++++++++++
 2 files changed, 332 insertions(+), 305 deletions(-)
 delete mode 100644 spec/controllers/concerns/signature_verification_spec.rb
 create mode 100644 spec/requests/signature_verification_spec.rb

diff --git a/spec/controllers/concerns/signature_verification_spec.rb b/spec/controllers/concerns/signature_verification_spec.rb
deleted file mode 100644
index 650cd21ea..000000000
--- a/spec/controllers/concerns/signature_verification_spec.rb
+++ /dev/null
@@ -1,305 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-describe SignatureVerification do
-  let(:wrapped_actor_class) do
-    Class.new do
-      attr_reader :wrapped_account
-
-      def initialize(wrapped_account)
-        @wrapped_account = wrapped_account
-      end
-
-      delegate :uri, :keypair, to: :wrapped_account
-    end
-  end
-
-  controller(ApplicationController) do
-    include SignatureVerification
-
-    before_action :require_actor_signature!, only: [:signature_required]
-
-    def success
-      head 200
-    end
-
-    def alternative_success
-      head 200
-    end
-
-    def signature_required
-      head 200
-    end
-  end
-
-  before do
-    routes.draw do
-      match :via => [:get, :post], 'success' => 'anonymous#success'
-      match :via => [:get, :post], 'signature_required' => 'anonymous#signature_required'
-    end
-  end
-
-  context 'without signature header' do
-    before do
-      get :success
-    end
-
-    describe '#signed_request?' do
-      it 'returns false' do
-        expect(controller.signed_request?).to be false
-      end
-    end
-
-    describe '#signed_request_account' do
-      it 'returns nil' do
-        expect(controller.signed_request_account).to be_nil
-      end
-    end
-  end
-
-  context 'with signature header' do
-    let!(:author) { Fabricate(:account, domain: 'example.com', uri: 'https://example.com/actor') }
-
-    context 'without body' do
-      before do
-        get :success
-
-        fake_request = Request.new(:get, request.url)
-        fake_request.on_behalf_of(author)
-
-        request.headers.merge!(fake_request.headers)
-      end
-
-      describe '#signed_request?' do
-        it 'returns true' do
-          expect(controller.signed_request?).to be true
-        end
-      end
-
-      describe '#signed_request_account' do
-        it 'returns an account' do
-          expect(controller.signed_request_account).to eq author
-        end
-
-        it 'returns nil when path does not match' do
-          request.path = '/alternative-path'
-          expect(controller.signed_request_account).to be_nil
-        end
-
-        it 'returns nil when method does not match' do
-          post :success
-          expect(controller.signed_request_account).to be_nil
-        end
-      end
-    end
-
-    context 'with a valid actor that is not an Account' do
-      let(:actor) { wrapped_actor_class.new(author) }
-
-      before do
-        get :success
-
-        fake_request = Request.new(:get, request.url)
-        fake_request.on_behalf_of(author)
-
-        request.headers.merge!(fake_request.headers)
-
-        allow(ActivityPub::TagManager.instance).to receive(:uri_to_actor).with(anything) do
-          actor
-        end
-      end
-
-      describe '#signed_request?' do
-        it 'returns true' do
-          expect(controller.signed_request?).to be true
-        end
-      end
-
-      describe '#signed_request_account' do
-        it 'returns nil' do
-          expect(controller.signed_request_account).to be_nil
-        end
-      end
-
-      describe '#signed_request_actor' do
-        it 'returns the expected actor' do
-          expect(controller.signed_request_actor).to eq actor
-        end
-      end
-    end
-
-    context 'with request with unparsable Date header' do
-      before do
-        get :success
-
-        fake_request = Request.new(:get, request.url)
-        fake_request.add_headers({ 'Date' => 'wrong date' })
-        fake_request.on_behalf_of(author)
-
-        request.headers.merge!(fake_request.headers)
-      end
-
-      describe '#signed_request?' do
-        it 'returns true' do
-          expect(controller.signed_request?).to be true
-        end
-      end
-
-      describe '#signed_request_account' do
-        it 'returns nil' do
-          expect(controller.signed_request_account).to be_nil
-        end
-      end
-
-      describe '#signature_verification_failure_reason' do
-        it 'contains an error description' do
-          controller.signed_request_account
-          expect(controller.signature_verification_failure_reason[:error]).to eq 'Invalid Date header: not RFC 2616 compliant date: "wrong date"'
-        end
-      end
-    end
-
-    context 'with request older than a day' do
-      before do
-        get :success
-
-        fake_request = Request.new(:get, request.url)
-        fake_request.add_headers({ 'Date' => 2.days.ago.utc.httpdate })
-        fake_request.on_behalf_of(author)
-
-        request.headers.merge!(fake_request.headers)
-      end
-
-      describe '#signed_request?' do
-        it 'returns true' do
-          expect(controller.signed_request?).to be true
-        end
-      end
-
-      describe '#signed_request_account' do
-        it 'returns nil' do
-          expect(controller.signed_request_account).to be_nil
-        end
-      end
-
-      describe '#signature_verification_failure_reason' do
-        it 'contains an error description' do
-          controller.signed_request_account
-          expect(controller.signature_verification_failure_reason[:error]).to eq 'Signed request date outside acceptable time window'
-        end
-      end
-    end
-
-    context 'with inaccessible key' do
-      before do
-        get :success
-
-        author = Fabricate(:account, domain: 'localhost:5000', uri: 'http://localhost:5000/actor')
-        fake_request = Request.new(:get, request.url)
-        fake_request.on_behalf_of(author)
-        author.destroy
-
-        request.headers.merge!(fake_request.headers)
-
-        stub_request(:get, 'http://localhost:5000/actor#main-key').to_raise(Mastodon::HostValidationError)
-      end
-
-      describe '#signed_request?' do
-        it 'returns true' do
-          expect(controller.signed_request?).to be true
-        end
-      end
-
-      describe '#signed_request_account' do
-        it 'returns nil' do
-          expect(controller.signed_request_account).to be_nil
-        end
-      end
-    end
-
-    context 'with body' do
-      before do
-        allow(controller).to receive(:actor_refresh_key!).and_return(author)
-        post :success, body: 'Hello world'
-
-        fake_request = Request.new(:post, request.url, body: 'Hello world')
-        fake_request.on_behalf_of(author)
-
-        request.headers.merge!(fake_request.headers)
-      end
-
-      describe '#signed_request?' do
-        it 'returns true' do
-          expect(controller.signed_request?).to be true
-        end
-      end
-
-      describe '#signed_request_account' do
-        it 'returns an account' do
-          expect(controller.signed_request_account).to eq author
-        end
-      end
-
-      context 'when path does not match' do
-        before do
-          request.path = '/alternative-path'
-        end
-
-        describe '#signed_request_account' do
-          it 'returns nil' do
-            expect(controller.signed_request_account).to be_nil
-          end
-        end
-
-        describe '#signature_verification_failure_reason' do
-          it 'contains an error description' do
-            controller.signed_request_account
-            expect(controller.signature_verification_failure_reason[:error]).to include('using rsa-sha256 (RSASSA-PKCS1-v1_5 with SHA-256)')
-            expect(controller.signature_verification_failure_reason[:signed_string]).to include("(request-target): post /alternative-path\n")
-          end
-        end
-      end
-
-      context 'when method does not match' do
-        before do
-          get :success
-        end
-
-        describe '#signed_request_account' do
-          it 'returns nil' do
-            expect(controller.signed_request_account).to be_nil
-          end
-        end
-      end
-
-      context 'when body has been tampered' do
-        before do
-          post :success, body: 'doo doo doo'
-        end
-
-        describe '#signed_request_account' do
-          it 'returns nil when body has been tampered' do
-            expect(controller.signed_request_account).to be_nil
-          end
-        end
-      end
-    end
-  end
-
-  context 'when a signature is required' do
-    before do
-      get :signature_required
-    end
-
-    context 'without signature header' do
-      it 'returns HTTP 401' do
-        expect(response).to have_http_status(401)
-      end
-
-      it 'returns an error' do
-        expect(Oj.load(response.body)['error']).to eq 'Request not signed'
-      end
-    end
-  end
-end
diff --git a/spec/requests/signature_verification_spec.rb b/spec/requests/signature_verification_spec.rb
new file mode 100644
index 000000000..b753750b8
--- /dev/null
+++ b/spec/requests/signature_verification_spec.rb
@@ -0,0 +1,332 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe 'signature verification concern' do
+  before do
+    stub_tests_controller
+
+    # Signature checking is time-dependent, so travel to a fixed date
+    travel_to '2023-12-20T10:00:00Z'
+  end
+
+  after { Rails.application.reload_routes! }
+
+  # Include the private key so the tests can be easily adjusted and reviewed
+  let(:actor_keypair) do
+    OpenSSL::PKey.read(<<~PEM_TEXT)
+      -----BEGIN RSA PRIVATE KEY-----
+      MIIEowIBAAKCAQEAqIAYvNFGbZ5g4iiK6feSdXD4bDStFM58A7tHycYXaYtzZQpI
+      eHXAmaXuZzXIwtrP4N0gIk8JNwZvXj2UPS+S07t0V9wNK94he01LV5EMz/GN4eNn
+      FmDL64HIEuKLvV8TvgjbUPRD6Y5X0UpKi2ZIFLSb96Q5w0Z/k7ntpVKV52y8kz5F
+      jr/O/0JuHryZe0yItzJh8kzFfeMf0EXzfSnaKvT7P9jhgC6uTre+jXyvVZjiHDrn
+      qvvucdI3I7DRfXo1OqARBrLjy+TdseUAjNYJ+OuPRI1URIWQI01DCHqcohVu9+Ar
+      +BiCjFp3ua+XMuJvrvbD61d1Fvig/9nbBRR+8QIDAQABAoIBAAgySHnFWI6gItR3
+      fkfiqIm80cHCN3Xk1C6iiVu+3oBOZbHpW9R7vl9e/WOA/9O+LPjiSsQOegtWnVvd
+      RRjrl7Hj20VDlZKv5Mssm6zOGAxksrcVbqwdj+fUJaNJCL0AyyseH0x/IE9T8rDC
+      I1GH+3tB3JkhkIN/qjipdX5ab8MswEPu8IC4ViTpdBgWYY/xBcAHPw4xuL0tcwzh
+      FBlf4DqoEVQo8GdK5GAJ2Ny0S4xbXHUURzx/R4y4CCts7niAiLGqd9jmLU1kUTMk
+      QcXfQYK6l+unLc7wDYAz7sFEHh04M48VjWwiIZJnlCqmQbLda7uhhu8zkF1DqZTu
+      ulWDGQECgYEA0TIAc8BQBVab979DHEEmMdgqBwxLY3OIAk0b+r50h7VBGWCDPRsC
+      STD73fQY3lNet/7/jgSGwwAlAJ5PpMXxXiZAE3bUwPmHzgF7pvIOOLhA8O07tHSO
+      L2mvQe6NPzjZ+6iAO2U9PkClxcvGvPx2OBvisfHqZLmxC9PIVxzruQECgYEAzjM6
+      BTUXa6T/qHvLFbN699BXsUOGmHBGaLRapFDBfVvgZrwqYQcZpBBhesLdGTGSqwE7
+      gWsITPIJ+Ldo+38oGYyVys+w/V67q6ud7hgSDTW3hSvm+GboCjk6gzxlt9hQ0t9X
+      8vfDOYhEXvVUJNv3mYO60ENqQhILO4bQ0zi+VfECgYBb/nUccfG+pzunU0Cb6Dp3
+      qOuydcGhVmj1OhuXxLFSDG84Tazo7juvHA9mp7VX76mzmDuhpHPuxN2AzB2SBEoE
+      cSW0aYld413JRfWukLuYTc6hJHIhBTCRwRQFFnae2s1hUdQySm8INT2xIc+fxBXo
+      zrp+Ljg5Wz90SAnN5TX0AQKBgDaatDOq0o/r+tPYLHiLtfWoE4Dau+rkWJDjqdk3
+      lXWn/e3WyHY3Vh/vQpEqxzgju45TXjmwaVtPATr+/usSykCxzP0PMPR3wMT+Rm1F
+      rIoY/odij+CaB7qlWwxj0x/zRbwB7x1lZSp4HnrzBpxYL+JUUwVRxPLIKndSBTza
+      GvVRAoGBAIVBcNcRQYF4fvZjDKAb4fdBsEuHmycqtRCsnkGOz6ebbEQznSaZ0tZE
+      +JuouZaGjyp8uPjNGD5D7mIGbyoZ3KyG4mTXNxDAGBso1hrNDKGBOrGaPhZx8LgO
+      4VXJ+ybXrATf4jr8ccZYsZdFpOphPzz+j55Mqg5vac5P1XjmsGTb
+      -----END RSA PRIVATE KEY-----
+    PEM_TEXT
+  end
+
+  context 'without a Signature header' do
+    it 'does not treat the request as signed' do
+      get '/activitypub/success'
+
+      expect(response).to have_http_status(200)
+      expect(body_as_json).to match(
+        signed_request: false,
+        signature_actor_id: nil,
+        error: 'Request not signed'
+      )
+    end
+
+    context 'when a signature is required' do
+      it 'returns http unauthorized with appropriate error' do
+        get '/activitypub/signature_required'
+
+        expect(response).to have_http_status(401)
+        expect(body_as_json).to match(
+          error: 'Request not signed'
+        )
+      end
+    end
+  end
+
+  context 'with an HTTP Signature from a known account' do
+    let!(:actor) { Fabricate(:account, domain: 'remote.domain', uri: 'https://remote.domain/users/bob', private_key: nil, public_key: actor_keypair.public_key.to_pem) }
+
+    context 'with a valid signature on a GET request' do
+      let(:signature_header) do
+        'keyId="https://remote.domain/users/bob#main-key",algorithm="rsa-sha256",headers="date host (request-target)",signature="Z8ilar3J7bOwqZkMp7sL8sRs4B1FT+UorbmvWoE+A5UeoOJ3KBcUmbsh+k3wQwbP5gMNUrra9rEWabpasZGphLsbDxfbsWL3Cf0PllAc7c1c7AFEwnewtExI83/qqgEkfWc2z7UDutXc2NfgAx89Ox8DXU/fA2GG0jILjB6UpFyNugkY9rg6oI31UnvfVi3R7sr3/x8Ea3I9thPvqI2byF6cojknSpDAwYzeKdngX3TAQEGzFHz3SDWwyp3jeMWfwvVVbM38FxhvAnSumw7YwWW4L7M7h4M68isLimoT3yfCn2ucBVL5Dz8koBpYf/40w7QidClAwCafZQFC29yDOg=="' # rubocop:disable Layout/LineLength
+      end
+
+      it 'successfuly verifies signature', :aggregate_failures do
+        expect(signature_header).to eq build_signature_string(actor_keypair, 'https://remote.domain/users/bob#main-key', 'get /activitypub/success', { 'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT', 'Host' => 'www.example.com' })
+
+        get '/activitypub/success', headers: {
+          'Host' => 'www.example.com',
+          'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT',
+          'Signature' => signature_header,
+        }
+
+        expect(response).to have_http_status(200)
+        expect(body_as_json).to match(
+          signed_request: true,
+          signature_actor_id: actor.id.to_s
+        )
+      end
+    end
+
+    context 'with a mismatching path' do
+      it 'fails to verify signature', :aggregate_failures do
+        get '/activitypub/alternative-path', headers: {
+          'Host' => 'www.example.com',
+          'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT',
+          'Signature' => 'keyId="https://remote.domain/users/bob#main-key",algorithm="rsa-sha256",headers="date host (request-target)",signature="Z8ilar3J7bOwqZkMp7sL8sRs4B1FT+UorbmvWoE+A5UeoOJ3KBcUmbsh+k3wQwbP5gMNUrra9rEWabpasZGphLsbDxfbsWL3Cf0PllAc7c1c7AFEwnewtExI83/qqgEkfWc2z7UDutXc2NfgAx89Ox8DXU/fA2GG0jILjB6UpFyNugkY9rg6oI31UnvfVi3R7sr3/x8Ea3I9thPvqI2byF6cojknSpDAwYzeKdngX3TAQEGzFHz3SDWwyp3jeMWfwvVVbM38FxhvAnSumw7YwWW4L7M7h4M68isLimoT3yfCn2ucBVL5Dz8koBpYf/40w7QidClAwCafZQFC29yDOg=="', # rubocop:disable Layout/LineLength
+        }
+
+        expect(body_as_json).to match(
+          signed_request: true,
+          signature_actor_id: nil,
+          error: anything
+        )
+      end
+    end
+
+    context 'with a mismatching method' do
+      it 'fails to verify signature', :aggregate_failures do
+        post '/activitypub/success', headers: {
+          'Host' => 'www.example.com',
+          'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT',
+          'Signature' => 'keyId="https://remote.domain/users/bob#main-key",algorithm="rsa-sha256",headers="date host (request-target)",signature="Z8ilar3J7bOwqZkMp7sL8sRs4B1FT+UorbmvWoE+A5UeoOJ3KBcUmbsh+k3wQwbP5gMNUrra9rEWabpasZGphLsbDxfbsWL3Cf0PllAc7c1c7AFEwnewtExI83/qqgEkfWc2z7UDutXc2NfgAx89Ox8DXU/fA2GG0jILjB6UpFyNugkY9rg6oI31UnvfVi3R7sr3/x8Ea3I9thPvqI2byF6cojknSpDAwYzeKdngX3TAQEGzFHz3SDWwyp3jeMWfwvVVbM38FxhvAnSumw7YwWW4L7M7h4M68isLimoT3yfCn2ucBVL5Dz8koBpYf/40w7QidClAwCafZQFC29yDOg=="', # rubocop:disable Layout/LineLength
+        }
+
+        expect(body_as_json).to match(
+          signed_request: true,
+          signature_actor_id: nil,
+          error: anything
+        )
+      end
+    end
+
+    context 'with an unparsable date' do
+      let(:signature_header) do
+        'keyId="https://remote.domain/users/bob#main-key",algorithm="rsa-sha256",headers="date host (request-target)",signature="d4B7nfx8RJcfdJDu1J//5WzPzK/hgtPkdzZx49lu5QhnE7qdV3lgyVimmhCFrO16bwvzIp9iRMyRLkNFxLiEeVaa1gqeKbldGSnU0B0OMjx7rFBa65vLuzWQOATDitVGiBEYqoK4v0DMuFCz2DtFaA/DIUZ3sty8bZ/Ea3U1nByLOO6MacARA3zhMSI0GNxGqsSmZmG0hPLavB3jIXoE3IDoQabMnC39jrlcO/a8h1iaxBm2WD8TejrImJullgqlJIFpKhIHI3ipQkvTGPlm9dx0y+beM06qBvWaWQcmT09eRIUefVsOAzIhUtS/7FVb/URhZvircIJDa7vtiFcmZQ=="' # rubocop:disable Layout/LineLength
+      end
+
+      it 'fails to verify signature', :aggregate_failures do
+        expect(signature_header).to eq build_signature_string(actor_keypair, 'https://remote.domain/users/bob#main-key', 'get /activitypub/success', { 'Date' => 'wrong date', 'Host' => 'www.example.com' })
+
+        get '/activitypub/success', headers: {
+          'Host' => 'www.example.com',
+          'Date' => 'wrong date',
+          'Signature' => signature_header,
+        }
+
+        expect(body_as_json).to match(
+          signed_request: true,
+          signature_actor_id: nil,
+          error: 'Invalid Date header: not RFC 2616 compliant date: "wrong date"'
+        )
+      end
+    end
+
+    context 'with a request older than a day' do
+      let(:signature_header) do
+        'keyId="https://remote.domain/users/bob#main-key",algorithm="rsa-sha256",headers="date host (request-target)",signature="G1NuJv4zgoZ3B/ZIjzDWZHK4RC+5pYee74q8/LJEMCWXhcnAomcb9YHaqk1QYfQvcBUIXw3UZ3Q9xO8F9y0i8G5mzJHfQ+OgHqCoJk8EmGwsUXJMh5s1S5YFCRt8TT12TmJZz0VMqLq85ubueSYBM7QtUE/FzFIVLvz4RysgXxaXQKzdnM6+gbUEEKdCURpXdQt2NXQhp4MAmZH3+0lQoR6VxdsK0hx0Ji2PNp1nuqFTlYqNWZazVdLBN+9rETLRmvGXknvg9jOxTTppBVWnkAIl26HtLS3wwFVvz4pJzi9OQDOvLziehVyLNbU61hky+oJ215e2HuKSe2hxHNl1MA=="' # rubocop:disable Layout/LineLength
+      end
+
+      it 'fails to verify signature', :aggregate_failures do
+        expect(signature_header).to eq build_signature_string(actor_keypair, 'https://remote.domain/users/bob#main-key', 'get /activitypub/success', { 'Date' => 'Wed, 18 Dec 2023 10:00:00 GMT', 'Host' => 'www.example.com' })
+
+        get '/activitypub/success', headers: {
+          'Host' => 'www.example.com',
+          'Date' => 'Wed, 18 Dec 2023 10:00:00 GMT',
+          'Signature' => signature_header,
+        }
+
+        expect(body_as_json).to match(
+          signed_request: true,
+          signature_actor_id: nil,
+          error: 'Signed request date outside acceptable time window'
+        )
+      end
+    end
+
+    context 'with a valid signature on a POST request' do
+      let(:digest_header) { 'SHA-256=ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=' }
+      let(:signature_header) do
+        'keyId="https://remote.domain/users/bob#main-key",algorithm="rsa-sha256",headers="host date digest (request-target)",signature="gmhMjgMROGElJU3fpehV2acD5kMHeELi8EFP2UPHOdQ54H0r55AxIpji+J3lPe+N2qSb/4H1KXIh6f0lRu8TGSsu12OQmg5hiO8VA9flcA/mh9Lpk+qwlQZIPRqKP9xUEfqD+Z7ti5wPzDKrWAUK/7FIqWgcT/mlqB1R1MGkpMFc/q4CIs2OSNiWgA4K+Kp21oQxzC2kUuYob04gAZ7cyE/FTia5t08uv6lVYFdRsn4XNPn1MsHgFBwBMRG79ng3SyhoG4PrqBEi5q2IdLq3zfre/M6He3wlCpyO2VJNdGVoTIzeZ0Zz8jUscPV3XtWUchpGclLGSaKaq/JyNZeiYQ=="' # rubocop:disable Layout/LineLength
+      end
+
+      it 'successfuly verifies signature', :aggregate_failures do
+        expect(digest_header).to eq digest_value('Hello world')
+        expect(signature_header).to eq build_signature_string(actor_keypair, 'https://remote.domain/users/bob#main-key', 'post /activitypub/success', { 'Host' => 'www.example.com', 'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT', 'Digest' => digest_header })
+
+        post '/activitypub/success', params: 'Hello world', headers: {
+          'Host' => 'www.example.com',
+          'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT',
+          'Digest' => digest_header,
+          'Signature' => signature_header,
+        }
+
+        expect(response).to have_http_status(200)
+        expect(body_as_json).to match(
+          signed_request: true,
+          signature_actor_id: actor.id.to_s
+        )
+      end
+    end
+
+    context 'when the Digest of a POST request is not signed' do
+      let(:digest_header) { 'SHA-256=ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=' }
+      let(:signature_header) do
+        'keyId="https://remote.domain/users/bob#main-key",algorithm="rsa-sha256",headers="host date (request-target)",signature="CPD704CG8aCm8X8qIP8kkkiGp1qwFLk/wMVQHOGP0Txxan8c2DZtg/KK7eN8RG8tHx8br/yS2hJs51x4kXImYukGzNJd7ihE3T8lp+9RI1tCcdobTzr/VcVJHDFySdQkg266GCMijRQRZfNvqlJLiisr817PI+gNVBI5qV+vnVd1XhWCEZ+YSmMe8UqYARXAYNqMykTheojqGpTeTFGPUpTQA2Fmt2BipwIjcFDm2Hpihl2kB0MUS0x3zPmHDuadvzoBbN6m3usPDLgYrpALlh+wDs1dYMntcwdwawRKY1oE1XNtgOSum12wntDq3uYL4gya2iPdcw3c929b4koUzw=="' # rubocop:disable Layout/LineLength
+      end
+
+      it 'fails to verify signature', :aggregate_failures do
+        expect(digest_header).to eq digest_value('Hello world')
+        expect(signature_header).to eq build_signature_string(actor_keypair, 'https://remote.domain/users/bob#main-key', 'post /activitypub/success', { 'Host' => 'www.example.com', 'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT' })
+
+        post '/activitypub/success', params: 'Hello world', headers: {
+          'Host' => 'www.example.com',
+          'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT',
+          'Digest' => digest_header,
+          'Signature' => signature_header,
+        }
+
+        expect(body_as_json).to match(
+          signed_request: true,
+          signature_actor_id: nil,
+          error: 'Mastodon requires the Digest header to be signed when doing a POST request'
+        )
+      end
+    end
+
+    context 'with a tampered body on a POST request' do
+      let(:digest_header) { 'SHA-256=ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=' }
+      let(:signature_header) do
+        'keyId="https://remote.domain/users/bob#main-key",algorithm="rsa-sha256",headers="host date digest (request-target)",signature="gmhMjgMROGElJU3fpehV2acD5kMHeELi8EFP2UPHOdQ54H0r55AxIpji+J3lPe+N2qSb/4H1KXIh6f0lRu8TGSsu12OQmg5hiO8VA9flcA/mh9Lpk+qwlQZIPRqKP9xUEfqD+Z7ti5wPzDKrWAUK/7FIqWgcT/mlqB1R1MGkpMFc/q4CIs2OSNiWgA4K+Kp21oQxzC2kUuYob04gAZ7cyE/FTia5t08uv6lVYFdRsn4XNPn1MsHgFBwBMRG79ng3SyhoG4PrqBEi5q2IdLq3zfre/M6He3wlCpyO2VJNdGVoTIzeZ0Zz8jUscPV3XtWUchpGclLGSaKaq/JyNZeiYQ=="' # rubocop:disable Layout/LineLength
+      end
+
+      it 'fails to verify signature', :aggregate_failures do
+        expect(digest_header).to_not eq digest_value('Hello world!')
+        expect(signature_header).to eq build_signature_string(actor_keypair, 'https://remote.domain/users/bob#main-key', 'post /activitypub/success', { 'Host' => 'www.example.com', 'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT', 'Digest' => digest_header })
+
+        post '/activitypub/success', params: 'Hello world!', headers: {
+          'Host' => 'www.example.com',
+          'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT',
+          'Digest' => 'SHA-256=ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=',
+          'Signature' => signature_header,
+        }
+
+        expect(body_as_json).to match(
+          signed_request: true,
+          signature_actor_id: nil,
+          error: 'Invalid Digest value. Computed SHA-256 digest: wFNeS+K3n/2TKRMFQ2v4iTFOSj+uwF7P/Lt98xrZ5Ro=; given: ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw='
+        )
+      end
+    end
+
+    context 'with a tampered path in a POST request' do
+      it 'fails to verify signature', :aggregate_failures do
+        post '/activitypub/alternative-path', params: 'Hello world', headers: {
+          'Host' => 'www.example.com',
+          'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT',
+          'Digest' => 'SHA-256=ZOyIygCyaOW6GjVnihtTFtIS9PNmskdyMlNKiuyjfzw=',
+          'Signature' => 'keyId="https://remote.domain/users/bob#main-key",algorithm="rsa-sha256",headers="host date digest (request-target)",signature="gmhMjgMROGElJU3fpehV2acD5kMHeELi8EFP2UPHOdQ54H0r55AxIpji+J3lPe+N2qSb/4H1KXIh6f0lRu8TGSsu12OQmg5hiO8VA9flcA/mh9Lpk+qwlQZIPRqKP9xUEfqD+Z7ti5wPzDKrWAUK/7FIqWgcT/mlqB1R1MGkpMFc/q4CIs2OSNiWgA4K+Kp21oQxzC2kUuYob04gAZ7cyE/FTia5t08uv6lVYFdRsn4XNPn1MsHgFBwBMRG79ng3SyhoG4PrqBEi5q2IdLq3zfre/M6He3wlCpyO2VJNdGVoTIzeZ0Zz8jUscPV3XtWUchpGclLGSaKaq/JyNZeiYQ=="', # rubocop:disable Layout/LineLength
+        }
+
+        expect(response).to have_http_status(200)
+        expect(body_as_json).to match(
+          signed_request: true,
+          signature_actor_id: nil,
+          error: anything
+        )
+      end
+    end
+  end
+
+  context 'with an inaccessible key' do
+    before do
+      stub_request(:get, 'https://remote.domain/users/alice#main-key').to_return(status: 404)
+    end
+
+    it 'fails to verify signature', :aggregate_failures do
+      get '/activitypub/success', headers: {
+        'Host' => 'www.example.com',
+        'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT',
+        'Signature' => 'keyId="https://remote.domain/users/alice#main-key",algorithm="rsa-sha256",headers="date host (request-target)",signature="Z8ilar3J7bOwqZkMp7sL8sRs4B1FT+UorbmvWoE+A5UeoOJ3KBcUmbsh+k3wQwbP5gMNUrra9rEWabpasZGphLsbDxfbsWL3Cf0PllAc7c1c7AFEwnewtExI83/qqgEkfWc2z7UDutXc2NfgAx89Ox8DXU/fA2GG0jILjB6UpFyNugkY9rg6oI31UnvfVi3R7sr3/x8Ea3I9thPvqI2byF6cojknSpDAwYzeKdngX3TAQEGzFHz3SDWwyp3jeMWfwvVVbM38FxhvAnSumw7YwWW4L7M7h4M68isLimoT3yfCn2ucBVL5Dz8koBpYf/40w7QidClAwCafZQFC29yDOg=="', # rubocop:disable Layout/LineLength
+      }
+
+      expect(body_as_json).to match(
+        signed_request: true,
+        signature_actor_id: nil,
+        error: 'Unable to fetch key JSON at https://remote.domain/users/alice#main-key'
+      )
+    end
+  end
+
+  private
+
+  def stub_tests_controller
+    stub_const('ActivityPub::TestsController', activitypub_tests_controller)
+
+    Rails.application.routes.draw do
+      # NOTE: RouteSet#draw removes all routes, so we need to re-insert one
+      resource :instance_actor, path: 'actor', only: [:show]
+
+      match :via => [:get, :post], '/activitypub/success' => 'activitypub/tests#success'
+      match :via => [:get, :post], '/activitypub/alternative-path' => 'activitypub/tests#alternative_success'
+      match :via => [:get, :post], '/activitypub/signature_required' => 'activitypub/tests#signature_required'
+    end
+  end
+
+  def activitypub_tests_controller
+    Class.new(ApplicationController) do
+      include SignatureVerification
+
+      before_action :require_actor_signature!, only: [:signature_required]
+
+      def success
+        render json: {
+          signed_request: signed_request?,
+          signature_actor_id: signed_request_actor&.id&.to_s,
+        }.merge(signature_verification_failure_reason || {})
+      end
+
+      alias_method :alternative_success, :success
+      alias_method :signature_required, :success
+    end
+  end
+
+  def digest_value(body)
+    "SHA-256=#{Digest::SHA256.base64digest(body)}"
+  end
+
+  def build_signature_string(keypair, key_id, request_target, headers)
+    algorithm = 'rsa-sha256'
+    signed_headers = headers.merge({ '(request-target)' => request_target })
+    signed_string = signed_headers.map { |key, value| "#{key.downcase}: #{value}" }.join("\n")
+    signature = Base64.strict_encode64(keypair.sign(OpenSSL::Digest.new('SHA256'), signed_string))
+
+    "keyId=\"#{key_id}\",algorithm=\"#{algorithm}\",headers=\"#{signed_headers.keys.join(' ').downcase}\",signature=\"#{signature}\""
+  end
+end

From 3837ec2227159d5b05f07d4d9c4a01647f29ad6f Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 3 Jan 2024 12:29:26 +0100
Subject: [PATCH 041/117] Fix Mastodon not correctly processing HTTP Signatures
 with query strings (#28476)

---
 .../concerns/signature_verification.rb        | 22 ++++++-
 app/lib/request.rb                            | 11 +++-
 spec/requests/signature_verification_spec.rb  | 66 +++++++++++++++++++
 3 files changed, 95 insertions(+), 4 deletions(-)

diff --git a/app/controllers/concerns/signature_verification.rb b/app/controllers/concerns/signature_verification.rb
index f0a344f1c..35391e64c 100644
--- a/app/controllers/concerns/signature_verification.rb
+++ b/app/controllers/concerns/signature_verification.rb
@@ -91,14 +91,23 @@ module SignatureVerification
     raise SignatureVerificationError, "Public key not found for key #{signature_params['keyId']}" if actor.nil?
 
     signature             = Base64.decode64(signature_params['signature'])
-    compare_signed_string = build_signed_string
+    compare_signed_string = build_signed_string(include_query_string: true)
 
     return actor unless verify_signature(actor, signature, compare_signed_string).nil?
 
+    # Compatibility quirk with older Mastodon versions
+    compare_signed_string = build_signed_string(include_query_string: false)
+    return actor unless verify_signature(actor, signature, compare_signed_string).nil?
+
     actor = stoplight_wrap_request { actor_refresh_key!(actor) }
 
     raise SignatureVerificationError, "Could not refresh public key #{signature_params['keyId']}" if actor.nil?
 
+    compare_signed_string = build_signed_string(include_query_string: true)
+    return actor unless verify_signature(actor, signature, compare_signed_string).nil?
+
+    # Compatibility quirk with older Mastodon versions
+    compare_signed_string = build_signed_string(include_query_string: false)
     return actor unless verify_signature(actor, signature, compare_signed_string).nil?
 
     fail_with! "Verification failed for #{actor.to_log_human_identifier} #{actor.uri} using rsa-sha256 (RSASSA-PKCS1-v1_5 with SHA-256)", signed_string: compare_signed_string, signature: signature_params['signature']
@@ -180,11 +189,18 @@ module SignatureVerification
     nil
   end
 
-  def build_signed_string
+  def build_signed_string(include_query_string: true)
     signed_headers.map do |signed_header|
       case signed_header
       when Request::REQUEST_TARGET
-        "#{Request::REQUEST_TARGET}: #{request.method.downcase} #{request.path}"
+        if include_query_string
+          "#{Request::REQUEST_TARGET}: #{request.method.downcase} #{request.original_fullpath}"
+        else
+          # Current versions of Mastodon incorrectly omit the query string from the (request-target) pseudo-header.
+          # Therefore, temporarily support such incorrect signatures for compatibility.
+          # TODO: remove eventually some time after release of the fixed version
+          "#{Request::REQUEST_TARGET}: #{request.method.downcase} #{request.path}"
+        end
       when '(created)'
         raise SignatureVerificationError, 'Invalid pseudo-header (created) for rsa-sha256' unless signature_algorithm == 'hs2019'
         raise SignatureVerificationError, 'Pseudo-header (created) used but corresponding argument missing' if signature_params['created'].blank?
diff --git a/app/lib/request.rb b/app/lib/request.rb
index 5f128af73..8d4120868 100644
--- a/app/lib/request.rb
+++ b/app/lib/request.rb
@@ -77,6 +77,7 @@ class Request
     @url         = Addressable::URI.parse(url).normalize
     @http_client = options.delete(:http_client)
     @allow_local = options.delete(:allow_local)
+    @full_path   = options.delete(:with_query_string)
     @options     = options.merge(socket_class: use_proxy? || @allow_local ? ProxySocket : Socket)
     @options     = @options.merge(timeout_class: PerOperationWithDeadline, timeout_options: TIMEOUT)
     @options     = @options.merge(proxy_url) if use_proxy?
@@ -146,7 +147,7 @@ class Request
   private
 
   def set_common_headers!
-    @headers[REQUEST_TARGET]    = "#{@verb} #{@url.path}"
+    @headers[REQUEST_TARGET]    = request_target
     @headers['User-Agent']      = Mastodon::Version.user_agent
     @headers['Host']            = @url.host
     @headers['Date']            = Time.now.utc.httpdate
@@ -157,6 +158,14 @@ class Request
     @headers['Digest'] = "SHA-256=#{Digest::SHA256.base64digest(@options[:body])}"
   end
 
+  def request_target
+    if @url.query.nil? || !@full_path
+      "#{@verb} #{@url.path}"
+    else
+      "#{@verb} #{@url.path}?#{@url.query}"
+    end
+  end
+
   def signature
     algorithm = 'rsa-sha256'
     signature = Base64.strict_encode64(@keypair.sign(OpenSSL::Digest.new('SHA256'), signed_string))
diff --git a/spec/requests/signature_verification_spec.rb b/spec/requests/signature_verification_spec.rb
index b753750b8..401828c4a 100644
--- a/spec/requests/signature_verification_spec.rb
+++ b/spec/requests/signature_verification_spec.rb
@@ -94,6 +94,72 @@ describe 'signature verification concern' do
       end
     end
 
+    context 'with a valid signature on a GET request that has a query string' do
+      let(:signature_header) do
+        'keyId="https://remote.domain/users/bob#main-key",algorithm="rsa-sha256",headers="date host (request-target)",signature="SDMa4r/DQYMXYxVgYO2yEqGWWUXugKjVuz0I8dniQAk+aunzBaF2aPu+4grBfawAshlx1Xytl8lhb0H2MllEz16/tKY7rUrb70MK0w8ohXgpb0qs3YvQgdj4X24L1x2MnkFfKHR/J+7TBlnivq0HZqXm8EIkPWLv+eQxu8fbowLwHIVvRd/3t6FzvcfsE0UZKkoMEX02542MhwSif6cu7Ec/clsY9qgKahb9JVGOGS1op9Lvg/9y1mc8KCgD83U5IxVygYeYXaVQ6gixA9NgZiTCwEWzHM5ELm7w5hpdLFYxYOHg/3G3fiqJzpzNQAcCD4S4JxfE7hMI0IzVlNLT6A=="' # rubocop:disable Layout/LineLength
+      end
+
+      it 'successfuly verifies signature', :aggregate_failures do
+        expect(signature_header).to eq build_signature_string(actor_keypair, 'https://remote.domain/users/bob#main-key', 'get /activitypub/success?foo=42', { 'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT', 'Host' => 'www.example.com' })
+
+        get '/activitypub/success?foo=42', headers: {
+          'Host' => 'www.example.com',
+          'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT',
+          'Signature' => signature_header,
+        }
+
+        expect(response).to have_http_status(200)
+        expect(body_as_json).to match(
+          signed_request: true,
+          signature_actor_id: actor.id.to_s
+        )
+      end
+    end
+
+    context 'when the query string is missing from the signature verification (compatibility quirk)' do
+      let(:signature_header) do
+        'keyId="https://remote.domain/users/bob#main-key",algorithm="rsa-sha256",headers="date host (request-target)",signature="Z8ilar3J7bOwqZkMp7sL8sRs4B1FT+UorbmvWoE+A5UeoOJ3KBcUmbsh+k3wQwbP5gMNUrra9rEWabpasZGphLsbDxfbsWL3Cf0PllAc7c1c7AFEwnewtExI83/qqgEkfWc2z7UDutXc2NfgAx89Ox8DXU/fA2GG0jILjB6UpFyNugkY9rg6oI31UnvfVi3R7sr3/x8Ea3I9thPvqI2byF6cojknSpDAwYzeKdngX3TAQEGzFHz3SDWwyp3jeMWfwvVVbM38FxhvAnSumw7YwWW4L7M7h4M68isLimoT3yfCn2ucBVL5Dz8koBpYf/40w7QidClAwCafZQFC29yDOg=="' # rubocop:disable Layout/LineLength
+      end
+
+      it 'successfuly verifies signature', :aggregate_failures do
+        expect(signature_header).to eq build_signature_string(actor_keypair, 'https://remote.domain/users/bob#main-key', 'get /activitypub/success', { 'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT', 'Host' => 'www.example.com' })
+
+        get '/activitypub/success?foo=42', headers: {
+          'Host' => 'www.example.com',
+          'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT',
+          'Signature' => signature_header,
+        }
+
+        expect(response).to have_http_status(200)
+        expect(body_as_json).to match(
+          signed_request: true,
+          signature_actor_id: actor.id.to_s
+        )
+      end
+    end
+
+    context 'with mismatching query string' do
+      let(:signature_header) do
+        'keyId="https://remote.domain/users/bob#main-key",algorithm="rsa-sha256",headers="date host (request-target)",signature="SDMa4r/DQYMXYxVgYO2yEqGWWUXugKjVuz0I8dniQAk+aunzBaF2aPu+4grBfawAshlx1Xytl8lhb0H2MllEz16/tKY7rUrb70MK0w8ohXgpb0qs3YvQgdj4X24L1x2MnkFfKHR/J+7TBlnivq0HZqXm8EIkPWLv+eQxu8fbowLwHIVvRd/3t6FzvcfsE0UZKkoMEX02542MhwSif6cu7Ec/clsY9qgKahb9JVGOGS1op9Lvg/9y1mc8KCgD83U5IxVygYeYXaVQ6gixA9NgZiTCwEWzHM5ELm7w5hpdLFYxYOHg/3G3fiqJzpzNQAcCD4S4JxfE7hMI0IzVlNLT6A=="' # rubocop:disable Layout/LineLength
+      end
+
+      it 'fails to verify signature', :aggregate_failures do
+        expect(signature_header).to eq build_signature_string(actor_keypair, 'https://remote.domain/users/bob#main-key', 'get /activitypub/success?foo=42', { 'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT', 'Host' => 'www.example.com' })
+
+        get '/activitypub/success?foo=43', headers: {
+          'Host' => 'www.example.com',
+          'Date' => 'Wed, 20 Dec 2023 10:00:00 GMT',
+          'Signature' => signature_header,
+        }
+
+        expect(body_as_json).to match(
+          signed_request: true,
+          signature_actor_id: nil,
+          error: anything
+        )
+      end
+    end
+
     context 'with a mismatching path' do
       it 'fails to verify signature', :aggregate_failures do
         get '/activitypub/alternative-path', headers: {

From 499bc716a5dc5b5ef2ca710ae8dd0af8b0028127 Mon Sep 17 00:00:00 2001
From: Jeong Arm <kjwonmail@gmail.com>
Date: Tue, 16 Jan 2024 17:35:54 +0900
Subject: [PATCH 042/117] Ignore RecordNotUnique errors in LinkCrawlWorker
 (#28748)

---
 app/workers/link_crawl_worker.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/workers/link_crawl_worker.rb b/app/workers/link_crawl_worker.rb
index b3d8aa264..c63af1e43 100644
--- a/app/workers/link_crawl_worker.rb
+++ b/app/workers/link_crawl_worker.rb
@@ -7,7 +7,7 @@ class LinkCrawlWorker
 
   def perform(status_id)
     FetchLinkCardService.new.call(Status.find(status_id))
-  rescue ActiveRecord::RecordNotFound
+  rescue ActiveRecord::RecordNotFound, ActiveRecord::RecordNotUnique
     true
   end
 end

From 2dbf176d239720edfc2efd3ae4082830d7137062 Mon Sep 17 00:00:00 2001
From: Jonathan de Jong <jonathandejong02@gmail.com>
Date: Fri, 19 Jan 2024 10:18:21 +0100
Subject: [PATCH 043/117] Retry 401 errors on replies fetching (#28788)

Co-authored-by: Claire <claire.github-309c@sitedethib.com>
---
 app/helpers/jsonld_helper.rb                      | 12 ++++++------
 app/services/activitypub/fetch_replies_service.rb | 15 ++++++++++++++-
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb
index ce3ff094f..b3d0d032c 100644
--- a/app/helpers/jsonld_helper.rb
+++ b/app/helpers/jsonld_helper.rb
@@ -155,7 +155,7 @@ module JsonLdHelper
     end
   end
 
-  def fetch_resource(uri, id, on_behalf_of = nil)
+  def fetch_resource(uri, id, on_behalf_of = nil, request_options: {})
     unless id
       json = fetch_resource_without_id_validation(uri, on_behalf_of)
 
@@ -164,14 +164,14 @@ module JsonLdHelper
       uri = json['id']
     end
 
-    json = fetch_resource_without_id_validation(uri, on_behalf_of)
+    json = fetch_resource_without_id_validation(uri, on_behalf_of, request_options: request_options)
     json.present? && json['id'] == uri ? json : nil
   end
 
-  def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_temporary_error = false)
+  def fetch_resource_without_id_validation(uri, on_behalf_of = nil, raise_on_temporary_error = false, request_options: {})
     on_behalf_of ||= Account.representative
 
-    build_request(uri, on_behalf_of).perform do |response|
+    build_request(uri, on_behalf_of, options: request_options).perform do |response|
       raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error
 
       body_to_json(response.body_with_limit) if response.code == 200
@@ -204,8 +204,8 @@ module JsonLdHelper
     response.code == 501 || ((400...500).cover?(response.code) && ![401, 408, 429].include?(response.code))
   end
 
-  def build_request(uri, on_behalf_of = nil)
-    Request.new(:get, uri).tap do |request|
+  def build_request(uri, on_behalf_of = nil, options: {})
+    Request.new(:get, uri, **options).tap do |request|
       request.on_behalf_of(on_behalf_of) if on_behalf_of
       request.add_headers('Accept' => 'application/activity+json, application/ld+json')
     end
diff --git a/app/services/activitypub/fetch_replies_service.rb b/app/services/activitypub/fetch_replies_service.rb
index b5c7759ec..a9dd327e9 100644
--- a/app/services/activitypub/fetch_replies_service.rb
+++ b/app/services/activitypub/fetch_replies_service.rb
@@ -37,7 +37,20 @@ class ActivityPub::FetchRepliesService < BaseService
     return unless @allow_synchronous_requests
     return if non_matching_uri_hosts?(@account.uri, collection_or_uri)
 
-    fetch_resource_without_id_validation(collection_or_uri, nil, true)
+    # NOTE: For backward compatibility reasons, Mastodon signs outgoing
+    # queries incorrectly by default.
+    #
+    # While this is relevant for all URLs with query strings, this is
+    # the only code path where this happens in practice.
+    #
+    # Therefore, retry with correct signatures if this fails.
+    begin
+      fetch_resource_without_id_validation(collection_or_uri, nil, true)
+    rescue Mastodon::UnexpectedResponseError => e
+      raise unless e.response && e.response.code == 401 && Addressable::URI.parse(collection_or_uri).query.present?
+
+      fetch_resource_without_id_validation(collection_or_uri, nil, true, request_options: { with_query_string: true })
+    end
   end
 
   def filtered_replies

From 6fe2a47357f893096f86866cb37d0bb052573757 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 19 Jan 2024 13:19:49 +0100
Subject: [PATCH 044/117] Add rate-limit of TOTP authentication attempts at
 controller level (#28801)

---
 app/controllers/auth/sessions_controller.rb   | 22 +++++++++++++++++++
 .../two_factor_authentication_concern.rb      |  5 +++++
 config/locales/en.yml                         |  1 +
 .../auth/sessions_controller_spec.rb          | 20 +++++++++++++++++
 4 files changed, 48 insertions(+)

diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb
index 06a3deee2..8212e2e63 100644
--- a/app/controllers/auth/sessions_controller.rb
+++ b/app/controllers/auth/sessions_controller.rb
@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
 class Auth::SessionsController < Devise::SessionsController
+  include Redisable
+
+  MAX_2FA_ATTEMPTS_PER_HOUR = 10
+
   layout 'auth'
 
   skip_before_action :require_no_authentication, only: [:create]
@@ -134,9 +138,23 @@ class Auth::SessionsController < Devise::SessionsController
     session.delete(:attempt_user_updated_at)
   end
 
+  def clear_2fa_attempt_from_user(user)
+    redis.del(second_factor_attempts_key(user))
+  end
+
+  def check_second_factor_rate_limits(user)
+    attempts, = redis.multi do |multi|
+      multi.incr(second_factor_attempts_key(user))
+      multi.expire(second_factor_attempts_key(user), 1.hour)
+    end
+
+    attempts >= MAX_2FA_ATTEMPTS_PER_HOUR
+  end
+
   def on_authentication_success(user, security_measure)
     @on_authentication_success_called = true
 
+    clear_2fa_attempt_from_user(user)
     clear_attempt_from_session
 
     user.update_sign_in!(new_sign_in: true)
@@ -168,4 +186,8 @@ class Auth::SessionsController < Devise::SessionsController
       user_agent: request.user_agent
     )
   end
+
+  def second_factor_attempts_key(user)
+    "2fa_auth_attempts:#{user.id}:#{Time.now.utc.hour}"
+  end
 end
diff --git a/app/controllers/concerns/two_factor_authentication_concern.rb b/app/controllers/concerns/two_factor_authentication_concern.rb
index 9eb45b90d..90fa392a1 100644
--- a/app/controllers/concerns/two_factor_authentication_concern.rb
+++ b/app/controllers/concerns/two_factor_authentication_concern.rb
@@ -65,6 +65,11 @@ module TwoFactorAuthenticationConcern
   end
 
   def authenticate_with_two_factor_via_otp(user)
+    if check_second_factor_rate_limits(user)
+      flash.now[:alert] = I18n.t('users.rate_limited')
+      return prompt_for_two_factor(user)
+    end
+
     if valid_otp_attempt?(user)
       on_authentication_success(user, :otp)
     else
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 71b8f27aa..2b3bfb4c7 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -1826,6 +1826,7 @@ en:
     go_to_sso_account_settings: Go to your identity provider's account settings
     invalid_otp_token: Invalid two-factor code
     otp_lost_help_html: If you lost access to both, you may get in touch with %{email}
+    rate_limited: Too many authentication attempts, try again later.
     seamless_external_login: You are logged in via an external service, so password and e-mail settings are not available.
     signed_in_as: 'Signed in as:'
   verification:
diff --git a/spec/controllers/auth/sessions_controller_spec.rb b/spec/controllers/auth/sessions_controller_spec.rb
index c727a7633..95055ade4 100644
--- a/spec/controllers/auth/sessions_controller_spec.rb
+++ b/spec/controllers/auth/sessions_controller_spec.rb
@@ -263,6 +263,26 @@ RSpec.describe Auth::SessionsController do
           end
         end
 
+        context 'when repeatedly using an invalid TOTP code before using a valid code' do
+          before do
+            stub_const('Auth::SessionsController::MAX_2FA_ATTEMPTS_PER_HOUR', 2)
+          end
+
+          it 'does not log the user in' do
+            # Travel to the beginning of an hour to avoid crossing rate-limit buckets
+            travel_to '2023-12-20T10:00:00Z'
+
+            Auth::SessionsController::MAX_2FA_ATTEMPTS_PER_HOUR.times do
+              post :create, params: { user: { otp_attempt: '1234' } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s }
+              expect(controller.current_user).to be_nil
+            end
+
+            post :create, params: { user: { otp_attempt: user.current_otp } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s }
+            expect(controller.current_user).to be_nil
+            expect(flash[:alert]).to match I18n.t('users.rate_limited')
+          end
+        end
+
         context 'when using a valid OTP' do
           before do
             post :create, params: { user: { otp_attempt: user.current_otp } }, session: { attempt_user_id: user.id, attempt_user_updated_at: user.updated_at.to_s }

From b377f82b1dcc7961f2f5e17b2830c1e61ad7a504 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 19 Jan 2024 13:43:10 +0100
Subject: [PATCH 045/117] Fix processing of compacted single-item JSON-LD
 collections (#28816)

---
 .../fetch_featured_collection_service.rb      |  4 +--
 .../activitypub/fetch_replies_service.rb      |  4 +--
 .../synchronize_followers_service.rb          |  4 +--
 app/services/keys/query_service.rb            |  2 +-
 .../fetch_featured_collection_service_spec.rb | 34 +++++++++++++++++--
 .../activitypub/fetch_replies_service_spec.rb | 12 +++++++
 6 files changed, 51 insertions(+), 9 deletions(-)

diff --git a/app/services/activitypub/fetch_featured_collection_service.rb b/app/services/activitypub/fetch_featured_collection_service.rb
index e8a31dade..d5d52c3cd 100644
--- a/app/services/activitypub/fetch_featured_collection_service.rb
+++ b/app/services/activitypub/fetch_featured_collection_service.rb
@@ -23,9 +23,9 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService
 
     case collection['type']
     when 'Collection', 'CollectionPage'
-      collection['items']
+      as_array(collection['items'])
     when 'OrderedCollection', 'OrderedCollectionPage'
-      collection['orderedItems']
+      as_array(collection['orderedItems'])
     end
   end
 
diff --git a/app/services/activitypub/fetch_replies_service.rb b/app/services/activitypub/fetch_replies_service.rb
index a9dd327e9..e2ecdef16 100644
--- a/app/services/activitypub/fetch_replies_service.rb
+++ b/app/services/activitypub/fetch_replies_service.rb
@@ -26,9 +26,9 @@ class ActivityPub::FetchRepliesService < BaseService
 
     case collection['type']
     when 'Collection', 'CollectionPage'
-      collection['items']
+      as_array(collection['items'])
     when 'OrderedCollection', 'OrderedCollectionPage'
-      collection['orderedItems']
+      as_array(collection['orderedItems'])
     end
   end
 
diff --git a/app/services/activitypub/synchronize_followers_service.rb b/app/services/activitypub/synchronize_followers_service.rb
index 9bd6034a5..a9aab653c 100644
--- a/app/services/activitypub/synchronize_followers_service.rb
+++ b/app/services/activitypub/synchronize_followers_service.rb
@@ -59,9 +59,9 @@ class ActivityPub::SynchronizeFollowersService < BaseService
 
     case collection['type']
     when 'Collection', 'CollectionPage'
-      collection['items']
+      as_array(collection['items'])
     when 'OrderedCollection', 'OrderedCollectionPage'
-      collection['orderedItems']
+      as_array(collection['orderedItems'])
     end
   end
 
diff --git a/app/services/keys/query_service.rb b/app/services/keys/query_service.rb
index 14c9d9205..33e13293f 100644
--- a/app/services/keys/query_service.rb
+++ b/app/services/keys/query_service.rb
@@ -69,7 +69,7 @@ class Keys::QueryService < BaseService
 
     return if json['items'].blank?
 
-    @devices = json['items'].map do |device|
+    @devices = as_array(json['items']).map do |device|
       Device.new(device_id: device['id'], name: device['name'], identity_key: device.dig('identityKey', 'publicKeyBase64'), fingerprint_key: device.dig('fingerprintKey', 'publicKeyBase64'), claim_url: device['claim'])
     end
   rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error => e
diff --git a/spec/services/activitypub/fetch_featured_collection_service_spec.rb b/spec/services/activitypub/fetch_featured_collection_service_spec.rb
index 5975c81a1..a096296ff 100644
--- a/spec/services/activitypub/fetch_featured_collection_service_spec.rb
+++ b/spec/services/activitypub/fetch_featured_collection_service_spec.rb
@@ -31,7 +31,7 @@ RSpec.describe ActivityPub::FetchFeaturedCollectionService, type: :service do
     }
   end
 
-  let(:status_json_pinned_unknown_unreachable) do
+  let(:status_json_pinned_unknown_reachable) do
     {
       '@context': 'https://www.w3.org/ns/activitystreams',
       type: 'Note',
@@ -65,7 +65,7 @@ RSpec.describe ActivityPub::FetchFeaturedCollectionService, type: :service do
       stub_request(:get, 'https://example.com/account/pinned/known').to_return(status: 200, body: Oj.dump(status_json_pinned_known))
       stub_request(:get, 'https://example.com/account/pinned/unknown-inlined').to_return(status: 200, body: Oj.dump(status_json_pinned_unknown_inlined))
       stub_request(:get, 'https://example.com/account/pinned/unknown-unreachable').to_return(status: 404)
-      stub_request(:get, 'https://example.com/account/pinned/unknown-reachable').to_return(status: 200, body: Oj.dump(status_json_pinned_unknown_unreachable))
+      stub_request(:get, 'https://example.com/account/pinned/unknown-reachable').to_return(status: 200, body: Oj.dump(status_json_pinned_unknown_reachable))
 
       subject.call(actor, note: true, hashtag: false)
     end
@@ -103,6 +103,21 @@ RSpec.describe ActivityPub::FetchFeaturedCollectionService, type: :service do
       end
 
       it_behaves_like 'sets pinned posts'
+
+      context 'when there is a single item, with the array compacted away' do
+        let(:items) { 'https://example.com/account/pinned/unknown-reachable' }
+
+        before do
+          stub_request(:get, 'https://example.com/account/pinned/unknown-reachable').to_return(status: 200, body: Oj.dump(status_json_pinned_unknown_reachable))
+          subject.call(actor, note: true, hashtag: false)
+        end
+
+        it 'sets expected posts as pinned posts' do
+          expect(actor.pinned_statuses.pluck(:uri)).to contain_exactly(
+            'https://example.com/account/pinned/unknown-reachable'
+          )
+        end
+      end
     end
 
     context 'when the endpoint is a paginated Collection' do
@@ -124,6 +139,21 @@ RSpec.describe ActivityPub::FetchFeaturedCollectionService, type: :service do
       end
 
       it_behaves_like 'sets pinned posts'
+
+      context 'when there is a single item, with the array compacted away' do
+        let(:items) { 'https://example.com/account/pinned/unknown-reachable' }
+
+        before do
+          stub_request(:get, 'https://example.com/account/pinned/unknown-reachable').to_return(status: 200, body: Oj.dump(status_json_pinned_unknown_reachable))
+          subject.call(actor, note: true, hashtag: false)
+        end
+
+        it 'sets expected posts as pinned posts' do
+          expect(actor.pinned_statuses.pluck(:uri)).to contain_exactly(
+            'https://example.com/account/pinned/unknown-reachable'
+          )
+        end
+      end
     end
   end
 end
diff --git a/spec/services/activitypub/fetch_replies_service_spec.rb b/spec/services/activitypub/fetch_replies_service_spec.rb
index bf8e29676..62841d7d2 100644
--- a/spec/services/activitypub/fetch_replies_service_spec.rb
+++ b/spec/services/activitypub/fetch_replies_service_spec.rb
@@ -34,6 +34,18 @@ RSpec.describe ActivityPub::FetchRepliesService, type: :service do
 
   describe '#call' do
     context 'when the payload is a Collection with inlined replies' do
+      context 'when there is a single reply, with the array compacted away' do
+        let(:items) { 'http://example.com/self-reply-1' }
+
+        it 'queues the expected worker' do
+          allow(FetchReplyWorker).to receive(:push_bulk)
+
+          subject.call(status, payload)
+
+          expect(FetchReplyWorker).to have_received(:push_bulk).with(['http://example.com/self-reply-1'])
+        end
+      end
+
       context 'when passing the collection itself' do
         it 'spawns workers for up to 5 replies on the same server' do
           expect(FetchReplyWorker).to receive(:push_bulk).with(['http://example.com/self-reply-1', 'http://example.com/self-reply-2', 'http://example.com/self-reply-3', 'http://example.com/self-reply-4', 'http://example.com/self-reply-5'])

From 779237f0546b9675e8a6f0b0d3a2dd37d396270f Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 19 Jan 2024 19:52:59 +0100
Subject: [PATCH 046/117] Fix error when processing remote files with unusually
 long names (#28823)

---
 lib/paperclip/response_with_limit_adapter.rb | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/lib/paperclip/response_with_limit_adapter.rb b/lib/paperclip/response_with_limit_adapter.rb
index deb89717a..ff7a938ab 100644
--- a/lib/paperclip/response_with_limit_adapter.rb
+++ b/lib/paperclip/response_with_limit_adapter.rb
@@ -16,7 +16,7 @@ module Paperclip
     private
 
     def cache_current_values
-      @original_filename = filename_from_content_disposition.presence || filename_from_path.presence || 'data'
+      @original_filename = truncated_filename
       @tempfile = copy_to_tempfile(@target)
       @content_type = ContentTypeDetector.new(@tempfile.path).detect
       @size = File.size(@tempfile)
@@ -43,6 +43,13 @@ module Paperclip
       source.response.connection.close
     end
 
+    def truncated_filename
+      filename = filename_from_content_disposition.presence || filename_from_path.presence || 'data'
+      extension = File.extname(filename)
+      basename = File.basename(filename, extension)
+      [basename[...20], extension[..4]].compact_blank.join
+    end
+
     def filename_from_content_disposition
       disposition = @target.response.headers['content-disposition']
       disposition&.match(/filename="([^"]*)"/)&.captures&.first

From c5c464804d7330211709a2be67fd8869000ce635 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Tue, 23 Jan 2024 15:02:19 +0100
Subject: [PATCH 047/117] Update dependency puma to v6.4.2

---
 Gemfile.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 76925a2bf..41045b5e5 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -479,7 +479,7 @@ GEM
     net-smtp (0.3.3)
       net-protocol
     net-ssh (7.1.0)
-    nio4r (2.5.9)
+    nio4r (2.7.0)
     nokogiri (1.15.4)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
@@ -534,7 +534,7 @@ GEM
       premailer (~> 1.7, >= 1.7.9)
     private_address_check (0.5.0)
     public_suffix (5.0.3)
-    puma (6.3.1)
+    puma (6.4.2)
       nio4r (~> 2.0)
     pundit (2.3.0)
       activesupport (>= 3.0.0)

From 7a22999f925728ab8c51a9e1a262307d73a1c156 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Tue, 23 Jan 2024 15:03:45 +0100
Subject: [PATCH 048/117] Bump ruby version to 3.2.3

---
 .ruby-version | 2 +-
 Dockerfile    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ruby-version b/.ruby-version
index be94e6f53..b347b11ea 100644
--- a/.ruby-version
+++ b/.ruby-version
@@ -1 +1 @@
-3.2.2
+3.2.3
diff --git a/Dockerfile b/Dockerfile
index f73bdcf78..980d70509 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -2,7 +2,7 @@
 # This needs to be bookworm-slim because the Ruby image is built on bookworm-slim
 ARG NODE_VERSION="20.6-bookworm-slim"
 
-FROM ghcr.io/moritzheiber/ruby-jemalloc:3.2.2-slim as ruby
+FROM ghcr.io/moritzheiber/ruby-jemalloc:3.2.3-slim as ruby
 FROM node:${NODE_VERSION} as build
 
 COPY --link --from=ruby /opt/ruby /opt/ruby

From 4eb98ef7552835d0b9327c50da2d26b54725b5ae Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 15 Jan 2024 11:45:48 +0100
Subject: [PATCH 049/117] Ignore the devise-two-factor advisory as we have rate
 limits in place (#28733)

---
 .bundler-audit.yml | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 .bundler-audit.yml

diff --git a/.bundler-audit.yml b/.bundler-audit.yml
new file mode 100644
index 000000000..0671df390
--- /dev/null
+++ b/.bundler-audit.yml
@@ -0,0 +1,6 @@
+---
+ignore:
+  # devise-two-factor advisory about brute-forcing TOTP
+  # We have rate-limits on authentication endpoints in place (including second
+  # factor verification) since Mastodon v3.2.0
+  - CVE-2024-0227

From 1ab050eb5201699e3d5d8c1abf673d7cd97c699b Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 24 Jan 2024 14:59:09 +0100
Subject: [PATCH 050/117] Change PostgreSQL version check to check for
 PostgreSQL 10+

---
 lib/tasks/db.rake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake
index e8a64b8fb..f51a2459c 100644
--- a/lib/tasks/db.rake
+++ b/lib/tasks/db.rake
@@ -17,7 +17,7 @@ namespace :db do
 
   task :pre_migration_check do
     version = ActiveRecord::Base.connection.select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
-    abort 'This version of Mastodon requires PostgreSQL 9.5 or newer. Please update PostgreSQL before updating Mastodon' if version < 90_500
+    abort 'This version of Mastodon requires PostgreSQL 10.0 or newer. Please update PostgreSQL before updating Mastodon' if version < 100_000
   end
 
   Rake::Task['db:migrate'].enhance(['db:pre_migration_check'])

From 4633bb8ce09f216170c3f8370a8e48f3a41872cb Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Tue, 23 Jan 2024 14:58:27 +0100
Subject: [PATCH 051/117] Bump version to v4.2.4

---
 CHANGELOG.md            | 26 ++++++++++++++++++++++++++
 docker-compose.yml      |  6 +++---
 lib/mastodon/version.rb |  2 +-
 3 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5f970519e..8c733f9a9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,32 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.2.4] - 2024-01-24
+
+### Fixed
+
+- Fix error when processing remote files with unusually long names ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28823))
+- Fix processing of compacted single-item JSON-LD collections ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28816))
+- Retry 401 errors on replies fetching ([ShadowJonathan](https://github.com/mastodon/mastodon/pull/28788))
+- Fix `RecordNotUnique` errors in LinkCrawlWorker ([tribela](https://github.com/mastodon/mastodon/pull/28748))
+- Fix Mastodon not correctly processing HTTP Signatures with query strings ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28443), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/28476))
+- Fix potential redirection loop of streaming endpoint ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28665))
+- Fix streaming API redirection ignoring the port of `streaming_api_base_url` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28558))
+- Fix error when processing link preview with an array as `inLanguage` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28252))
+- Fix unsupported time zone or locale preventing sign-up ([Gargron](https://github.com/mastodon/mastodon/pull/28035))
+- Fix "Hide these posts from home" list setting not refreshing when switching lists ([brianholley](https://github.com/mastodon/mastodon/pull/27763))
+- Fix missing background behind dismissable banner in web UI ([Gargron](https://github.com/mastodon/mastodon/pull/27479))
+- Fix line wrapping of language selection button with long locale codes ([gunchleoc](https://github.com/mastodon/mastodon/pull/27100), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/27127))
+- Fix `Undo Announce` activity not being sent to non-follower authors ([MitarashiDango](https://github.com/mastodon/mastodon/pull/18482))
+- Fix N+1s because of association preloaders not actually getting called ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28339))
+- Fix empty column explainer getting cropped under certain conditions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28337))
+- Fix `LinkCrawlWorker` error when encountering empty OEmbed response ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28268))
+- Fix call to inefficient `delete_matched` cache method in domain blocks ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28367))
+
+### Security
+
+- Add rate-limit of TOTP authentication attempts at controller level ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28801))
+
 ## [4.2.3] - 2023-12-05
 
 ### Fixed
diff --git a/docker-compose.yml b/docker-compose.yml
index dda71cb09..b096012de 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -56,7 +56,7 @@ services:
 
   web:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.3
+    image: ghcr.io/mastodon/mastodon:v4.2.4
     restart: always
     env_file: .env.production
     command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@@ -77,7 +77,7 @@ services:
 
   streaming:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.3
+    image: ghcr.io/mastodon/mastodon:v4.2.4
     restart: always
     env_file: .env.production
     command: node ./streaming
@@ -95,7 +95,7 @@ services:
 
   sidekiq:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.3
+    image: ghcr.io/mastodon/mastodon:v4.2.4
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq
diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb
index 2e68df474..6382712cd 100644
--- a/lib/mastodon/version.rb
+++ b/lib/mastodon/version.rb
@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      3
+      4
     end
 
     def default_prerelease

From a6641f828b9e6f5806be01754318279c2532ae82 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 1 Feb 2024 15:56:46 +0100
Subject: [PATCH 052/117] Merge pull request from GHSA-3fjr-858r-92rw

* Fix insufficient origin validation

* Bump version to v4.2.5
---
 CHANGELOG.md                                    |  6 ++++++
 SECURITY.md                                     |  4 +---
 .../concerns/signature_verification.rb          |  2 +-
 app/helpers/jsonld_helper.rb                    |  4 ++--
 app/lib/activitypub/activity.rb                 |  2 +-
 app/lib/activitypub/linked_data_signature.rb    |  2 +-
 .../activitypub/fetch_remote_account_service.rb |  2 +-
 .../activitypub/fetch_remote_actor_service.rb   |  6 +++---
 .../activitypub/fetch_remote_key_service.rb     | 17 ++---------------
 .../activitypub/fetch_remote_status_service.rb  |  8 ++++----
 .../activitypub/process_account_service.rb      |  2 +-
 app/services/fetch_resource_service.rb          | 10 +++++++++-
 docker-compose.yml                              |  6 +++---
 lib/mastodon/version.rb                         |  2 +-
 .../activitypub/linked_data_signature_spec.rb   |  4 ++--
 .../fetch_remote_account_service_spec.rb        |  2 +-
 .../fetch_remote_actor_service_spec.rb          |  2 +-
 .../fetch_remote_key_service_spec.rb            |  2 +-
 spec/services/fetch_resource_service_spec.rb    | 10 +++++-----
 spec/services/resolve_url_service_spec.rb       |  1 +
 20 files changed, 47 insertions(+), 47 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8c733f9a9..a34545be1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.2.5] - 2024-02-01
+
+### Security
+
+- Fix insufficient origin validation (CVE-2024-23832, [GHSA-3fjr-858r-92rw](https://github.com/mastodon/mastodon/security/advisories/GHSA-3fjr-858r-92rw))
+
 ## [4.2.4] - 2024-01-24
 
 ### Fixed
diff --git a/SECURITY.md b/SECURITY.md
index 954ff73a2..667096324 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -17,6 +17,4 @@ A "vulnerability in Mastodon" is a vulnerability in the code distributed through
 | ------- | ---------------- |
 | 4.2.x   | Yes              |
 | 4.1.x   | Yes              |
-| 4.0.x   | No               |
-| 3.5.x   | Until 2023-12-31 |
-| < 3.5   | No               |
+| < 4.1   | No               |
diff --git a/app/controllers/concerns/signature_verification.rb b/app/controllers/concerns/signature_verification.rb
index 35391e64c..92f1eb5a1 100644
--- a/app/controllers/concerns/signature_verification.rb
+++ b/app/controllers/concerns/signature_verification.rb
@@ -266,7 +266,7 @@ module SignatureVerification
       stoplight_wrap_request { ResolveAccountService.new.call(key_id.delete_prefix('acct:'), suppress_errors: false) }
     elsif !ActivityPub::TagManager.instance.local_uri?(key_id)
       account   = ActivityPub::TagManager.instance.uri_to_actor(key_id)
-      account ||= stoplight_wrap_request { ActivityPub::FetchRemoteKeyService.new.call(key_id, id: false, suppress_errors: false) }
+      account ||= stoplight_wrap_request { ActivityPub::FetchRemoteKeyService.new.call(key_id, suppress_errors: false) }
       account
     end
   rescue Mastodon::PrivateNetworkAddressError => e
diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb
index b3d0d032c..cc05b7a40 100644
--- a/app/helpers/jsonld_helper.rb
+++ b/app/helpers/jsonld_helper.rb
@@ -155,8 +155,8 @@ module JsonLdHelper
     end
   end
 
-  def fetch_resource(uri, id, on_behalf_of = nil, request_options: {})
-    unless id
+  def fetch_resource(uri, id_is_known, on_behalf_of = nil, request_options: {})
+    unless id_is_known
       json = fetch_resource_without_id_validation(uri, on_behalf_of)
 
       return if !json.is_a?(Hash) || unsupported_uri_scheme?(json['id'])
diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index 51384ef98..322f3e27a 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -154,7 +154,7 @@ class ActivityPub::Activity
     if object_uri.start_with?('http')
       return if ActivityPub::TagManager.instance.local_uri?(object_uri)
 
-      ActivityPub::FetchRemoteStatusService.new.call(object_uri, id: true, on_behalf_of: @account.followers.local.first, request_id: @options[:request_id])
+      ActivityPub::FetchRemoteStatusService.new.call(object_uri, on_behalf_of: @account.followers.local.first, request_id: @options[:request_id])
     elsif @object['url'].present?
       ::FetchRemoteStatusService.new.call(@object['url'], request_id: @options[:request_id])
     end
diff --git a/app/lib/activitypub/linked_data_signature.rb b/app/lib/activitypub/linked_data_signature.rb
index faea63e8f..9459fdd8b 100644
--- a/app/lib/activitypub/linked_data_signature.rb
+++ b/app/lib/activitypub/linked_data_signature.rb
@@ -19,7 +19,7 @@ class ActivityPub::LinkedDataSignature
     return unless type == 'RsaSignature2017'
 
     creator = ActivityPub::TagManager.instance.uri_to_actor(creator_uri)
-    creator = ActivityPub::FetchRemoteKeyService.new.call(creator_uri, id: false) if creator&.public_key.blank?
+    creator = ActivityPub::FetchRemoteKeyService.new.call(creator_uri) if creator&.public_key.blank?
 
     return if creator.nil?
 
diff --git a/app/services/activitypub/fetch_remote_account_service.rb b/app/services/activitypub/fetch_remote_account_service.rb
index 567dd8a14..7b083d889 100644
--- a/app/services/activitypub/fetch_remote_account_service.rb
+++ b/app/services/activitypub/fetch_remote_account_service.rb
@@ -2,7 +2,7 @@
 
 class ActivityPub::FetchRemoteAccountService < ActivityPub::FetchRemoteActorService
   # Does a WebFinger roundtrip on each call, unless `only_key` is true
-  def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true, request_id: nil)
+  def call(uri, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true, request_id: nil)
     actor = super
     return actor if actor.nil? || actor.is_a?(Account)
 
diff --git a/app/services/activitypub/fetch_remote_actor_service.rb b/app/services/activitypub/fetch_remote_actor_service.rb
index 8df8c7587..86a134bb4 100644
--- a/app/services/activitypub/fetch_remote_actor_service.rb
+++ b/app/services/activitypub/fetch_remote_actor_service.rb
@@ -10,15 +10,15 @@ class ActivityPub::FetchRemoteActorService < BaseService
   SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze
 
   # Does a WebFinger roundtrip on each call, unless `only_key` is true
-  def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true, request_id: nil)
+  def call(uri, prefetched_body: nil, break_on_redirect: false, only_key: false, suppress_errors: true, request_id: nil)
     return if domain_not_allowed?(uri)
     return ActivityPub::TagManager.instance.uri_to_actor(uri) if ActivityPub::TagManager.instance.local_uri?(uri)
 
     @json = begin
       if prefetched_body.nil?
-        fetch_resource(uri, id)
+        fetch_resource(uri, true)
       else
-        body_to_json(prefetched_body, compare_id: id ? uri : nil)
+        body_to_json(prefetched_body, compare_id: uri)
       end
     rescue Oj::ParseError
       raise Error, "Error parsing JSON-LD document #{uri}"
diff --git a/app/services/activitypub/fetch_remote_key_service.rb b/app/services/activitypub/fetch_remote_key_service.rb
index 8eb97c1e6..e96b5ad3b 100644
--- a/app/services/activitypub/fetch_remote_key_service.rb
+++ b/app/services/activitypub/fetch_remote_key_service.rb
@@ -6,23 +6,10 @@ class ActivityPub::FetchRemoteKeyService < BaseService
   class Error < StandardError; end
 
   # Returns actor that owns the key
-  def call(uri, id: true, prefetched_body: nil, suppress_errors: true)
+  def call(uri, suppress_errors: true)
     raise Error, 'No key URI given' if uri.blank?
 
-    if prefetched_body.nil?
-      if id
-        @json = fetch_resource_without_id_validation(uri)
-        if actor_type?
-          @json = fetch_resource(@json['id'], true)
-        elsif uri != @json['id']
-          raise Error, "Fetched URI #{uri} has wrong id #{@json['id']}"
-        end
-      else
-        @json = fetch_resource(uri, id)
-      end
-    else
-      @json = body_to_json(prefetched_body, compare_id: id ? uri : nil)
-    end
+    @json = fetch_resource(uri, false)
 
     raise Error, "Unable to fetch key JSON at #{uri}" if @json.nil?
     raise Error, "Unsupported JSON-LD context for document #{uri}" unless supported_context?(@json)
diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb
index a491b32b2..5a3eeeaf4 100644
--- a/app/services/activitypub/fetch_remote_status_service.rb
+++ b/app/services/activitypub/fetch_remote_status_service.rb
@@ -8,14 +8,14 @@ class ActivityPub::FetchRemoteStatusService < BaseService
   DISCOVERIES_PER_REQUEST = 1000
 
   # Should be called when uri has already been checked for locality
-  def call(uri, id: true, prefetched_body: nil, on_behalf_of: nil, expected_actor_uri: nil, request_id: nil)
+  def call(uri, prefetched_body: nil, on_behalf_of: nil, expected_actor_uri: nil, request_id: nil)
     return if domain_not_allowed?(uri)
 
     @request_id = request_id || "#{Time.now.utc.to_i}-status-#{uri}"
     @json = if prefetched_body.nil?
-              fetch_resource(uri, id, on_behalf_of)
+              fetch_resource(uri, true, on_behalf_of)
             else
-              body_to_json(prefetched_body, compare_id: id ? uri : nil)
+              body_to_json(prefetched_body, compare_id: uri)
             end
 
     return unless supported_context?
@@ -65,7 +65,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService
 
   def account_from_uri(uri)
     actor = ActivityPub::TagManager.instance.uri_to_resource(uri, Account)
-    actor = ActivityPub::FetchRemoteAccountService.new.call(uri, id: true, request_id: @request_id) if actor.nil? || actor.possibly_stale?
+    actor = ActivityPub::FetchRemoteAccountService.new.call(uri, request_id: @request_id) if actor.nil? || actor.possibly_stale?
     actor
   end
 
diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index 1304ca824..dc845ce97 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -277,7 +277,7 @@ class ActivityPub::ProcessAccountService < BaseService
 
   def moved_account
     account   = ActivityPub::TagManager.instance.uri_to_resource(@json['movedTo'], Account)
-    account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], id: true, break_on_redirect: true, request_id: @options[:request_id])
+    account ||= ActivityPub::FetchRemoteAccountService.new.call(@json['movedTo'], break_on_redirect: true, request_id: @options[:request_id])
     account
   end
 
diff --git a/app/services/fetch_resource_service.rb b/app/services/fetch_resource_service.rb
index a3406e5a5..71c6cca79 100644
--- a/app/services/fetch_resource_service.rb
+++ b/app/services/fetch_resource_service.rb
@@ -48,7 +48,15 @@ class FetchResourceService < BaseService
       body = response.body_with_limit
       json = body_to_json(body)
 
-      [json['id'], { prefetched_body: body, id: true }] if supported_context?(json) && (equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteActorService::SUPPORTED_TYPES) || expected_type?(json))
+      return unless supported_context?(json) && (equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteActorService::SUPPORTED_TYPES) || expected_type?(json))
+
+      if json['id'] != @url
+        return if terminal
+
+        return process(json['id'], terminal: true)
+      end
+
+      [@url, { prefetched_body: body }]
     elsif !terminal
       link_header = response['Link'] && parse_link_header(response)
 
diff --git a/docker-compose.yml b/docker-compose.yml
index b096012de..8aa93b1c0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -56,7 +56,7 @@ services:
 
   web:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.4
+    image: ghcr.io/mastodon/mastodon:v4.2.5
     restart: always
     env_file: .env.production
     command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@@ -77,7 +77,7 @@ services:
 
   streaming:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.4
+    image: ghcr.io/mastodon/mastodon:v4.2.5
     restart: always
     env_file: .env.production
     command: node ./streaming
@@ -95,7 +95,7 @@ services:
 
   sidekiq:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.4
+    image: ghcr.io/mastodon/mastodon:v4.2.5
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq
diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb
index 6382712cd..f9f801c77 100644
--- a/lib/mastodon/version.rb
+++ b/lib/mastodon/version.rb
@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      4
+      5
     end
 
     def default_prerelease
diff --git a/spec/lib/activitypub/linked_data_signature_spec.rb b/spec/lib/activitypub/linked_data_signature_spec.rb
index 03d0c5a87..e821cee6b 100644
--- a/spec/lib/activitypub/linked_data_signature_spec.rb
+++ b/spec/lib/activitypub/linked_data_signature_spec.rb
@@ -60,7 +60,7 @@ RSpec.describe ActivityPub::LinkedDataSignature do
 
         allow(ActivityPub::FetchRemoteKeyService).to receive(:new).and_return(service_stub)
 
-        allow(service_stub).to receive(:call).with('http://example.com/alice', id: false) do
+        allow(service_stub).to receive(:call).with('http://example.com/alice') do
           sender.update!(public_key: old_key)
           sender
         end
@@ -68,7 +68,7 @@ RSpec.describe ActivityPub::LinkedDataSignature do
 
       it 'fetches key and returns creator' do
         expect(subject.verify_actor!).to eq sender
-        expect(service_stub).to have_received(:call).with('http://example.com/alice', id: false).once
+        expect(service_stub).to have_received(:call).with('http://example.com/alice').once
       end
     end
 
diff --git a/spec/services/activitypub/fetch_remote_account_service_spec.rb b/spec/services/activitypub/fetch_remote_account_service_spec.rb
index ac7484d96..f33a928da 100644
--- a/spec/services/activitypub/fetch_remote_account_service_spec.rb
+++ b/spec/services/activitypub/fetch_remote_account_service_spec.rb
@@ -18,7 +18,7 @@ RSpec.describe ActivityPub::FetchRemoteAccountService, type: :service do
   end
 
   describe '#call' do
-    let(:account) { subject.call('https://example.com/alice', id: true) }
+    let(:account) { subject.call('https://example.com/alice') }
 
     shared_examples 'sets profile data' do
       it 'returns an account' do
diff --git a/spec/services/activitypub/fetch_remote_actor_service_spec.rb b/spec/services/activitypub/fetch_remote_actor_service_spec.rb
index 93d31b69d..944a2f8b1 100644
--- a/spec/services/activitypub/fetch_remote_actor_service_spec.rb
+++ b/spec/services/activitypub/fetch_remote_actor_service_spec.rb
@@ -18,7 +18,7 @@ RSpec.describe ActivityPub::FetchRemoteActorService, type: :service do
   end
 
   describe '#call' do
-    let(:account) { subject.call('https://example.com/alice', id: true) }
+    let(:account) { subject.call('https://example.com/alice') }
 
     shared_examples 'sets profile data' do
       it 'returns an account' do
diff --git a/spec/services/activitypub/fetch_remote_key_service_spec.rb b/spec/services/activitypub/fetch_remote_key_service_spec.rb
index e210d20ec..0b14da4f4 100644
--- a/spec/services/activitypub/fetch_remote_key_service_spec.rb
+++ b/spec/services/activitypub/fetch_remote_key_service_spec.rb
@@ -55,7 +55,7 @@ RSpec.describe ActivityPub::FetchRemoteKeyService, type: :service do
   end
 
   describe '#call' do
-    let(:account) { subject.call(public_key_id, id: false) }
+    let(:account) { subject.call(public_key_id) }
 
     context 'when the key is a sub-object from the actor' do
       before do
diff --git a/spec/services/fetch_resource_service_spec.rb b/spec/services/fetch_resource_service_spec.rb
index 0f1068471..78037a06c 100644
--- a/spec/services/fetch_resource_service_spec.rb
+++ b/spec/services/fetch_resource_service_spec.rb
@@ -57,7 +57,7 @@ RSpec.describe FetchResourceService, type: :service do
 
       let(:json) do
         {
-          id: 1,
+          id: 'http://example.com/foo',
           '@context': ActivityPub::TagManager::CONTEXT,
           type: 'Note',
         }.to_json
@@ -83,27 +83,27 @@ RSpec.describe FetchResourceService, type: :service do
         let(:content_type) { 'application/activity+json; charset=utf-8' }
         let(:body) { json }
 
-        it { is_expected.to eq [1, { prefetched_body: body, id: true }] }
+        it { is_expected.to eq ['http://example.com/foo', { prefetched_body: body }] }
       end
 
       context 'when content type is ld+json with profile' do
         let(:content_type) { 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"' }
         let(:body) { json }
 
-        it { is_expected.to eq [1, { prefetched_body: body, id: true }] }
+        it { is_expected.to eq ['http://example.com/foo', { prefetched_body: body }] }
       end
 
       context 'when link header is present' do
         let(:headers) { { 'Link' => '<http://example.com/foo>; rel="alternate"; type="application/activity+json"' } }
 
-        it { is_expected.to eq [1, { prefetched_body: json, id: true }] }
+        it { is_expected.to eq ['http://example.com/foo', { prefetched_body: json }] }
       end
 
       context 'when content type is text/html' do
         let(:content_type) { 'text/html' }
         let(:body) { '<html><head><link rel="alternate" href="http://example.com/foo" type="application/activity+json"/></head></html>' }
 
-        it { is_expected.to eq [1, { prefetched_body: json, id: true }] }
+        it { is_expected.to eq ['http://example.com/foo', { prefetched_body: json }] }
       end
     end
   end
diff --git a/spec/services/resolve_url_service_spec.rb b/spec/services/resolve_url_service_spec.rb
index 7991aa6ef..38d35a3a1 100644
--- a/spec/services/resolve_url_service_spec.rb
+++ b/spec/services/resolve_url_service_spec.rb
@@ -139,6 +139,7 @@ describe ResolveURLService, type: :service do
         stub_request(:get, url).to_return(status: 302, headers: { 'Location' => status_url })
         body = ActiveModelSerializers::SerializableResource.new(status, serializer: ActivityPub::NoteSerializer, adapter: ActivityPub::Adapter).to_json
         stub_request(:get, status_url).to_return(body: body, headers: { 'Content-Type' => 'application/activity+json' })
+        stub_request(:get, uri).to_return(body: body, headers: { 'Content-Type' => 'application/activity+json' })
       end
 
       it 'returns status by url' do

From b7230cd759719c3cdb83d7d013ccdc897720f186 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 14 Feb 2024 11:02:11 +0100
Subject: [PATCH 053/117] Update dependency nokogiri to 1.16.2

---
 Gemfile.lock | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 41045b5e5..c9c9586d8 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -457,7 +457,7 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2023.0808)
     mini_mime (1.1.5)
-    mini_portile2 (2.8.4)
+    mini_portile2 (2.8.5)
     minitest (5.19.0)
     msgpack (1.7.1)
     multi_json (1.15.0)
@@ -480,7 +480,7 @@ GEM
       net-protocol
     net-ssh (7.1.0)
     nio4r (2.7.0)
-    nokogiri (1.15.4)
+    nokogiri (1.16.2)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     oj (3.16.1)
@@ -539,7 +539,7 @@ GEM
     pundit (2.3.0)
       activesupport (>= 3.0.0)
     raabro (1.4.0)
-    racc (1.7.1)
+    racc (1.7.3)
     rack (2.2.8)
     rack-attack (6.7.0)
       rack (>= 1.0, < 4)

From ae2dce813a9fa40ff1c568e8fd7d6f8f71207e3d Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 14 Feb 2024 11:02:55 +0100
Subject: [PATCH 054/117] Update dependency sidekiq-unique-jobs to 7.1.33

---
 Gemfile.lock | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index c9c9586d8..1c767859b 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -212,7 +212,7 @@ GEM
     climate_control (0.2.0)
     cocoon (1.2.15)
     color_diff (0.1)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.2.3)
     connection_pool (2.4.1)
     cose (1.3.0)
       cbor (~> 0.5.9)
@@ -693,7 +693,7 @@ GEM
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
     semantic_range (3.0.0)
-    sidekiq (6.5.10)
+    sidekiq (6.5.12)
       connection_pool (>= 2.2.5, < 3)
       rack (~> 2.0)
       redis (>= 4.5.0, < 5)
@@ -703,7 +703,7 @@ GEM
       rufus-scheduler (~> 3.2)
       sidekiq (>= 6, < 8)
       tilt (>= 1.4.0)
-    sidekiq-unique-jobs (7.1.29)
+    sidekiq-unique-jobs (7.1.33)
       brpoplpush-redis_script (> 0.1.1, <= 2.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.5)
       redis (< 5.0)
@@ -748,7 +748,7 @@ GEM
     terrapin (0.6.0)
       climate_control (>= 0.0.3, < 1.0)
     test-prof (1.2.3)
-    thor (1.2.2)
+    thor (1.3.0)
     tilt (2.2.0)
     timeout (0.4.0)
     tpm-key_attestation (0.12.0)

From 6d43b63275017738218e1e98744168d260180d78 Mon Sep 17 00:00:00 2001
From: Emelia Smith <ThisIsMissEm@users.noreply.github.com>
Date: Tue, 13 Feb 2024 19:11:47 +0100
Subject: [PATCH 055/117] Disable administrative doorkeeper routes (#29187)

---
 config/initializers/doorkeeper.rb             |  9 +-
 .../requests/disabled_oauth_endpoints_spec.rb | 83 +++++++++++++++++++
 2 files changed, 90 insertions(+), 2 deletions(-)
 create mode 100644 spec/requests/disabled_oauth_endpoints_spec.rb

diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb
index a78405ace..9f06c1120 100644
--- a/config/initializers/doorkeeper.rb
+++ b/config/initializers/doorkeeper.rb
@@ -21,9 +21,14 @@ Doorkeeper.configure do
     user unless user&.otp_required_for_login?
   end
 
-  # If you want to restrict access to the web interface for adding oauth authorized applications, you need to declare the block below.
+  # Doorkeeper provides some administrative interfaces for managing OAuth
+  # Applications, allowing creation, edit, and deletion of applications from the
+  # server. At present, these administrative routes are not integrated into
+  # Mastodon, and as such, we've disabled them by always return a 403 forbidden
+  # response for them. This does not affect the ability for users to manage
+  # their own OAuth Applications.
   admin_authenticator do
-    current_user&.admin? || redirect_to(new_user_session_url)
+    head 403
   end
 
   # Authorization Code expiration time (default 10 minutes).
diff --git a/spec/requests/disabled_oauth_endpoints_spec.rb b/spec/requests/disabled_oauth_endpoints_spec.rb
new file mode 100644
index 000000000..7c2c09f38
--- /dev/null
+++ b/spec/requests/disabled_oauth_endpoints_spec.rb
@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe 'Disabled OAuth routes' do
+  # These routes are disabled via the doorkeeper configuration for
+  # `admin_authenticator`, as these routes should only be accessible by server
+  # administrators. For now, these routes are not properly designed and
+  # integrated into Mastodon, so we're disabling them completely
+  describe 'GET /oauth/applications' do
+    it 'returns 403 forbidden' do
+      get oauth_applications_path
+
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'POST /oauth/applications' do
+    it 'returns 403 forbidden' do
+      post oauth_applications_path
+
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'GET /oauth/applications/new' do
+    it 'returns 403 forbidden' do
+      get new_oauth_application_path
+
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'GET /oauth/applications/:id' do
+    let(:application) { Fabricate(:application, scopes: 'read') }
+
+    it 'returns 403 forbidden' do
+      get oauth_application_path(application)
+
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'PATCH /oauth/applications/:id' do
+    let(:application) { Fabricate(:application, scopes: 'read') }
+
+    it 'returns 403 forbidden' do
+      patch oauth_application_path(application)
+
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'PUT /oauth/applications/:id' do
+    let(:application) { Fabricate(:application, scopes: 'read') }
+
+    it 'returns 403 forbidden' do
+      put oauth_application_path(application)
+
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'DELETE /oauth/applications/:id' do
+    let(:application) { Fabricate(:application, scopes: 'read') }
+
+    it 'returns 403 forbidden' do
+      delete oauth_application_path(application)
+
+      expect(response).to have_http_status(403)
+    end
+  end
+
+  describe 'GET /oauth/applications/:id/edit' do
+    let(:application) { Fabricate(:application, scopes: 'read') }
+
+    it 'returns 403 forbidden' do
+      get edit_oauth_application_path(application)
+
+      expect(response).to have_http_status(403)
+    end
+  end
+end

From 1a33d348d093bd3d9ae099553f4c5a28e3dd8e29 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 14 Feb 2024 13:12:13 +0100
Subject: [PATCH 056/117] Add `sidekiq_unique_jobs:delete_all_locks` task and
 disable `sidekiq-unique-jobs` UI by default (#29199)

---
 config/routes.rb                   |  2 +-
 lib/tasks/sidekiq_unique_jobs.rake | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)
 create mode 100644 lib/tasks/sidekiq_unique_jobs.rake

diff --git a/config/routes.rb b/config/routes.rb
index 379262f4c..f97bd4ef3 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'sidekiq_unique_jobs/web'
+require 'sidekiq_unique_jobs/web' if ENV['ENABLE_SIDEKIQ_UNIQUE_JOBS_UI'] == true
 require 'sidekiq-scheduler/web'
 
 class RedirectWithVary < ActionDispatch::Routing::PathRedirect
diff --git a/lib/tasks/sidekiq_unique_jobs.rake b/lib/tasks/sidekiq_unique_jobs.rake
new file mode 100644
index 000000000..bedc8fe4c
--- /dev/null
+++ b/lib/tasks/sidekiq_unique_jobs.rake
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+namespace :sidekiq_unique_jobs do
+  task delete_all_locks: :environment do
+    digests = SidekiqUniqueJobs::Digests.new
+    digests.delete_by_pattern('*', count: digests.count)
+
+    expiring_digests = SidekiqUniqueJobs::ExpiringDigests.new
+    expiring_digests.delete_by_pattern('*', count: expiring_digests.count)
+  end
+end

From 0b0c7af2c112606c441b6b4083e6638b14b9263c Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 14 Feb 2024 15:15:34 +0100
Subject: [PATCH 057/117] Merge pull request from GHSA-7w3c-p9j8-mq3x

* Ensure destruction of OAuth Applications notifies streaming

Due to doorkeeper using a dependent: delete_all relationship, the destroy of an OAuth Application bypassed the existing AccessTokenExtension callbacks for announcing destructing of access tokens.

* Ensure password resets revoke access to Streaming API

* Improve performance of deleting OAuth tokens

---------

Co-authored-by: Emelia Smith <ThisIsMissEm@users.noreply.github.com>
---
 app/lib/application_extension.rb | 20 ++++++++++++++++++++
 app/models/user.rb               | 10 ++++++++++
 spec/models/user_spec.rb         |  7 +++++++
 3 files changed, 37 insertions(+)

diff --git a/app/lib/application_extension.rb b/app/lib/application_extension.rb
index fb442e2c2..400c51a02 100644
--- a/app/lib/application_extension.rb
+++ b/app/lib/application_extension.rb
@@ -4,14 +4,34 @@ module ApplicationExtension
   extend ActiveSupport::Concern
 
   included do
+    include Redisable
+
     has_many :created_users, class_name: 'User', foreign_key: 'created_by_application_id', inverse_of: :created_by_application
 
     validates :name, length: { maximum: 60 }
     validates :website, url: true, length: { maximum: 2_000 }, if: :website?
     validates :redirect_uri, length: { maximum: 2_000 }
+
+    # The relationship used between Applications and AccessTokens is using
+    # dependent: delete_all, which means the ActiveRecord callback in
+    # AccessTokenExtension is not run, so instead we manually announce to
+    # streaming that these tokens are being deleted.
+    before_destroy :push_to_streaming_api, prepend: true
   end
 
   def confirmation_redirect_uri
     redirect_uri.lines.first.strip
   end
+
+  def push_to_streaming_api
+    # TODO: #28793 Combine into a single topic
+    payload = Oj.dump(event: :kill)
+    access_tokens.in_batches do |tokens|
+      redis.pipelined do |pipeline|
+        tokens.ids.each do |id|
+          pipeline.publish("timeline:access_token:#{id}", payload)
+        end
+      end
+    end
+  end
 end
diff --git a/app/models/user.rb b/app/models/user.rb
index 309a00d1c..ee9097927 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -360,6 +360,16 @@ class User < ApplicationRecord
     Doorkeeper::AccessToken.by_resource_owner(self).in_batches do |batch|
       batch.update_all(revoked_at: Time.now.utc)
       Web::PushSubscription.where(access_token_id: batch).delete_all
+
+      # Revoke each access token for the Streaming API, since `update_all``
+      # doesn't trigger ActiveRecord Callbacks:
+      # TODO: #28793 Combine into a single topic
+      payload = Oj.dump(event: :kill)
+      redis.pipelined do |pipeline|
+        batch.ids.each do |id|
+          pipeline.publish("timeline:access_token:#{id}", payload)
+        end
+      end
     end
   end
 
diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb
index 0dad96cc8..ffd1889cb 100644
--- a/spec/models/user_spec.rb
+++ b/spec/models/user_spec.rb
@@ -438,7 +438,10 @@ RSpec.describe User do
     let!(:access_token) { Fabricate(:access_token, resource_owner_id: user.id) }
     let!(:web_push_subscription) { Fabricate(:web_push_subscription, access_token: access_token) }
 
+    let(:redis_pipeline_stub) { instance_double(Redis::Namespace, publish: nil) }
+
     before do
+      allow(redis).to receive(:pipelined).and_yield(redis_pipeline_stub)
       user.reset_password!
     end
 
@@ -454,6 +457,10 @@ RSpec.describe User do
       expect(Doorkeeper::AccessToken.active_for(user).count).to eq 0
     end
 
+    it 'revokes streaming access for all access tokens' do
+      expect(redis_pipeline_stub).to have_received(:publish).with("timeline:access_token:#{access_token.id}", Oj.dump(event: :kill)).once
+    end
+
     it 'removes push subscriptions' do
       expect(Web::PushSubscription.where(user: user).or(Web::PushSubscription.where(access_token: access_token)).count).to eq 0
     end

From f1700523f15cd714fbb05b4f277a6feb40ab8e4b Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 14 Feb 2024 15:16:07 +0100
Subject: [PATCH 058/117] Merge pull request from GHSA-vm39-j3vx-pch3

* Prevent different identities from a same SSO provider from accessing a same account

* Lock auth provider changes behind `ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH=true`

* Rename methods to avoid confusion between OAuth and OmniAuth
---
 .../auth/omniauth_callbacks_controller.rb     |  2 +-
 app/models/concerns/omniauthable.rb           | 52 ++++++++++++++-----
 app/models/identity.rb                        |  2 +-
 spec/models/identity_spec.rb                  |  6 +--
 spec/requests/omniauth_callbacks_spec.rb      |  2 +-
 5 files changed, 44 insertions(+), 20 deletions(-)

diff --git a/app/controllers/auth/omniauth_callbacks_controller.rb b/app/controllers/auth/omniauth_callbacks_controller.rb
index 4723806b9..7bccac7f6 100644
--- a/app/controllers/auth/omniauth_callbacks_controller.rb
+++ b/app/controllers/auth/omniauth_callbacks_controller.rb
@@ -6,7 +6,7 @@ class Auth::OmniauthCallbacksController < Devise::OmniauthCallbacksController
   def self.provides_callback_for(provider)
     define_method provider do
       @provider = provider
-      @user = User.find_for_oauth(request.env['omniauth.auth'], current_user)
+      @user = User.find_for_omniauth(request.env['omniauth.auth'], current_user)
 
       if @user.persisted?
         record_login_activity
diff --git a/app/models/concerns/omniauthable.rb b/app/models/concerns/omniauthable.rb
index 3983fbcda..9c004a308 100644
--- a/app/models/concerns/omniauthable.rb
+++ b/app/models/concerns/omniauthable.rb
@@ -19,17 +19,18 @@ module Omniauthable
   end
 
   class_methods do
-    def find_for_oauth(auth, signed_in_resource = nil)
+    def find_for_omniauth(auth, signed_in_resource = nil)
       # EOLE-SSO Patch
       auth.uid = (auth.uid[0][:uid] || auth.uid[0][:user]) if auth.uid.is_a? Hashie::Array
-      identity = Identity.find_for_oauth(auth)
+      identity = Identity.find_for_omniauth(auth)
 
       # If a signed_in_resource is provided it always overrides the existing user
       # to prevent the identity being locked with accidentally created accounts.
       # Note that this may leave zombie accounts (with no associated identity) which
       # can be cleaned up at a later date.
       user   = signed_in_resource || identity.user
-      user ||= create_for_oauth(auth)
+      user ||= reattach_for_auth(auth)
+      user ||= create_for_auth(auth)
 
       if identity.user.nil?
         identity.user = user
@@ -39,19 +40,35 @@ module Omniauthable
       user
     end
 
-    def create_for_oauth(auth)
-      # Check if the user exists with provided email. If no email was provided,
+    private
+
+    def reattach_for_auth(auth)
+      # If allowed, check if a user exists with the provided email address,
+      # and return it if they does not have an associated identity with the
+      # current authentication provider.
+
+      # This can be used to provide a choice of alternative auth providers
+      # or provide smooth gradual transition between multiple auth providers,
+      # but this is discouraged because any insecure provider will put *all*
+      # local users at risk, regardless of which provider they registered with.
+
+      return unless ENV['ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH'] == 'true'
+
+      email, email_is_verified = email_from_auth(auth)
+      return unless email_is_verified
+
+      user = User.find_by(email: email)
+      return if user.nil? || Identity.exists?(provider: auth.provider, user_id: user.id)
+
+      user
+    end
+
+    def create_for_auth(auth)
+      # Create a user for the given auth params. If no email was provided,
       # we assign a temporary email and ask the user to verify it on
       # the next step via Auth::SetupController.show
 
-      strategy          = Devise.omniauth_configs[auth.provider.to_sym].strategy
-      assume_verified   = strategy&.security&.assume_email_is_verified
-      email_is_verified = auth.info.verified || auth.info.verified_email || auth.info.email_verified || assume_verified
-      email             = auth.info.verified_email || auth.info.email
-
-      user = User.find_by(email: email) if email_is_verified
-
-      return user unless user.nil?
+      email, email_is_verified = email_from_auth(auth)
 
       user = User.new(user_params_from_auth(email, auth))
 
@@ -66,7 +83,14 @@ module Omniauthable
       user
     end
 
-    private
+    def email_from_auth(auth)
+      strategy          = Devise.omniauth_configs[auth.provider.to_sym].strategy
+      assume_verified   = strategy&.security&.assume_email_is_verified
+      email_is_verified = auth.info.verified || auth.info.verified_email || auth.info.email_verified || assume_verified
+      email             = auth.info.verified_email || auth.info.email
+
+      [email, email_is_verified]
+    end
 
     def user_params_from_auth(email, auth)
       {
diff --git a/app/models/identity.rb b/app/models/identity.rb
index c95a68a6f..77821b78f 100644
--- a/app/models/identity.rb
+++ b/app/models/identity.rb
@@ -17,7 +17,7 @@ class Identity < ApplicationRecord
   validates :uid, presence: true, uniqueness: { scope: :provider }
   validates :provider, presence: true
 
-  def self.find_for_oauth(auth)
+  def self.find_for_omniauth(auth)
     find_or_create_by(uid: auth.uid, provider: auth.provider)
   end
 end
diff --git a/spec/models/identity_spec.rb b/spec/models/identity_spec.rb
index 2fca1e1c1..22c8dbf22 100644
--- a/spec/models/identity_spec.rb
+++ b/spec/models/identity_spec.rb
@@ -3,16 +3,16 @@
 require 'rails_helper'
 
 RSpec.describe Identity do
-  describe '.find_for_oauth' do
+  describe '.find_for_omniauth' do
     let(:auth) { Fabricate(:identity, user: Fabricate(:user)) }
 
     it 'calls .find_or_create_by' do
       expect(described_class).to receive(:find_or_create_by).with(uid: auth.uid, provider: auth.provider)
-      described_class.find_for_oauth(auth)
+      described_class.find_for_omniauth(auth)
     end
 
     it 'returns an instance of Identity' do
-      expect(described_class.find_for_oauth(auth)).to be_instance_of described_class
+      expect(described_class.find_for_omniauth(auth)).to be_instance_of described_class
     end
   end
 end
diff --git a/spec/requests/omniauth_callbacks_spec.rb b/spec/requests/omniauth_callbacks_spec.rb
index 27aa5ec50..c40ef14c9 100644
--- a/spec/requests/omniauth_callbacks_spec.rb
+++ b/spec/requests/omniauth_callbacks_spec.rb
@@ -96,7 +96,7 @@ describe 'OmniAuth callbacks' do
 
     context 'when a user cannot be built' do
       before do
-        allow(User).to receive(:find_for_oauth).and_return(User.new)
+        allow(User).to receive(:find_for_omniauth).and_return(User.new)
       end
 
       it 'redirects to the new user signup page' do

From 7c8ca0c6d612329523fc4e468e82fc3e51634c1c Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 14 Feb 2024 13:29:38 +0100
Subject: [PATCH 059/117] Bump version to v4.2.6

---
 CHANGELOG.md            | 19 +++++++++++++++++++
 docker-compose.yml      |  6 +++---
 lib/mastodon/version.rb |  2 +-
 3 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a34545be1..14749caa3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,25 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.2.6] - 2024-02-14
+
+### Security
+
+- Update the `sidekiq-unique-jobs` dependency (see [GHSA-cmh9-rx85-xj38](https://github.com/mhenrixon/sidekiq-unique-jobs/security/advisories/GHSA-cmh9-rx85-xj38))
+  In addition, we have disabled the web interface for `sidekiq-unique-jobs` out of caution.
+  If you need it, you can re-enable it by setting `ENABLE_SIDEKIQ_UNIQUE_JOBS_UI=true`.
+  If you only need to clear all locks, you can now use `bundle exec rake sidekiq_unique_jobs:delete_all_locks`.
+- Update the `nokogiri` dependency (see [GHSA-xc9x-jj77-9p9j](https://github.com/sparklemotion/nokogiri/security/advisories/GHSA-xc9x-jj77-9p9j))
+- Disable administrative Doorkeeper routes ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/29187))
+- Fix ongoing streaming sessions not being invalidated when applications get deleted in some cases ([GHSA-7w3c-p9j8-mq3x](https://github.com/mastodon/mastodon/security/advisories/GHSA-7w3c-p9j8-mq3x))
+  In some rare cases, the streaming server was not notified of access tokens revocation on application deletion.
+- Change external authentication behavior to never reattach a new identity to an existing user by default ([GHSA-vm39-j3vx-pch3](https://github.com/mastodon/mastodon/security/advisories/GHSA-vm39-j3vx-pch3))
+  Up until now, Mastodon has allowed new identities from external authentication providers to attach to an existing local user based on their verified e-mail address.
+  This allowed upgrading users from a database-stored password to an external authentication provider, or move from one authentication provider to another.
+  However, this behavior may be unexpected, and means that when multiple authentication providers are configured, the overall security would be that of the least secure authentication provider.
+  For these reasons, this behavior is now locked under the `ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH` environment variable.
+  In addition, regardless of this environment variable, Mastodon will refuse to attach two identities from the same authentication provider to the same account.
+
 ## [4.2.5] - 2024-02-01
 
 ### Security
diff --git a/docker-compose.yml b/docker-compose.yml
index 8aa93b1c0..2f03db04d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -56,7 +56,7 @@ services:
 
   web:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.5
+    image: ghcr.io/mastodon/mastodon:v4.2.6
     restart: always
     env_file: .env.production
     command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@@ -77,7 +77,7 @@ services:
 
   streaming:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.5
+    image: ghcr.io/mastodon/mastodon:v4.2.6
     restart: always
     env_file: .env.production
     command: node ./streaming
@@ -95,7 +95,7 @@ services:
 
   sidekiq:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.5
+    image: ghcr.io/mastodon/mastodon:v4.2.6
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq
diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb
index f9f801c77..2256b20a6 100644
--- a/lib/mastodon/version.rb
+++ b/lib/mastodon/version.rb
@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      5
+      6
     end
 
     def default_prerelease

From 76a37bd040861549757714d2c1df733feec2f3bc Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 14 Feb 2024 15:57:49 +0100
Subject: [PATCH 060/117] Fix OmniAuth tests (#29201)

---
 spec/requests/omniauth_callbacks_spec.rb | 35 ++++++++++++++++++------
 1 file changed, 26 insertions(+), 9 deletions(-)

diff --git a/spec/requests/omniauth_callbacks_spec.rb b/spec/requests/omniauth_callbacks_spec.rb
index c40ef14c9..6bbe8c5a7 100644
--- a/spec/requests/omniauth_callbacks_spec.rb
+++ b/spec/requests/omniauth_callbacks_spec.rb
@@ -39,16 +39,33 @@ describe 'OmniAuth callbacks' do
           Fabricate(:user, email: 'user@host.example')
         end
 
-        it 'matches the existing user, creates an identity, and redirects to root path' do
-          expect { subject }
-            .to not_change(User, :count)
-            .and change(Identity, :count)
-            .by(1)
-            .and change(LoginActivity, :count)
-            .by(1)
+        context 'when ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH is set to true' do
+          around do |example|
+            ClimateControl.modify ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH: 'true' do
+              example.run
+            end
+          end
 
-          expect(Identity.find_by(user: User.last).uid).to eq('123')
-          expect(response).to redirect_to(root_path)
+          it 'matches the existing user, creates an identity, and redirects to root path' do
+            expect { subject }
+              .to not_change(User, :count)
+              .and change(Identity, :count)
+              .by(1)
+              .and change(LoginActivity, :count)
+              .by(1)
+
+            expect(Identity.find_by(user: User.last).uid).to eq('123')
+            expect(response).to redirect_to(root_path)
+          end
+        end
+
+        context 'when ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH is not set to true' do
+          it 'does not match the existing user or create an identity' do
+            expect { subject }
+              .to not_change(User, :count)
+              .and not_change(Identity, :count)
+              .and not_change(LoginActivity, :count)
+          end
         end
       end
 

From 870ee80fd366c54ee2a7fc1508d7588d3fdeb878 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 14 Feb 2024 22:49:45 +0100
Subject: [PATCH 061/117] Fix user creation failure handling in OAuth paths
 (#29207)

---
 app/controllers/auth/omniauth_callbacks_controller.rb | 3 +++
 config/locales/devise.en.yml                          | 1 +
 spec/requests/omniauth_callbacks_spec.rb              | 4 +++-
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/app/controllers/auth/omniauth_callbacks_controller.rb b/app/controllers/auth/omniauth_callbacks_controller.rb
index 7bccac7f6..b8570d0bf 100644
--- a/app/controllers/auth/omniauth_callbacks_controller.rb
+++ b/app/controllers/auth/omniauth_callbacks_controller.rb
@@ -16,6 +16,9 @@ class Auth::OmniauthCallbacksController < Devise::OmniauthCallbacksController
         session["devise.#{provider}_data"] = request.env['omniauth.auth']
         redirect_to new_user_registration_url
       end
+    rescue ActiveRecord::RecordInvalid
+      flash[:alert] = I18n.t('devise.failure.omniauth_user_creation_failure') if is_navigational_format?
+      redirect_to new_user_session_url
     end
   end
 
diff --git a/config/locales/devise.en.yml b/config/locales/devise.en.yml
index eef821481..b7c060016 100644
--- a/config/locales/devise.en.yml
+++ b/config/locales/devise.en.yml
@@ -12,6 +12,7 @@ en:
       last_attempt: You have one more attempt before your account is locked.
       locked: Your account is locked.
       not_found_in_database: Invalid %{authentication_keys} or password.
+      omniauth_user_creation_failure: Error creating an account for this identity.
       pending: Your account is still under review.
       timeout: Your session expired. Please login again to continue.
       unauthenticated: You need to login or sign up before continuing.
diff --git a/spec/requests/omniauth_callbacks_spec.rb b/spec/requests/omniauth_callbacks_spec.rb
index 6bbe8c5a7..6381bf066 100644
--- a/spec/requests/omniauth_callbacks_spec.rb
+++ b/spec/requests/omniauth_callbacks_spec.rb
@@ -60,11 +60,13 @@ describe 'OmniAuth callbacks' do
         end
 
         context 'when ALLOW_UNSAFE_AUTH_PROVIDER_REATTACH is not set to true' do
-          it 'does not match the existing user or create an identity' do
+          it 'does not match the existing user or create an identity, and redirects to login page' do
             expect { subject }
               .to not_change(User, :count)
               .and not_change(Identity, :count)
               .and not_change(LoginActivity, :count)
+
+            expect(response).to redirect_to(new_user_session_url)
           end
         end
       end

From e4ec4ce217e4f1c10cbae5df75a8c22d813b9ba5 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 14 Feb 2024 23:27:02 +0100
Subject: [PATCH 062/117] Update `nsa` gem to version 0.3.0 (#29065) (#29206)

Co-authored-by: Matt Jankowski <matt@jankowski.online>
---
 Gemfile      |  2 +-
 Gemfile.lock | 18 ++++++------------
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/Gemfile b/Gemfile
index 6f20ff25a..829e7d857 100644
--- a/Gemfile
+++ b/Gemfile
@@ -61,7 +61,7 @@ gem 'kaminari', '~> 1.2'
 gem 'link_header', '~> 0.0'
 gem 'mime-types', '~> 3.5.0', require: 'mime/types/columnar'
 gem 'nokogiri', '~> 1.15'
-gem 'nsa', github: 'jhawthorn/nsa', ref: 'e020fcc3a54d993ab45b7194d89ab720296c111b'
+gem 'nsa'
 gem 'oj', '~> 3.14'
 gem 'ox', '~> 2.14'
 gem 'parslet'
diff --git a/Gemfile.lock b/Gemfile.lock
index 1c767859b..37553641c 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -7,17 +7,6 @@ GIT
       hkdf (~> 0.2)
       jwt (~> 2.0)
 
-GIT
-  remote: https://github.com/jhawthorn/nsa.git
-  revision: e020fcc3a54d993ab45b7194d89ab720296c111b
-  ref: e020fcc3a54d993ab45b7194d89ab720296c111b
-  specs:
-    nsa (0.2.8)
-      activesupport (>= 4.2, < 7.2)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
-      sidekiq (>= 3.5)
-      statsd-ruby (~> 1.4, >= 1.4.0)
-
 GIT
   remote: https://github.com/mastodon/rails-settings-cached.git
   revision: 86328ef0bd04ce21cc0504ff5e334591e8c2ccab
@@ -483,6 +472,11 @@ GEM
     nokogiri (1.16.2)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
+    nsa (0.3.0)
+      activesupport (>= 4.2, < 7.2)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      sidekiq (>= 3.5)
+      statsd-ruby (~> 1.4, >= 1.4.0)
     oj (3.16.1)
     omniauth (2.1.1)
       hashie (>= 3.4.6)
@@ -884,7 +878,7 @@ DEPENDENCIES
   net-http (~> 0.3.2)
   net-ldap (~> 0.18)
   nokogiri (~> 1.15)
-  nsa!
+  nsa
   oj (~> 3.14)
   omniauth (~> 2.0)
   omniauth-cas!

From 684f99908f8beab6c002aac5871f5165c583a5ac Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 16 Feb 2024 09:19:35 +0100
Subject: [PATCH 063/117] Update dependency pg to 1.5.5

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 37553641c..134e41a7e 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -514,7 +514,7 @@ GEM
     parslet (2.0.0)
     pastel (0.8.0)
       tty-color (~> 0.5)
-    pg (1.5.4)
+    pg (1.5.5)
     pghero (3.3.4)
       activerecord (>= 6)
     posix-spawn (0.3.15)

From 15de5202012a2f9c4417724987e54bc885051c0d Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 16 Feb 2024 11:56:12 +0100
Subject: [PATCH 064/117] Merge pull request from GHSA-jhrq-qvrm-qr36

* Fix insufficient Content-Type checking of fetched ActivityStreams objects

* Allow JSON-LD documents with multiple profiles
---
 app/helpers/jsonld_helper.rb                     | 14 +++++++++++++-
 app/services/fetch_resource_service.rb           |  2 +-
 spec/helpers/jsonld_helper_spec.rb               | 14 +++++++-------
 spec/lib/activitypub/activity/announce_spec.rb   |  4 ++--
 .../fetch_featured_collection_service_spec.rb    | 16 ++++++++--------
 ...etch_featured_tags_collection_service_spec.rb |  8 ++++----
 .../fetch_remote_account_service_spec.rb         | 10 +++++-----
 .../fetch_remote_actor_service_spec.rb           | 10 +++++-----
 .../activitypub/fetch_remote_key_service_spec.rb |  8 ++++----
 .../activitypub/fetch_replies_service_spec.rb    |  6 +++---
 .../synchronize_followers_service_spec.rb        |  6 +++---
 .../activitypub/fetch_replies_worker_spec.rb     |  2 +-
 12 files changed, 56 insertions(+), 44 deletions(-)

diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb
index cc05b7a40..b0f2077db 100644
--- a/app/helpers/jsonld_helper.rb
+++ b/app/helpers/jsonld_helper.rb
@@ -174,7 +174,19 @@ module JsonLdHelper
     build_request(uri, on_behalf_of, options: request_options).perform do |response|
       raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response) || !raise_on_temporary_error
 
-      body_to_json(response.body_with_limit) if response.code == 200
+      body_to_json(response.body_with_limit) if response.code == 200 && valid_activitypub_content_type?(response)
+    end
+  end
+
+  def valid_activitypub_content_type?(response)
+    return true if response.mime_type == 'application/activity+json'
+
+    # When the mime type is `application/ld+json`, we need to check the profile,
+    # but `http.rb` does not parse it for us.
+    return false unless response.mime_type == 'application/ld+json'
+
+    response.headers[HTTP::Headers::CONTENT_TYPE]&.split(';')&.map(&:strip)&.any? do |str|
+      str.start_with?('profile="') && str[9...-1].split.include?('https://www.w3.org/ns/activitystreams')
     end
   end
 
diff --git a/app/services/fetch_resource_service.rb b/app/services/fetch_resource_service.rb
index 71c6cca79..84c36f6a1 100644
--- a/app/services/fetch_resource_service.rb
+++ b/app/services/fetch_resource_service.rb
@@ -44,7 +44,7 @@ class FetchResourceService < BaseService
     @response_code = response.code
     return nil if response.code != 200
 
-    if ['application/activity+json', 'application/ld+json'].include?(response.mime_type)
+    if valid_activitypub_content_type?(response)
       body = response.body_with_limit
       json = body_to_json(body)
 
diff --git a/spec/helpers/jsonld_helper_spec.rb b/spec/helpers/jsonld_helper_spec.rb
index 3575bba85..6367492b9 100644
--- a/spec/helpers/jsonld_helper_spec.rb
+++ b/spec/helpers/jsonld_helper_spec.rb
@@ -56,15 +56,15 @@ describe JsonLdHelper do
   describe '#fetch_resource' do
     context 'when the second argument is false' do
       it 'returns resource even if the retrieved ID and the given URI does not match' do
-        stub_request(:get, 'https://bob.test/').to_return body: '{"id": "https://alice.test/"}'
-        stub_request(:get, 'https://alice.test/').to_return body: '{"id": "https://alice.test/"}'
+        stub_request(:get, 'https://bob.test/').to_return(body: '{"id": "https://alice.test/"}', headers: { 'Content-Type': 'application/activity+json' })
+        stub_request(:get, 'https://alice.test/').to_return(body: '{"id": "https://alice.test/"}', headers: { 'Content-Type': 'application/activity+json' })
 
         expect(fetch_resource('https://bob.test/', false)).to eq({ 'id' => 'https://alice.test/' })
       end
 
       it 'returns nil if the object identified by the given URI and the object identified by the retrieved ID does not match' do
-        stub_request(:get, 'https://mallory.test/').to_return body: '{"id": "https://marvin.test/"}'
-        stub_request(:get, 'https://marvin.test/').to_return body: '{"id": "https://alice.test/"}'
+        stub_request(:get, 'https://mallory.test/').to_return(body: '{"id": "https://marvin.test/"}', headers: { 'Content-Type': 'application/activity+json' })
+        stub_request(:get, 'https://marvin.test/').to_return(body: '{"id": "https://alice.test/"}', headers: { 'Content-Type': 'application/activity+json' })
 
         expect(fetch_resource('https://mallory.test/', false)).to be_nil
       end
@@ -72,7 +72,7 @@ describe JsonLdHelper do
 
     context 'when the second argument is true' do
       it 'returns nil if the retrieved ID and the given URI does not match' do
-        stub_request(:get, 'https://mallory.test/').to_return body: '{"id": "https://alice.test/"}'
+        stub_request(:get, 'https://mallory.test/').to_return(body: '{"id": "https://alice.test/"}', headers: { 'Content-Type': 'application/activity+json' })
         expect(fetch_resource('https://mallory.test/', true)).to be_nil
       end
     end
@@ -80,12 +80,12 @@ describe JsonLdHelper do
 
   describe '#fetch_resource_without_id_validation' do
     it 'returns nil if the status code is not 200' do
-      stub_request(:get, 'https://host.test/').to_return status: 400, body: '{}'
+      stub_request(:get, 'https://host.test/').to_return(status: 400, body: '{}', headers: { 'Content-Type': 'application/activity+json' })
       expect(fetch_resource_without_id_validation('https://host.test/')).to be_nil
     end
 
     it 'returns hash' do
-      stub_request(:get, 'https://host.test/').to_return status: 200, body: '{}'
+      stub_request(:get, 'https://host.test/').to_return(status: 200, body: '{}', headers: { 'Content-Type': 'application/activity+json' })
       expect(fetch_resource_without_id_validation('https://host.test/')).to eq({})
     end
   end
diff --git a/spec/lib/activitypub/activity/announce_spec.rb b/spec/lib/activitypub/activity/announce_spec.rb
index 8ad892975..b556bfd6c 100644
--- a/spec/lib/activitypub/activity/announce_spec.rb
+++ b/spec/lib/activitypub/activity/announce_spec.rb
@@ -35,7 +35,7 @@ RSpec.describe ActivityPub::Activity::Announce do
     context 'when sender is followed by a local account' do
       before do
         Fabricate(:account).follow!(sender)
-        stub_request(:get, 'https://example.com/actor/hello-world').to_return(body: Oj.dump(unknown_object_json))
+        stub_request(:get, 'https://example.com/actor/hello-world').to_return(body: Oj.dump(unknown_object_json), headers: { 'Content-Type': 'application/activity+json' })
         subject.perform
       end
 
@@ -120,7 +120,7 @@ RSpec.describe ActivityPub::Activity::Announce do
       let(:object_json) { 'https://example.com/actor/hello-world' }
 
       before do
-        stub_request(:get, 'https://example.com/actor/hello-world').to_return(body: Oj.dump(unknown_object_json))
+        stub_request(:get, 'https://example.com/actor/hello-world').to_return(body: Oj.dump(unknown_object_json), headers: { 'Content-Type': 'application/activity+json' })
       end
 
       context 'when the relay is enabled' do
diff --git a/spec/services/activitypub/fetch_featured_collection_service_spec.rb b/spec/services/activitypub/fetch_featured_collection_service_spec.rb
index a096296ff..583212c37 100644
--- a/spec/services/activitypub/fetch_featured_collection_service_spec.rb
+++ b/spec/services/activitypub/fetch_featured_collection_service_spec.rb
@@ -62,10 +62,10 @@ RSpec.describe ActivityPub::FetchFeaturedCollectionService, type: :service do
 
   shared_examples 'sets pinned posts' do
     before do
-      stub_request(:get, 'https://example.com/account/pinned/known').to_return(status: 200, body: Oj.dump(status_json_pinned_known))
-      stub_request(:get, 'https://example.com/account/pinned/unknown-inlined').to_return(status: 200, body: Oj.dump(status_json_pinned_unknown_inlined))
+      stub_request(:get, 'https://example.com/account/pinned/known').to_return(status: 200, body: Oj.dump(status_json_pinned_known), headers: { 'Content-Type': 'application/activity+json' })
+      stub_request(:get, 'https://example.com/account/pinned/unknown-inlined').to_return(status: 200, body: Oj.dump(status_json_pinned_unknown_inlined), headers: { 'Content-Type': 'application/activity+json' })
       stub_request(:get, 'https://example.com/account/pinned/unknown-unreachable').to_return(status: 404)
-      stub_request(:get, 'https://example.com/account/pinned/unknown-reachable').to_return(status: 200, body: Oj.dump(status_json_pinned_unknown_reachable))
+      stub_request(:get, 'https://example.com/account/pinned/unknown-reachable').to_return(status: 200, body: Oj.dump(status_json_pinned_unknown_reachable), headers: { 'Content-Type': 'application/activity+json' })
 
       subject.call(actor, note: true, hashtag: false)
     end
@@ -82,7 +82,7 @@ RSpec.describe ActivityPub::FetchFeaturedCollectionService, type: :service do
   describe '#call' do
     context 'when the endpoint is a Collection' do
       before do
-        stub_request(:get, actor.featured_collection_url).to_return(status: 200, body: Oj.dump(payload))
+        stub_request(:get, actor.featured_collection_url).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' })
       end
 
       it_behaves_like 'sets pinned posts'
@@ -99,7 +99,7 @@ RSpec.describe ActivityPub::FetchFeaturedCollectionService, type: :service do
       end
 
       before do
-        stub_request(:get, actor.featured_collection_url).to_return(status: 200, body: Oj.dump(payload))
+        stub_request(:get, actor.featured_collection_url).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' })
       end
 
       it_behaves_like 'sets pinned posts'
@@ -108,7 +108,7 @@ RSpec.describe ActivityPub::FetchFeaturedCollectionService, type: :service do
         let(:items) { 'https://example.com/account/pinned/unknown-reachable' }
 
         before do
-          stub_request(:get, 'https://example.com/account/pinned/unknown-reachable').to_return(status: 200, body: Oj.dump(status_json_pinned_unknown_reachable))
+          stub_request(:get, 'https://example.com/account/pinned/unknown-reachable').to_return(status: 200, body: Oj.dump(status_json_pinned_unknown_reachable), headers: { 'Content-Type': 'application/activity+json' })
           subject.call(actor, note: true, hashtag: false)
         end
 
@@ -135,7 +135,7 @@ RSpec.describe ActivityPub::FetchFeaturedCollectionService, type: :service do
       end
 
       before do
-        stub_request(:get, actor.featured_collection_url).to_return(status: 200, body: Oj.dump(payload))
+        stub_request(:get, actor.featured_collection_url).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' })
       end
 
       it_behaves_like 'sets pinned posts'
@@ -144,7 +144,7 @@ RSpec.describe ActivityPub::FetchFeaturedCollectionService, type: :service do
         let(:items) { 'https://example.com/account/pinned/unknown-reachable' }
 
         before do
-          stub_request(:get, 'https://example.com/account/pinned/unknown-reachable').to_return(status: 200, body: Oj.dump(status_json_pinned_unknown_reachable))
+          stub_request(:get, 'https://example.com/account/pinned/unknown-reachable').to_return(status: 200, body: Oj.dump(status_json_pinned_unknown_reachable), headers: { 'Content-Type': 'application/activity+json' })
           subject.call(actor, note: true, hashtag: false)
         end
 
diff --git a/spec/services/activitypub/fetch_featured_tags_collection_service_spec.rb b/spec/services/activitypub/fetch_featured_tags_collection_service_spec.rb
index 071e4d92d..638278a10 100644
--- a/spec/services/activitypub/fetch_featured_tags_collection_service_spec.rb
+++ b/spec/services/activitypub/fetch_featured_tags_collection_service_spec.rb
@@ -38,7 +38,7 @@ RSpec.describe ActivityPub::FetchFeaturedTagsCollectionService, type: :service d
   describe '#call' do
     context 'when the endpoint is a Collection' do
       before do
-        stub_request(:get, collection_url).to_return(status: 200, body: Oj.dump(payload))
+        stub_request(:get, collection_url).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' })
       end
 
       it_behaves_like 'sets featured tags'
@@ -46,7 +46,7 @@ RSpec.describe ActivityPub::FetchFeaturedTagsCollectionService, type: :service d
 
     context 'when the account already has featured tags' do
       before do
-        stub_request(:get, collection_url).to_return(status: 200, body: Oj.dump(payload))
+        stub_request(:get, collection_url).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' })
 
         actor.featured_tags.create!(name: 'FoO')
         actor.featured_tags.create!(name: 'baz')
@@ -67,7 +67,7 @@ RSpec.describe ActivityPub::FetchFeaturedTagsCollectionService, type: :service d
       end
 
       before do
-        stub_request(:get, collection_url).to_return(status: 200, body: Oj.dump(payload))
+        stub_request(:get, collection_url).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' })
       end
 
       it_behaves_like 'sets featured tags'
@@ -88,7 +88,7 @@ RSpec.describe ActivityPub::FetchFeaturedTagsCollectionService, type: :service d
       end
 
       before do
-        stub_request(:get, collection_url).to_return(status: 200, body: Oj.dump(payload))
+        stub_request(:get, collection_url).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' })
       end
 
       it_behaves_like 'sets featured tags'
diff --git a/spec/services/activitypub/fetch_remote_account_service_spec.rb b/spec/services/activitypub/fetch_remote_account_service_spec.rb
index f33a928da..42badde05 100644
--- a/spec/services/activitypub/fetch_remote_account_service_spec.rb
+++ b/spec/services/activitypub/fetch_remote_account_service_spec.rb
@@ -44,7 +44,7 @@ RSpec.describe ActivityPub::FetchRemoteAccountService, type: :service do
       before do
         actor[:inbox] = nil
 
-        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor))
+        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor), headers: { 'Content-Type': 'application/activity+json' })
         stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
       end
 
@@ -67,7 +67,7 @@ RSpec.describe ActivityPub::FetchRemoteAccountService, type: :service do
       let!(:webfinger) { { subject: 'acct:alice@example.com', links: [{ rel: 'self', href: 'https://example.com/alice' }] } }
 
       before do
-        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor))
+        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor), headers: { 'Content-Type': 'application/activity+json' })
         stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
       end
 
@@ -93,7 +93,7 @@ RSpec.describe ActivityPub::FetchRemoteAccountService, type: :service do
       let!(:webfinger) { { subject: 'acct:alice@iscool.af', links: [{ rel: 'self', href: 'https://example.com/alice' }] } }
 
       before do
-        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor))
+        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor), headers: { 'Content-Type': 'application/activity+json' })
         stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
         stub_request(:get, 'https://iscool.af/.well-known/webfinger?resource=acct:alice@iscool.af').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
       end
@@ -125,7 +125,7 @@ RSpec.describe ActivityPub::FetchRemoteAccountService, type: :service do
       let!(:webfinger) { { subject: 'acct:alice@example.com', links: [{ rel: 'self', href: 'https://example.com/bob' }] } }
 
       before do
-        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor))
+        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor), headers: { 'Content-Type': 'application/activity+json' })
         stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
       end
 
@@ -148,7 +148,7 @@ RSpec.describe ActivityPub::FetchRemoteAccountService, type: :service do
       let!(:webfinger) { { subject: 'acct:alice@iscool.af', links: [{ rel: 'self', href: 'https://example.com/bob' }] } }
 
       before do
-        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor))
+        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor), headers: { 'Content-Type': 'application/activity+json' })
         stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
         stub_request(:get, 'https://iscool.af/.well-known/webfinger?resource=acct:alice@iscool.af').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
       end
diff --git a/spec/services/activitypub/fetch_remote_actor_service_spec.rb b/spec/services/activitypub/fetch_remote_actor_service_spec.rb
index 944a2f8b1..6d264b7b8 100644
--- a/spec/services/activitypub/fetch_remote_actor_service_spec.rb
+++ b/spec/services/activitypub/fetch_remote_actor_service_spec.rb
@@ -44,7 +44,7 @@ RSpec.describe ActivityPub::FetchRemoteActorService, type: :service do
       before do
         actor[:inbox] = nil
 
-        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor))
+        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor), headers: { 'Content-Type': 'application/activity+json' })
         stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
       end
 
@@ -67,7 +67,7 @@ RSpec.describe ActivityPub::FetchRemoteActorService, type: :service do
       let!(:webfinger) { { subject: 'acct:alice@example.com', links: [{ rel: 'self', href: 'https://example.com/alice' }] } }
 
       before do
-        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor))
+        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor), headers: { 'Content-Type': 'application/activity+json' })
         stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
       end
 
@@ -93,7 +93,7 @@ RSpec.describe ActivityPub::FetchRemoteActorService, type: :service do
       let!(:webfinger) { { subject: 'acct:alice@iscool.af', links: [{ rel: 'self', href: 'https://example.com/alice' }] } }
 
       before do
-        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor))
+        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor), headers: { 'Content-Type': 'application/activity+json' })
         stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
         stub_request(:get, 'https://iscool.af/.well-known/webfinger?resource=acct:alice@iscool.af').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
       end
@@ -125,7 +125,7 @@ RSpec.describe ActivityPub::FetchRemoteActorService, type: :service do
       let!(:webfinger) { { subject: 'acct:alice@example.com', links: [{ rel: 'self', href: 'https://example.com/bob' }] } }
 
       before do
-        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor))
+        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor), headers: { 'Content-Type': 'application/activity+json' })
         stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
       end
 
@@ -148,7 +148,7 @@ RSpec.describe ActivityPub::FetchRemoteActorService, type: :service do
       let!(:webfinger) { { subject: 'acct:alice@iscool.af', links: [{ rel: 'self', href: 'https://example.com/bob' }] } }
 
       before do
-        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor))
+        stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor), headers: { 'Content-Type': 'application/activity+json' })
         stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
         stub_request(:get, 'https://iscool.af/.well-known/webfinger?resource=acct:alice@iscool.af').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
       end
diff --git a/spec/services/activitypub/fetch_remote_key_service_spec.rb b/spec/services/activitypub/fetch_remote_key_service_spec.rb
index 0b14da4f4..478778cc9 100644
--- a/spec/services/activitypub/fetch_remote_key_service_spec.rb
+++ b/spec/services/activitypub/fetch_remote_key_service_spec.rb
@@ -50,7 +50,7 @@ RSpec.describe ActivityPub::FetchRemoteKeyService, type: :service do
   end
 
   before do
-    stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor))
+    stub_request(:get, 'https://example.com/alice').to_return(body: Oj.dump(actor), headers: { 'Content-Type': 'application/activity+json' })
     stub_request(:get, 'https://example.com/.well-known/webfinger?resource=acct:alice@example.com').to_return(body: Oj.dump(webfinger), headers: { 'Content-Type': 'application/jrd+json' })
   end
 
@@ -59,7 +59,7 @@ RSpec.describe ActivityPub::FetchRemoteKeyService, type: :service do
 
     context 'when the key is a sub-object from the actor' do
       before do
-        stub_request(:get, public_key_id).to_return(body: Oj.dump(actor))
+        stub_request(:get, public_key_id).to_return(body: Oj.dump(actor), headers: { 'Content-Type': 'application/activity+json' })
       end
 
       it 'returns the expected account' do
@@ -71,7 +71,7 @@ RSpec.describe ActivityPub::FetchRemoteKeyService, type: :service do
       let(:public_key_id) { 'https://example.com/alice-public-key.json' }
 
       before do
-        stub_request(:get, public_key_id).to_return(body: Oj.dump(key_json.merge({ '@context': ['https://www.w3.org/ns/activitystreams', 'https://w3id.org/security/v1'] })))
+        stub_request(:get, public_key_id).to_return(body: Oj.dump(key_json.merge({ '@context': ['https://www.w3.org/ns/activitystreams', 'https://w3id.org/security/v1'] })), headers: { 'Content-Type': 'application/activity+json' })
       end
 
       it 'returns the expected account' do
@@ -84,7 +84,7 @@ RSpec.describe ActivityPub::FetchRemoteKeyService, type: :service do
       let(:actor_public_key) { 'https://example.com/alice-public-key.json' }
 
       before do
-        stub_request(:get, public_key_id).to_return(body: Oj.dump(key_json.merge({ '@context': ['https://www.w3.org/ns/activitystreams', 'https://w3id.org/security/v1'] })))
+        stub_request(:get, public_key_id).to_return(body: Oj.dump(key_json.merge({ '@context': ['https://www.w3.org/ns/activitystreams', 'https://w3id.org/security/v1'] })), headers: { 'Content-Type': 'application/activity+json' })
       end
 
       it 'returns the nil' do
diff --git a/spec/services/activitypub/fetch_replies_service_spec.rb b/spec/services/activitypub/fetch_replies_service_spec.rb
index 62841d7d2..73c2b4506 100644
--- a/spec/services/activitypub/fetch_replies_service_spec.rb
+++ b/spec/services/activitypub/fetch_replies_service_spec.rb
@@ -55,7 +55,7 @@ RSpec.describe ActivityPub::FetchRepliesService, type: :service do
 
       context 'when passing the URL to the collection' do
         before do
-          stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload))
+          stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' })
         end
 
         it 'spawns workers for up to 5 replies on the same server' do
@@ -84,7 +84,7 @@ RSpec.describe ActivityPub::FetchRepliesService, type: :service do
 
       context 'when passing the URL to the collection' do
         before do
-          stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload))
+          stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' })
         end
 
         it 'spawns workers for up to 5 replies on the same server' do
@@ -117,7 +117,7 @@ RSpec.describe ActivityPub::FetchRepliesService, type: :service do
 
       context 'when passing the URL to the collection' do
         before do
-          stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload))
+          stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' })
         end
 
         it 'spawns workers for up to 5 replies on the same server' do
diff --git a/spec/services/activitypub/synchronize_followers_service_spec.rb b/spec/services/activitypub/synchronize_followers_service_spec.rb
index c9a513e24..f62376ab9 100644
--- a/spec/services/activitypub/synchronize_followers_service_spec.rb
+++ b/spec/services/activitypub/synchronize_followers_service_spec.rb
@@ -60,7 +60,7 @@ RSpec.describe ActivityPub::SynchronizeFollowersService, type: :service do
   describe '#call' do
     context 'when the endpoint is a Collection of actor URIs' do
       before do
-        stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload))
+        stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' })
       end
 
       it_behaves_like 'synchronizes followers'
@@ -77,7 +77,7 @@ RSpec.describe ActivityPub::SynchronizeFollowersService, type: :service do
       end
 
       before do
-        stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload))
+        stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' })
       end
 
       it_behaves_like 'synchronizes followers'
@@ -98,7 +98,7 @@ RSpec.describe ActivityPub::SynchronizeFollowersService, type: :service do
       end
 
       before do
-        stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload))
+        stub_request(:get, collection_uri).to_return(status: 200, body: Oj.dump(payload), headers: { 'Content-Type': 'application/activity+json' })
       end
 
       it_behaves_like 'synchronizes followers'
diff --git a/spec/workers/activitypub/fetch_replies_worker_spec.rb b/spec/workers/activitypub/fetch_replies_worker_spec.rb
index ff4d049a2..2d080e286 100644
--- a/spec/workers/activitypub/fetch_replies_worker_spec.rb
+++ b/spec/workers/activitypub/fetch_replies_worker_spec.rb
@@ -21,7 +21,7 @@ describe ActivityPub::FetchRepliesWorker do
 
   describe 'perform' do
     it 'performs a request if the collection URI is from the same host' do
-      stub_request(:get, 'https://example.com/statuses_replies/1').to_return(status: 200, body: json)
+      stub_request(:get, 'https://example.com/statuses_replies/1').to_return(status: 200, body: json, headers: { 'Content-Type': 'application/activity+json' })
       subject.perform(status.id, 'https://example.com/statuses_replies/1')
       expect(a_request(:get, 'https://example.com/statuses_replies/1')).to have_been_made.once
     end

From 0e4e98fad1e21e4f356b3a71eef3fca79890105c Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 15 Feb 2024 11:56:35 +0100
Subject: [PATCH 065/117] Bump version to v4.2.7

---
 CHANGELOG.md            | 11 +++++++++++
 docker-compose.yml      |  6 +++---
 lib/mastodon/version.rb |  2 +-
 3 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 14749caa3..2a932250b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,17 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.2.7] - 2024-02-16
+
+### Fixed
+
+- Fix OmniAuth tests and edge cases in error handling ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29201), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/29207))
+- Fix new installs by upgrading to the latest release of the `nsa` gem, instead of a no longer existing commit ([mjankowski](https://github.com/mastodon/mastodon/pull/29065))
+
+### Security
+
+- Fix insufficient checking of remote posts ([GHSA-jhrq-qvrm-qr36](https://github.com/mastodon/mastodon/security/advisories/GHSA-jhrq-qvrm-qr36))
+
 ## [4.2.6] - 2024-02-14
 
 ### Security
diff --git a/docker-compose.yml b/docker-compose.yml
index 2f03db04d..4563e2317 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -56,7 +56,7 @@ services:
 
   web:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.6
+    image: ghcr.io/mastodon/mastodon:v4.2.7
     restart: always
     env_file: .env.production
     command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@@ -77,7 +77,7 @@ services:
 
   streaming:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.6
+    image: ghcr.io/mastodon/mastodon:v4.2.7
     restart: always
     env_file: .env.production
     command: node ./streaming
@@ -95,7 +95,7 @@ services:
 
   sidekiq:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.6
+    image: ghcr.io/mastodon/mastodon:v4.2.7
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq
diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb
index 2256b20a6..486dea920 100644
--- a/lib/mastodon/version.rb
+++ b/lib/mastodon/version.rb
@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      6
+      7
     end
 
     def default_prerelease

From c5d56de98d784825edc464b6e5e47884294ea701 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 16 Feb 2024 13:57:04 +0100
Subject: [PATCH 066/117] Fix linting failure

---
 SECURITY.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/SECURITY.md b/SECURITY.md
index 667096324..81472b01b 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -13,8 +13,8 @@ A "vulnerability in Mastodon" is a vulnerability in the code distributed through
 
 ## Supported Versions
 
-| Version | Supported        |
-| ------- | ---------------- |
-| 4.2.x   | Yes              |
-| 4.1.x   | Yes              |
-| < 4.1   | No               |
+| Version | Supported |
+| ------- | --------- |
+| 4.2.x   | Yes       |
+| 4.1.x   | Yes       |
+| < 4.1   | No        |

From fbb07893b82002686a144699ded4401ccd5b8ea7 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 22 Feb 2024 13:25:53 +0100
Subject: [PATCH 067/117] Update dependencies (#29346)

---
 Gemfile.lock | 120 +++++++++++++++++++++++++--------------------------
 1 file changed, 60 insertions(+), 60 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 134e41a7e..9290a84ae 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -28,47 +28,47 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (7.0.8)
-      actionpack (= 7.0.8)
-      activesupport (= 7.0.8)
+    actioncable (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (7.0.8)
-      actionpack (= 7.0.8)
-      activejob (= 7.0.8)
-      activerecord (= 7.0.8)
-      activestorage (= 7.0.8)
-      activesupport (= 7.0.8)
+    actionmailbox (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      activejob (= 7.0.8.1)
+      activerecord (= 7.0.8.1)
+      activestorage (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       mail (>= 2.7.1)
       net-imap
       net-pop
       net-smtp
-    actionmailer (7.0.8)
-      actionpack (= 7.0.8)
-      actionview (= 7.0.8)
-      activejob (= 7.0.8)
-      activesupport (= 7.0.8)
+    actionmailer (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      actionview (= 7.0.8.1)
+      activejob (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       mail (~> 2.5, >= 2.5.4)
       net-imap
       net-pop
       net-smtp
       rails-dom-testing (~> 2.0)
-    actionpack (7.0.8)
-      actionview (= 7.0.8)
-      activesupport (= 7.0.8)
+    actionpack (7.0.8.1)
+      actionview (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       rack (~> 2.0, >= 2.2.4)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (7.0.8)
-      actionpack (= 7.0.8)
-      activerecord (= 7.0.8)
-      activestorage (= 7.0.8)
-      activesupport (= 7.0.8)
+    actiontext (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      activerecord (= 7.0.8.1)
+      activestorage (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       globalid (>= 0.6.0)
       nokogiri (>= 1.8.5)
-    actionview (7.0.8)
-      activesupport (= 7.0.8)
+    actionview (7.0.8.1)
+      activesupport (= 7.0.8.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
@@ -78,22 +78,22 @@ GEM
       activemodel (>= 4.1, < 7.1)
       case_transform (>= 0.2)
       jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
-    activejob (7.0.8)
-      activesupport (= 7.0.8)
+    activejob (7.0.8.1)
+      activesupport (= 7.0.8.1)
       globalid (>= 0.3.6)
-    activemodel (7.0.8)
-      activesupport (= 7.0.8)
-    activerecord (7.0.8)
-      activemodel (= 7.0.8)
-      activesupport (= 7.0.8)
-    activestorage (7.0.8)
-      actionpack (= 7.0.8)
-      activejob (= 7.0.8)
-      activerecord (= 7.0.8)
-      activesupport (= 7.0.8)
+    activemodel (7.0.8.1)
+      activesupport (= 7.0.8.1)
+    activerecord (7.0.8.1)
+      activemodel (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
+    activestorage (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      activejob (= 7.0.8.1)
+      activerecord (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (7.0.8)
+    activesupport (7.0.8.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -215,7 +215,7 @@ GEM
       activerecord (>= 5.a)
       database_cleaner-core (~> 2.0.0)
     database_cleaner-core (2.0.1)
-    date (3.3.3)
+    date (3.3.4)
     debug_inspector (1.1.0)
     devise (4.9.2)
       bcrypt (~> 3.0)
@@ -424,7 +424,7 @@ GEM
       activesupport (>= 4)
       railties (>= 4)
       request_store (~> 1.0)
-    loofah (2.21.3)
+    loofah (2.21.4)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     mail (2.8.1)
@@ -461,11 +461,11 @@ GEM
     net-ldap (0.18.0)
     net-pop (0.1.2)
       net-protocol
-    net-protocol (0.2.1)
+    net-protocol (0.2.2)
       timeout
     net-scp (4.0.0)
       net-ssh (>= 2.6.5, < 8.0.0)
-    net-smtp (0.3.3)
+    net-smtp (0.3.4)
       net-protocol
     net-ssh (7.1.0)
     nio4r (2.7.0)
@@ -534,7 +534,7 @@ GEM
       activesupport (>= 3.0.0)
     raabro (1.4.0)
     racc (1.7.3)
-    rack (2.2.8)
+    rack (2.2.8.1)
     rack-attack (6.7.0)
       rack (>= 1.0, < 4)
     rack-cors (2.0.1)
@@ -551,20 +551,20 @@ GEM
       rack
     rack-test (2.1.0)
       rack (>= 1.3)
-    rails (7.0.8)
-      actioncable (= 7.0.8)
-      actionmailbox (= 7.0.8)
-      actionmailer (= 7.0.8)
-      actionpack (= 7.0.8)
-      actiontext (= 7.0.8)
-      actionview (= 7.0.8)
-      activejob (= 7.0.8)
-      activemodel (= 7.0.8)
-      activerecord (= 7.0.8)
-      activestorage (= 7.0.8)
-      activesupport (= 7.0.8)
+    rails (7.0.8.1)
+      actioncable (= 7.0.8.1)
+      actionmailbox (= 7.0.8.1)
+      actionmailer (= 7.0.8.1)
+      actionpack (= 7.0.8.1)
+      actiontext (= 7.0.8.1)
+      actionview (= 7.0.8.1)
+      activejob (= 7.0.8.1)
+      activemodel (= 7.0.8.1)
+      activerecord (= 7.0.8.1)
+      activestorage (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       bundler (>= 1.15.0)
-      railties (= 7.0.8)
+      railties (= 7.0.8.1)
     rails-controller-testing (1.0.5)
       actionpack (>= 5.0.1.rc1)
       actionview (>= 5.0.1.rc1)
@@ -579,9 +579,9 @@ GEM
     rails-i18n (7.0.7)
       i18n (>= 0.7, < 2)
       railties (>= 6.0.0, < 8)
-    railties (7.0.8)
-      actionpack (= 7.0.8)
-      activesupport (= 7.0.8)
+    railties (7.0.8.1)
+      actionpack (= 7.0.8.1)
+      activesupport (= 7.0.8.1)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
@@ -744,7 +744,7 @@ GEM
     test-prof (1.2.3)
     thor (1.3.0)
     tilt (2.2.0)
-    timeout (0.4.0)
+    timeout (0.4.1)
     tpm-key_attestation (0.12.0)
       bindata (~> 2.4)
       openssl (> 2.0)
@@ -808,7 +808,7 @@ GEM
     xorcist (1.1.3)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.6.11)
+    zeitwerk (2.6.13)
 
 PLATFORMS
   ruby

From 28b666b0d522071e38a5a0a3376d4548c36ee3e9 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 22 Feb 2024 14:39:42 +0100
Subject: [PATCH 068/117] Automatically switch from open to approved
 registrations in absence of moderators (#29337)

---
 app/mailers/admin_mailer.rb                   |  6 ++
 .../auto_close_registrations.text.erb         |  3 +
 .../auto_close_registrations_scheduler.rb     | 33 ++++++++++
 config/locales/en.yml                         |  3 +
 config/sidekiq.yml                            |  4 ++
 ...auto_close_registrations_scheduler_spec.rb | 60 +++++++++++++++++++
 6 files changed, 109 insertions(+)
 create mode 100644 app/views/admin_mailer/auto_close_registrations.text.erb
 create mode 100644 app/workers/scheduler/auto_close_registrations_scheduler.rb
 create mode 100644 spec/workers/scheduler/auto_close_registrations_scheduler_spec.rb

diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb
index 990b92c33..6b08aa8cc 100644
--- a/app/mailers/admin_mailer.rb
+++ b/app/mailers/admin_mailer.rb
@@ -61,6 +61,12 @@ class AdminMailer < ApplicationMailer
     end
   end
 
+  def auto_close_registrations
+    locale_for_account(@me) do
+      mail subject: default_i18n_subject(instance: @instance)
+    end
+  end
+
   private
 
   def process_params
diff --git a/app/views/admin_mailer/auto_close_registrations.text.erb b/app/views/admin_mailer/auto_close_registrations.text.erb
new file mode 100644
index 000000000..c0f848692
--- /dev/null
+++ b/app/views/admin_mailer/auto_close_registrations.text.erb
@@ -0,0 +1,3 @@
+<%= raw t('admin_mailer.auto_close_registrations.body', instance: @instance) %>
+
+<%= raw t('application_mailer.view')%> <%= admin_settings_registrations_url %>
diff --git a/app/workers/scheduler/auto_close_registrations_scheduler.rb b/app/workers/scheduler/auto_close_registrations_scheduler.rb
new file mode 100644
index 000000000..17516dd23
--- /dev/null
+++ b/app/workers/scheduler/auto_close_registrations_scheduler.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class Scheduler::AutoCloseRegistrationsScheduler
+  include Sidekiq::Worker
+  include Redisable
+
+  sidekiq_options retry: 0
+
+  # Automatically switch away from open registrations if no
+  # moderator had any activity in that period of time
+  OPEN_REGISTRATIONS_MODERATOR_THRESHOLD = 1.week + UserTrackingConcern::SIGN_IN_UPDATE_FREQUENCY
+
+  def perform
+    return if Rails.configuration.x.email_domains_whitelist.present? || ENV['DISABLE_AUTOMATIC_SWITCHING_TO_APPROVED_REGISTRATIONS'] == 'true'
+    return unless Setting.registrations_mode == 'open'
+
+    switch_to_approval_mode! unless active_moderators?
+  end
+
+  private
+
+  def active_moderators?
+    User.those_who_can(:manage_reports).exists?(current_sign_in_at: OPEN_REGISTRATIONS_MODERATOR_THRESHOLD.ago...)
+  end
+
+  def switch_to_approval_mode!
+    Setting.registrations_mode = 'approved'
+
+    User.those_who_can(:view_devops).includes(:account).find_each do |user|
+      AdminMailer.with(recipient: user.account).auto_close_registrations.deliver_later
+    end
+  end
+end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 2b3bfb4c7..85a0a9ff9 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -963,6 +963,9 @@ en:
       title: Webhooks
       webhook: Webhook
   admin_mailer:
+    auto_close_registrations:
+      body: Due to a lack of recent moderator activity, registrations on %{instance} have been automatically switched to requiring manual review, to prevent %{instance} from being used as a platform for potential bad actors. You can switch it back to open registrations at any time.
+      subject: Registrations for %{instance} have been automatically switched to requiring approval
     new_appeal:
       actions:
         delete_statuses: to delete their posts
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index f1ba5651d..8c9481c05 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -62,3 +62,7 @@
       interval: 30 minutes
       class: Scheduler::SoftwareUpdateCheckScheduler
       queue: scheduler
+    auto_close_registrations_scheduler:
+      interval: 1 hour
+      class: Scheduler::AutoCloseRegistrationsScheduler
+      queue: scheduler
diff --git a/spec/workers/scheduler/auto_close_registrations_scheduler_spec.rb b/spec/workers/scheduler/auto_close_registrations_scheduler_spec.rb
new file mode 100644
index 000000000..c0c50b128
--- /dev/null
+++ b/spec/workers/scheduler/auto_close_registrations_scheduler_spec.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Scheduler::AutoCloseRegistrationsScheduler do
+  subject { described_class.new }
+
+  describe '#perform' do
+    let(:moderator_activity_date) { Time.now.utc }
+
+    before do
+      Fabricate(:user, role: UserRole.find_by(name: 'Owner'), current_sign_in_at: 10.years.ago)
+      Fabricate(:user, role: UserRole.find_by(name: 'Moderator'), current_sign_in_at: moderator_activity_date)
+    end
+
+    context 'when registrations are open' do
+      before do
+        Setting.registrations_mode = 'open'
+      end
+
+      context 'when a moderator has logged in recently' do
+        let(:moderator_activity_date) { Time.now.utc }
+
+        it 'does not change registrations mode' do
+          expect { subject.perform }.to_not change(Setting, :registrations_mode)
+        end
+      end
+
+      context 'when a moderator has not recently signed in' do
+        let(:moderator_activity_date) { 1.year.ago }
+
+        it 'changes registrations mode from open to approved' do
+          expect { subject.perform }.to change(Setting, :registrations_mode).from('open').to('approved')
+        end
+      end
+    end
+
+    context 'when registrations are closed' do
+      before do
+        Setting.registrations_mode = 'none'
+      end
+
+      context 'when a moderator has logged in recently' do
+        let(:moderator_activity_date) { Time.now.utc }
+
+        it 'does not change registrations mode' do
+          expect { subject.perform }.to_not change(Setting, :registrations_mode)
+        end
+      end
+
+      context 'when a moderator has not recently signed in' do
+        let(:moderator_activity_date) { 1.year.ago }
+
+        it 'does not change registrations mode' do
+          expect { subject.perform }.to_not change(Setting, :registrations_mode)
+        end
+      end
+    end
+  end
+end

From 4fd22acb4ad625b263794afa6d4383e73ce9e5b1 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 22 Feb 2024 18:15:38 +0100
Subject: [PATCH 069/117] Fix auto-close email being sent to users with devops
 permissions instead of settings permissions (#29356)

---
 app/workers/scheduler/auto_close_registrations_scheduler.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/workers/scheduler/auto_close_registrations_scheduler.rb b/app/workers/scheduler/auto_close_registrations_scheduler.rb
index 17516dd23..687450291 100644
--- a/app/workers/scheduler/auto_close_registrations_scheduler.rb
+++ b/app/workers/scheduler/auto_close_registrations_scheduler.rb
@@ -26,7 +26,7 @@ class Scheduler::AutoCloseRegistrationsScheduler
   def switch_to_approval_mode!
     Setting.registrations_mode = 'approved'
 
-    User.those_who_can(:view_devops).includes(:account).find_each do |user|
+    User.those_who_can(:manage_settings).includes(:account).find_each do |user|
       AdminMailer.with(recipient: user.account).auto_close_registrations.deliver_later
     end
   end

From 328a9b81574baf1644df9b0792f716c64be12a5c Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 22 Feb 2024 18:15:59 +0100
Subject: [PATCH 070/117] Change registrations to be disabled by default for
 new servers (#29353)

---
 app/javascript/packs/admin.jsx                |  4 ++++
 .../settings/registrations/show.html.haml     |  4 +++-
 config/locales/en.yml                         |  2 ++
 config/settings.yml                           |  2 +-
 spec/models/setting_spec.rb                   | 20 +++++--------------
 spec/rails_helper.rb                          |  7 +++++++
 spec/spec_helper.rb                           |  6 ++++++
 7 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/app/javascript/packs/admin.jsx b/app/javascript/packs/admin.jsx
index ebcc6903f..524b6a5c4 100644
--- a/app/javascript/packs/admin.jsx
+++ b/app/javascript/packs/admin.jsx
@@ -145,6 +145,10 @@ delegate(document, '#form_admin_settings_enable_bootstrap_timeline_accounts', 'c
 const onChangeRegistrationMode = (target) => {
   const enabled = target.value === 'approved';
 
+  [].forEach.call(document.querySelectorAll('.form_admin_settings_registrations_mode .warning-hint'), (warning_hint) => {
+    warning_hint.style.display = target.value === 'open' ? 'inline' : 'none';
+  });
+
   [].forEach.call(document.querySelectorAll('#form_admin_settings_require_invite_text'), (input) => {
     input.disabled = !enabled;
     if (enabled) {
diff --git a/app/views/admin/settings/registrations/show.html.haml b/app/views/admin/settings/registrations/show.html.haml
index 168f10975..4ece27bf4 100644
--- a/app/views/admin/settings/registrations/show.html.haml
+++ b/app/views/admin/settings/registrations/show.html.haml
@@ -10,9 +10,11 @@
 
   %p.lead= t('admin.settings.registrations.preamble')
 
+  .flash-message= t('admin.settings.registrations.moderation_recommandation')
+
   .fields-row
     .fields-row__column.fields-row__column-6.fields-group
-      = f.input :registrations_mode, collection: %w(open approved none), wrapper: :with_label, include_blank: false, label_method: ->(mode) { I18n.t("admin.settings.registrations_mode.modes.#{mode}") }
+      = f.input :registrations_mode, collection: %w(open approved none), wrapper: :with_label, include_blank: false, label_method: ->(mode) { I18n.t("admin.settings.registrations_mode.modes.#{mode}") }, warning_hint: I18n.t('admin.settings.registrations_mode.warning_hint')
 
     .fields-row__column.fields-row__column-6.fields-group
       = f.input :require_invite_text, as: :boolean, wrapper: :with_label, disabled: !approved_registrations?
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 85a0a9ff9..7db5e1bad 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -764,6 +764,7 @@ en:
         disabled: To no one
         users: To logged-in local users
       registrations:
+        moderation_recommandation: Please make sure you have an adequate and reactive moderation team before you open registrations to everyone!
         preamble: Control who can create an account on your server.
         title: Registrations
       registrations_mode:
@@ -771,6 +772,7 @@ en:
           approved: Approval required for sign up
           none: Nobody can sign up
           open: Anyone can sign up
+        warning_hint: We recommend using “Approval required for sign up” unless you are confident your moderation team can handle spam and malicious registrations in a timely fashion.
       security:
         authorized_fetch: Require authentication from federated servers
         authorized_fetch_hint: Requiring authentication from federated servers enables stricter enforcement of both user-level and server-level blocks. However, this comes at the cost of a performance penalty, reduces the reach of your replies, and may introduce compatibility issues with some federated services. In addition, this will not prevent dedicated actors from fetching your public posts and accounts.
diff --git a/config/settings.yml b/config/settings.yml
index 67297c26c..208c8e376 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -9,7 +9,7 @@ defaults: &defaults
   site_terms: ''
   site_contact_username: ''
   site_contact_email: ''
-  registrations_mode: 'open'
+  registrations_mode: 'none'
   profile_directory: true
   closed_registrations_message: ''
   timeline_preview: true
diff --git a/spec/models/setting_spec.rb b/spec/models/setting_spec.rb
index 5ed5c5d76..97e548e09 100644
--- a/spec/models/setting_spec.rb
+++ b/spec/models/setting_spec.rb
@@ -142,22 +142,12 @@ RSpec.describe Setting do
     context 'when records includes nothing' do
       let(:records) { [] }
 
-      context 'when default_value is not a Hash' do
-        it 'includes Setting with value of default_value' do
-          setting = described_class.all_as_records[key]
+      it 'includes Setting with value of default_value' do
+        setting = described_class.all_as_records[key]
 
-          expect(setting).to be_a described_class
-          expect(setting).to have_attributes(var: key)
-          expect(setting).to have_attributes(value: 'default_value')
-        end
-      end
-
-      context 'when default_value is a Hash' do
-        let(:default_value) { { 'foo' => 'fuga' } }
-
-        it 'returns {}' do
-          expect(described_class.all_as_records).to eq({})
-        end
+        expect(setting).to be_a described_class
+        expect(setting).to have_attributes(var: key)
+        expect(setting).to have_attributes(value: default_value)
       end
     end
   end
diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb
index 7b8dccb6a..17067c58f 100644
--- a/spec/rails_helper.rb
+++ b/spec/rails_helper.rb
@@ -152,6 +152,13 @@ RSpec.configure do |config|
     self.use_transactional_tests = false
 
     DatabaseCleaner.cleaning do
+      # NOTE: we switched registrations mode to closed by default, but the specs
+      # very heavily rely on having it enabled by default, as it relies on users
+      # being approved by default except in select cases where explicitly testing
+      # other registration modes
+      # Also needs to be set per-example here because of the database cleaner.
+      Setting.registrations_mode = 'open'
+
       example.run
     end
 
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index b4c20545f..030bc81fb 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -31,6 +31,12 @@ RSpec.configure do |config|
   config.before :suite do
     Rails.application.load_seed
     Chewy.strategy(:bypass)
+
+    # NOTE: we switched registrations mode to closed by default, but the specs
+    # very heavily rely on having it enabled by default, as it relies on users
+    # being approved by default except in select cases where explicitly testing
+    # other registration modes
+    Setting.registrations_mode = 'open'
   end
 
   config.after :suite do

From 9a7802655f652524be58fe73efa0e6077ff65ea2 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 22 Feb 2024 19:12:53 +0100
Subject: [PATCH 071/117] Fix link verifications when page size exceeds 1MB
 (#29361)

---
 app/services/verify_link_service.rb       |  2 +-
 spec/services/verify_link_service_spec.rb | 27 ++++++++++++-----------
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/app/services/verify_link_service.rb b/app/services/verify_link_service.rb
index 707aeb4e0..b317fc31a 100644
--- a/app/services/verify_link_service.rb
+++ b/app/services/verify_link_service.rb
@@ -19,7 +19,7 @@ class VerifyLinkService < BaseService
 
   def perform_request!
     @body = Request.new(:get, @url).add_headers('Accept' => 'text/html').perform do |res|
-      res.code == 200 ? res.body_with_limit : nil
+      res.code == 200 ? res.truncated_body : nil
     end
   end
 
diff --git a/spec/services/verify_link_service_spec.rb b/spec/services/verify_link_service_spec.rb
index 415788cb5..d06344f9c 100644
--- a/spec/services/verify_link_service_spec.rb
+++ b/spec/services/verify_link_service_spec.rb
@@ -76,6 +76,20 @@ RSpec.describe VerifyLinkService, type: :service do
     end
 
     context 'when a document is truncated but the link back is valid' do
+      let(:html) do
+        "
+          <!doctype html>
+          <body>
+            <a rel=\"me\" href=\"#{ActivityPub::TagManager.instance.url_for(account)}\">
+        "
+      end
+
+      it 'marks the field as verified' do
+        expect(field.verified?).to be true
+      end
+    end
+
+    context 'when a link tag might be truncated' do
       let(:html) do
         "
           <!doctype html>
@@ -89,19 +103,6 @@ RSpec.describe VerifyLinkService, type: :service do
       end
     end
 
-    context 'when a link back might be truncated' do
-      let(:html) do
-        "
-          <!doctype html>
-          <body>
-            <a rel=\"me\" href=\"#{ActivityPub::TagManager.instance.url_for(account)}"
-      end
-
-      it 'does not mark the field as verified' do
-        expect(field.verified?).to be false
-      end
-    end
-
     context 'when a link does not contain a link back' do
       let(:html) { '' }
 

From f3ad91895028c0d593a9412f2754ecbf9400ad39 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 23 Feb 2024 09:53:04 +0100
Subject: [PATCH 072/117] Fix processing of `Link` objects in `Image` objects
 (#29363)

---
 .../activitypub/process_account_service.rb    | 11 ++--
 .../process_account_service_spec.rb           | 51 +++++++++++++++----
 2 files changed, 49 insertions(+), 13 deletions(-)

diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb
index dc845ce97..5edf52325 100644
--- a/app/services/activitypub/process_account_service.rb
+++ b/app/services/activitypub/process_account_service.rb
@@ -201,10 +201,15 @@ class ActivityPub::ProcessAccountService < BaseService
     value = first_of_value(@json[key])
 
     return if value.nil?
-    return value['url'] if value.is_a?(Hash)
 
-    image = fetch_resource_without_id_validation(value)
-    image['url'] if image
+    if value.is_a?(String)
+      value = fetch_resource_without_id_validation(value)
+      return if value.nil?
+    end
+
+    value = first_of_value(value['url']) if value.is_a?(Hash) && value['type'] == 'Image'
+    value = value['href'] if value.is_a?(Hash)
+    value if value.is_a?(String)
   end
 
   def public_key
diff --git a/spec/services/activitypub/process_account_service_spec.rb b/spec/services/activitypub/process_account_service_spec.rb
index c02a0800a..60214d2ed 100644
--- a/spec/services/activitypub/process_account_service_spec.rb
+++ b/spec/services/activitypub/process_account_service_spec.rb
@@ -5,7 +5,7 @@ require 'rails_helper'
 RSpec.describe ActivityPub::ProcessAccountService, type: :service do
   subject { described_class.new }
 
-  context 'with property values' do
+  context 'with property values, an avatar, and a profile header' do
     let(:payload) do
       {
         id: 'https://foo.test',
@@ -16,19 +16,50 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do
           { type: 'PropertyValue', name: 'Occupation', value: 'Unit test' },
           { type: 'PropertyValue', name: 'non-string', value: %w(foo bar) },
         ],
+        image: {
+          type: 'Image',
+          mediaType: 'image/png',
+          url: 'https://foo.test/image.png',
+        },
+        icon: {
+          type: 'Image',
+          url: [
+            {
+              mediaType: 'image/png',
+              href: 'https://foo.test/icon.png',
+            },
+          ],
+        },
       }.with_indifferent_access
     end
 
-    it 'parses out of attachment' do
+    before do
+      stub_request(:get, 'https://foo.test/image.png').to_return(request_fixture('avatar.txt'))
+      stub_request(:get, 'https://foo.test/icon.png').to_return(request_fixture('avatar.txt'))
+    end
+
+    it 'parses property values, avatar and profile header as expected' do
       account = subject.call('alice', 'example.com', payload)
-      expect(account.fields).to be_a Array
-      expect(account.fields.size).to eq 2
-      expect(account.fields[0]).to be_a Account::Field
-      expect(account.fields[0].name).to eq 'Pronouns'
-      expect(account.fields[0].value).to eq 'They/them'
-      expect(account.fields[1]).to be_a Account::Field
-      expect(account.fields[1].name).to eq 'Occupation'
-      expect(account.fields[1].value).to eq 'Unit test'
+
+      expect(account.fields)
+        .to be_an(Array)
+        .and have_attributes(size: 2)
+      expect(account.fields.first)
+        .to be_an(Account::Field)
+        .and have_attributes(
+          name: eq('Pronouns'),
+          value: eq('They/them')
+        )
+      expect(account.fields.last)
+        .to be_an(Account::Field)
+        .and have_attributes(
+          name: eq('Occupation'),
+          value: eq('Unit test')
+        )
+      expect(account).to have_attributes(
+        avatar_remote_url: 'https://foo.test/icon.png',
+        header_remote_url: 'https://foo.test/image.png'
+      )
     end
   end
 

From bdb6650ebc73566a09482521a43b457270b3c44a Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 23 Feb 2024 14:09:41 +0100
Subject: [PATCH 073/117] Bump version to v4.2.8 (#29370)

---
 CHANGELOG.md            | 20 ++++++++++++++++++++
 docker-compose.yml      |  6 +++---
 lib/mastodon/version.rb |  2 +-
 3 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2a932250b..636b3c161 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,26 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.2.8] - 2024-02-23
+
+### Added
+
+- Add hourly task to automatically require approval for new registrations in the absence of moderators ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29318), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/29355))
+  In order to prevent future abandoned Mastodon servers from being used for spam, harassment and other malicious activity, Mastodon will now automatically switch new user registrations to require moderator approval whenever they are left open and no activity (including non-moderation actions from apps) from any logged-in user with permission to access moderation reports has been detected in a full week.
+  When this happens, users with the permission to change server settings will receive an email notification.
+  This feature is disabled when `EMAIL_DOMAIN_ALLOWLIST` is used, and can also be disabled with `DISABLE_AUTOMATIC_SWITCHING_TO_APPROVED_REGISTRATIONS=true`.
+
+### Changed
+
+- Change registrations to be closed by default on new installations ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29280))
+  If you are running a server and never changed your registrations mode from the default, updating will automatically close your registrations.
+  Simply re-enable them through the administration interface or using `tootctl settings registrations open` if you want to enable them again.
+
+### Fixed
+
+- Fix processing of remote ActivityPub actors making use of `Link` objects as `Image` `url` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29335))
+- Fix link verifications when page size exceeds 1MB ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29358))
+
 ## [4.2.7] - 2024-02-16
 
 ### Fixed
diff --git a/docker-compose.yml b/docker-compose.yml
index 4563e2317..004f5dbb0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -56,7 +56,7 @@ services:
 
   web:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.7
+    image: ghcr.io/mastodon/mastodon:v4.2.8
     restart: always
     env_file: .env.production
     command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@@ -77,7 +77,7 @@ services:
 
   streaming:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.7
+    image: ghcr.io/mastodon/mastodon:v4.2.8
     restart: always
     env_file: .env.production
     command: node ./streaming
@@ -95,7 +95,7 @@ services:
 
   sidekiq:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.7
+    image: ghcr.io/mastodon/mastodon:v4.2.8
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq
diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb
index 486dea920..b28349dce 100644
--- a/lib/mastodon/version.rb
+++ b/lib/mastodon/version.rb
@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      7
+      8
     end
 
     def default_prerelease

From ed8e4bab4c8232533bd40d36c2ea5524e078f0af Mon Sep 17 00:00:00 2001
From: Matt Jankowski <matt@jankowski.online>
Date: Thu, 14 Dec 2023 09:02:33 -0500
Subject: [PATCH 074/117] Fix reference to non-existent var in CLI maintenance
 command (#28363)

---
 lib/mastodon/cli/maintenance.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/mastodon/cli/maintenance.rb b/lib/mastodon/cli/maintenance.rb
index e73bcbf86..c2a6802e1 100644
--- a/lib/mastodon/cli/maintenance.rb
+++ b/lib/mastodon/cli/maintenance.rb
@@ -224,7 +224,7 @@ module Mastodon::CLI
         users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
         ref_user = users.shift
         say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow
-        say "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}", :yellow
+        say "e-mail will be disabled for the following accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow
         say 'Please reach out to them and set another address with `tootctl account modify` or delete them.', :yellow
 
         users.each_with_index do |user, index|

From 6536d96d1b211cb4b462010474cb63b241de9998 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 4 Jan 2024 15:14:46 +0100
Subject: [PATCH 075/117] Add fallback redirection when getting a webfinger
 query `WEB_DOMAIN@WEB_DOMAIN` (#28592)

---
 app/controllers/well_known/webfinger_controller.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/controllers/well_known/webfinger_controller.rb b/app/controllers/well_known/webfinger_controller.rb
index 4748940f7..6cf37c2ff 100644
--- a/app/controllers/well_known/webfinger_controller.rb
+++ b/app/controllers/well_known/webfinger_controller.rb
@@ -21,7 +21,7 @@ module WellKnown
       username = username_from_resource
 
       @account = begin
-        if username == Rails.configuration.x.local_domain
+        if username == Rails.configuration.x.local_domain || username == Rails.configuration.x.web_domain
           Account.representative
         else
           Account.find_local!(username)

From f784213c6405cbca2f3ff70479aff2e263b28730 Mon Sep 17 00:00:00 2001
From: Emelia Smith <ThisIsMissEm@users.noreply.github.com>
Date: Tue, 6 Feb 2024 13:38:14 +0100
Subject: [PATCH 076/117] Return domain block digests from admin domain blocks
 API (#29092)

---
 app/serializers/rest/admin/domain_block_serializer.rb | 6 +++++-
 spec/requests/api/v1/admin/domain_blocks_spec.rb      | 3 +++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/app/serializers/rest/admin/domain_block_serializer.rb b/app/serializers/rest/admin/domain_block_serializer.rb
index b955d008a..e94a337cb 100644
--- a/app/serializers/rest/admin/domain_block_serializer.rb
+++ b/app/serializers/rest/admin/domain_block_serializer.rb
@@ -1,11 +1,15 @@
 # frozen_string_literal: true
 
 class REST::Admin::DomainBlockSerializer < ActiveModel::Serializer
-  attributes :id, :domain, :created_at, :severity,
+  attributes :id, :domain, :digest, :created_at, :severity,
              :reject_media, :reject_reports,
              :private_comment, :public_comment, :obfuscate
 
   def id
     object.id.to_s
   end
+
+  def digest
+    object.domain_digest
+  end
 end
diff --git a/spec/requests/api/v1/admin/domain_blocks_spec.rb b/spec/requests/api/v1/admin/domain_blocks_spec.rb
index 7a5ac28c5..a6cfdb48d 100644
--- a/spec/requests/api/v1/admin/domain_blocks_spec.rb
+++ b/spec/requests/api/v1/admin/domain_blocks_spec.rb
@@ -49,6 +49,7 @@ RSpec.describe 'Domain Blocks' do
           {
             id: domain_block.id.to_s,
             domain: domain_block.domain,
+            digest: domain_block.domain_digest,
             created_at: domain_block.created_at.strftime('%Y-%m-%dT%H:%M:%S.%LZ'),
             severity: domain_block.severity.to_s,
             reject_media: domain_block.reject_media,
@@ -102,6 +103,7 @@ RSpec.describe 'Domain Blocks' do
         {
           id: domain_block.id.to_s,
           domain: domain_block.domain,
+          digest: domain_block.domain_digest,
           created_at: domain_block.created_at.strftime('%Y-%m-%dT%H:%M:%S.%LZ'),
           severity: domain_block.severity.to_s,
           reject_media: domain_block.reject_media,
@@ -212,6 +214,7 @@ RSpec.describe 'Domain Blocks' do
         {
           id: domain_block.id.to_s,
           domain: domain_block.domain,
+          digest: domain_block.domain_digest,
           severity: 'suspend',
         }
       )

From 7af69f5cf575b1124a78c0c8e5b24ebce8060621 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 23 Feb 2024 20:04:57 +0100
Subject: [PATCH 077/117] Fix admin account created by `mastodon:setup` not
 being auto-approved (#29379)

---
 lib/tasks/mastodon.rake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake
index f68d1cf1f..8e024077e 100644
--- a/lib/tasks/mastodon.rake
+++ b/lib/tasks/mastodon.rake
@@ -515,6 +515,7 @@ namespace :mastodon do
           owner_role = UserRole.find_by(name: 'Owner')
           user = User.new(email: email, password: password, confirmed_at: Time.now.utc, account_attributes: { username: username }, bypass_invite_request_check: true, role: owner_role)
           user.save(validate: false)
+          user.approve!
 
           Setting.site_contact_username = username
 

From ab3f9852f23a2180ae77b3a0963fe61e97ab129f Mon Sep 17 00:00:00 2001
From: Jeong Arm <kjwonmail@gmail.com>
Date: Mon, 11 Mar 2024 18:28:08 +0900
Subject: [PATCH 078/117] Normalize idna domain before account unblock domain
 (#29530)

---
 app/models/concerns/account_interactions.rb   |  6 ++-
 .../concerns/account_interactions_spec.rb     | 40 +++++++++++++++++++
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb
index 2de15031f..74ba1695f 100644
--- a/app/models/concerns/account_interactions.rb
+++ b/app/models/concerns/account_interactions.rb
@@ -185,7 +185,7 @@ module AccountInteractions
   end
 
   def unblock_domain!(other_domain)
-    block = domain_blocks.find_by(domain: other_domain)
+    block = domain_blocks.find_by(domain: normalized_domain(other_domain))
     block&.destroy
   end
 
@@ -313,4 +313,8 @@ module AccountInteractions
   def remove_potential_friendship(other_account)
     PotentialFriendshipTracker.remove(id, other_account.id)
   end
+
+  def normalized_domain(domain)
+    TagManager.instance.normalize_domain(domain)
+  end
 end
diff --git a/spec/models/concerns/account_interactions_spec.rb b/spec/models/concerns/account_interactions_spec.rb
index 84e2c91a8..64d1d46d3 100644
--- a/spec/models/concerns/account_interactions_spec.rb
+++ b/spec/models/concerns/account_interactions_spec.rb
@@ -250,6 +250,24 @@ describe AccountInteractions do
     end
   end
 
+  describe '#block_idna_domain!' do
+    subject do
+      [
+        account.block_domain!(idna_domain),
+        account.block_domain!(punycode_domain),
+      ]
+    end
+
+    let(:idna_domain) { '대한민국.한국' }
+    let(:punycode_domain) { 'xn--3e0bs9hfvinn1a.xn--3e0b707e' }
+
+    it 'creates single AccountDomainBlock' do
+      expect do
+        expect(subject).to all(be_a AccountDomainBlock)
+      end.to change { account.domain_blocks.count }.by 1
+    end
+  end
+
   describe '#unfollow!' do
     subject { account.unfollow!(target_account) }
 
@@ -345,6 +363,28 @@ describe AccountInteractions do
     end
   end
 
+  describe '#unblock_idna_domain!' do
+    subject { account.unblock_domain!(punycode_domain) }
+
+    let(:idna_domain) { '대한민국.한국' }
+    let(:punycode_domain) { 'xn--3e0bs9hfvinn1a.xn--3e0b707e' }
+
+    context 'when blocking the domain' do
+      it 'returns destroyed AccountDomainBlock' do
+        account_domain_block = Fabricate(:account_domain_block, domain: idna_domain)
+        account.domain_blocks << account_domain_block
+        expect(subject).to be_a AccountDomainBlock
+        expect(subject).to be_destroyed
+      end
+    end
+
+    context 'when unblocking idna domain' do
+      it 'returns nil' do
+        expect(subject).to be_nil
+      end
+    end
+  end
+
   describe '#following?' do
     subject { account.following?(target_account) }
 

From 0143c9d3e1d7ae11b14aff5a0326885cbbd3ed19 Mon Sep 17 00:00:00 2001
From: Matt Jankowski <matt@jankowski.online>
Date: Fri, 22 Mar 2024 11:08:27 -0400
Subject: [PATCH 079/117] Fix results/query in
 `api/v1/featured_tags/suggestions` (#29597)

---
 .../featured_tags/suggestions_controller.rb   |  6 +++-
 .../suggestions_controller_spec.rb            | 28 +++++++++++++++++--
 spec/fabricators/featured_tag_fabricator.rb   |  2 +-
 3 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/app/controllers/api/v1/featured_tags/suggestions_controller.rb b/app/controllers/api/v1/featured_tags/suggestions_controller.rb
index 76633210a..4f732ed2d 100644
--- a/app/controllers/api/v1/featured_tags/suggestions_controller.rb
+++ b/app/controllers/api/v1/featured_tags/suggestions_controller.rb
@@ -12,6 +12,10 @@ class Api::V1::FeaturedTags::SuggestionsController < Api::BaseController
   private
 
   def set_recently_used_tags
-    @recently_used_tags = Tag.recently_used(current_account).where.not(id: current_account.featured_tags).limit(10)
+    @recently_used_tags = Tag.recently_used(current_account).where.not(id: featured_tag_ids).limit(10)
+  end
+
+  def featured_tag_ids
+    current_account.featured_tags.pluck(:tag_id)
   end
 end
diff --git a/spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb b/spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb
index 54c63dcc6..8cb928ea2 100644
--- a/spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb
+++ b/spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb
@@ -7,17 +7,39 @@ describe Api::V1::FeaturedTags::SuggestionsController do
 
   let(:user)    { Fabricate(:user) }
   let(:token)   { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:accounts') }
-  let(:account) { Fabricate(:account) }
+  let(:account) { Fabricate(:account, user: user) }
 
   before do
     allow(controller).to receive(:doorkeeper_token) { token }
   end
 
   describe 'GET #index' do
-    it 'returns http success' do
+    let!(:unused_featured_tag) { Fabricate(:tag, name: 'unused_featured_tag') }
+    let!(:used_tag) { Fabricate(:tag, name: 'used_tag') }
+    let!(:used_featured_tag) { Fabricate(:tag, name: 'used_featured_tag') }
+
+    before do
+      _unused_tag = Fabricate(:tag, name: 'unused_tag')
+
+      # Make relevant tags used by account
+      status = Fabricate(:status, account: account)
+      status.tags << used_tag
+      status.tags << used_featured_tag
+
+      # Feature the relevant tags
+      Fabricate :featured_tag, account: account, name: unused_featured_tag.name
+      Fabricate :featured_tag, account: account, name: used_featured_tag.name
+    end
+
+    it 'returns http success and recently used but not featured tags', :aggregate_failures do
       get :index, params: { account_id: account.id, limit: 2 }
 
-      expect(response).to have_http_status(200)
+      expect(response)
+        .to have_http_status(200)
+      expect(body_as_json)
+        .to contain_exactly(
+          include(name: used_tag.name)
+        )
     end
   end
 end
diff --git a/spec/fabricators/featured_tag_fabricator.rb b/spec/fabricators/featured_tag_fabricator.rb
index 0803dc43a..6003099db 100644
--- a/spec/fabricators/featured_tag_fabricator.rb
+++ b/spec/fabricators/featured_tag_fabricator.rb
@@ -2,6 +2,6 @@
 
 Fabricator(:featured_tag) do
   account { Fabricate.build(:account) }
-  tag { Fabricate.build(:tag) }
+  tag { nil }
   name { sequence(:name) { |i| "Tag#{i}" } }
 end

From 86807e4799d21863084d52d4070f3da7c120e260 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 5 Apr 2024 09:48:45 +0200
Subject: [PATCH 080/117] Improve email address validation (#29838)

---
 Gemfile                                   |  2 ++
 Gemfile.lock                              |  1 +
 app/models/user.rb                        |  2 ++
 app/validators/email_address_validator.rb | 18 ++++++++++++++++++
 spec/models/user_spec.rb                  |  6 ++++++
 5 files changed, 29 insertions(+)
 create mode 100644 app/validators/email_address_validator.rb

diff --git a/Gemfile b/Gemfile
index 829e7d857..1d038ad9f 100644
--- a/Gemfile
+++ b/Gemfile
@@ -204,3 +204,5 @@ gem 'net-http', '~> 0.3.2'
 gem 'rubyzip', '~> 2.3'
 
 gem 'hcaptcha', '~> 7.1'
+
+gem 'mail', '~> 2.8'
diff --git a/Gemfile.lock b/Gemfile.lock
index 9290a84ae..94d2d0ba4 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -871,6 +871,7 @@ DEPENDENCIES
   letter_opener_web (~> 2.0)
   link_header (~> 0.0)
   lograge (~> 0.12)
+  mail (~> 2.8)
   mario-redis-lock (~> 1.2)
   md-paperclip-azure (~> 2.2)
   memory_profiler
diff --git a/app/models/user.rb b/app/models/user.rb
index ee9097927..0c8d481c4 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -96,6 +96,8 @@ class User < ApplicationRecord
   accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? && !Setting.require_invite_text }
   validates :invite_request, presence: true, on: :create, if: :invite_text_required?
 
+  validates :email, presence: true, email_address: true
+
   validates_with BlacklistedEmailValidator, if: -> { ENV['EMAIL_DOMAIN_LISTS_APPLY_AFTER_CONFIRMATION'] == 'true' || !confirmed? }
   validates_with EmailMxValidator, if: :validate_email_dns?
   validates :agreement, acceptance: { allow_nil: false, accept: [true, 'true', '1'] }, on: :create
diff --git a/app/validators/email_address_validator.rb b/app/validators/email_address_validator.rb
new file mode 100644
index 000000000..ed0bb1165
--- /dev/null
+++ b/app/validators/email_address_validator.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# NOTE: I initially wrote this as `EmailValidator` but it ended up clashing
+# with an indirect dependency of ours, `validate_email`, which, turns out,
+# has the same approach as we do, but with an extra check disallowing
+# single-label domains. Decided to not switch to `validate_email` because
+# we do want to allow at least `localhost`.
+
+class EmailAddressValidator < ActiveModel::EachValidator
+  def validate_each(record, attribute, value)
+    value = value.strip
+
+    address = Mail::Address.new(value)
+    record.errors.add(attribute, :invalid) if address.address != value
+  rescue Mail::Field::FieldError
+    record.errors.add(attribute, :invalid)
+  end
+end
diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb
index ffd1889cb..f06150f02 100644
--- a/spec/models/user_spec.rb
+++ b/spec/models/user_spec.rb
@@ -39,6 +39,12 @@ RSpec.describe User do
       expect(user.valid?).to be true
     end
 
+    it 'is valid with a localhost e-mail address' do
+      user = Fabricate.build(:user, email: 'admin@localhost')
+      user.valid?
+      expect(user.valid?).to be true
+    end
+
     it 'cleans out invalid locale' do
       user = Fabricate.build(:user, locale: 'toto')
       expect(user.valid?).to be true

From c3be5a3d2eff947a04a32d78726ff511d7dec98f Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 8 Apr 2024 15:46:13 +0200
Subject: [PATCH 081/117] Remove caching in `cache_collection` (#29862)

---
 app/controllers/concerns/cache_concern.rb     | 29 ++++-----------
 app/models/concerns/cacheable.rb              |  4 +++
 app/models/feed.rb                            |  2 +-
 app/models/public_feed.rb                     |  2 +-
 app/models/status.rb                          | 32 -----------------
 app/models/tag_feed.rb                        |  2 +-
 .../application_controller_spec.rb            | 35 -------------------
 spec/models/home_feed_spec.rb                 |  1 -
 8 files changed, 14 insertions(+), 93 deletions(-)

diff --git a/app/controllers/concerns/cache_concern.rb b/app/controllers/concerns/cache_concern.rb
index 55ebe1bd6..c8ed9f9a5 100644
--- a/app/controllers/concerns/cache_concern.rb
+++ b/app/controllers/concerns/cache_concern.rb
@@ -198,34 +198,19 @@ module CacheConcern
     end
   end
 
+  # TODO: Rename this method, as it does not perform any caching anymore.
   def cache_collection(raw, klass)
-    return raw unless klass.respond_to?(:with_includes)
+    return raw unless klass.respond_to?(:preload_cacheable_associations)
 
-    raw = raw.cache_ids.to_a if raw.is_a?(ActiveRecord::Relation)
-    return [] if raw.empty?
+    records = raw.to_a
 
-    cached_keys_with_value = begin
-      Rails.cache.read_multi(*raw).transform_keys(&:id).transform_values { |r| ActiveRecordCoder.load(r) }
-    rescue ActiveRecordCoder::Error
-      {} # The serialization format may have changed, let's pretend it's a cache miss.
-    end
+    klass.preload_cacheable_associations(records)
 
-    uncached_ids = raw.map(&:id) - cached_keys_with_value.keys
-
-    klass.reload_stale_associations!(cached_keys_with_value.values) if klass.respond_to?(:reload_stale_associations!)
-
-    unless uncached_ids.empty?
-      uncached = klass.where(id: uncached_ids).with_includes.index_by(&:id)
-
-      uncached.each_value do |item|
-        Rails.cache.write(item, ActiveRecordCoder.dump(item))
-      end
-    end
-
-    raw.filter_map { |item| cached_keys_with_value[item.id] || uncached[item.id] }
+    records
   end
 
+  # TODO: Rename this method, as it does not perform any caching anymore.
   def cache_collection_paginated_by_id(raw, klass, limit, options)
-    cache_collection raw.cache_ids.to_a_paginated_by_id(limit, options), klass
+    cache_collection raw.to_a_paginated_by_id(limit, options), klass
   end
 end
diff --git a/app/models/concerns/cacheable.rb b/app/models/concerns/cacheable.rb
index d7524cdfd..0633f20c7 100644
--- a/app/models/concerns/cacheable.rb
+++ b/app/models/concerns/cacheable.rb
@@ -14,6 +14,10 @@ module Cacheable
       includes(@cache_associated)
     end
 
+    def preload_cacheable_associations(records)
+      ActiveRecord::Associations::Preloader.new(records: records, associations: @cache_associated).call
+    end
+
     def cache_ids
       select(:id, :updated_at)
     end
diff --git a/app/models/feed.rb b/app/models/feed.rb
index f51dcfab1..30073fed4 100644
--- a/app/models/feed.rb
+++ b/app/models/feed.rb
@@ -28,7 +28,7 @@ class Feed
       unhydrated = redis.zrangebyscore(key, "(#{min_id}", "(#{max_id}", limit: [0, limit], with_scores: true).map(&:first).map(&:to_i)
     end
 
-    Status.where(id: unhydrated).cache_ids
+    Status.where(id: unhydrated)
   end
 
   def key
diff --git a/app/models/public_feed.rb b/app/models/public_feed.rb
index c208d6f66..df59c9c22 100644
--- a/app/models/public_feed.rb
+++ b/app/models/public_feed.rb
@@ -29,7 +29,7 @@ class PublicFeed
     scope.merge!(media_only_scope) if media_only?
     scope.merge!(language_scope) if account&.chosen_languages.present?
 
-    scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
+    scope.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
   end
 
   private
diff --git a/app/models/status.rb b/app/models/status.rb
index 1c41ef1d5..990ac8061 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -338,38 +338,6 @@ class Status < ApplicationRecord
       StatusPin.select('status_id').where(status_id: status_ids).where(account_id: account_id).each_with_object({}) { |p, h| h[p.status_id] = true }
     end
 
-    def reload_stale_associations!(cached_items)
-      account_ids = []
-
-      cached_items.each do |item|
-        account_ids << item.account_id
-        account_ids << item.reblog.account_id if item.reblog?
-      end
-
-      account_ids.uniq!
-
-      status_ids = cached_items.map { |item| item.reblog? ? item.reblog_of_id : item.id }.uniq
-
-      return if account_ids.empty?
-
-      accounts = Account.where(id: account_ids).includes(:account_stat, :user).index_by(&:id)
-
-      status_stats = StatusStat.where(status_id: status_ids).index_by(&:status_id)
-
-      cached_items.each do |item|
-        item.account = accounts[item.account_id]
-        item.reblog.account = accounts[item.reblog.account_id] if item.reblog?
-
-        if item.reblog?
-          status_stat = status_stats[item.reblog.id]
-          item.reblog.status_stat = status_stat if status_stat.present?
-        else
-          status_stat = status_stats[item.id]
-          item.status_stat = status_stat if status_stat.present?
-        end
-      end
-    end
-
     def from_text(text)
       return [] if text.blank?
 
diff --git a/app/models/tag_feed.rb b/app/models/tag_feed.rb
index b8cd63557..6b5831d24 100644
--- a/app/models/tag_feed.rb
+++ b/app/models/tag_feed.rb
@@ -33,7 +33,7 @@ class TagFeed < PublicFeed
     scope.merge!(account_filters_scope) if account?
     scope.merge!(media_only_scope) if media_only?
 
-    scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
+    scope.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
   end
 
   private
diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb
index e9d479603..5cd45e991 100644
--- a/spec/controllers/application_controller_spec.rb
+++ b/spec/controllers/application_controller_spec.rb
@@ -221,39 +221,4 @@ describe ApplicationController do
 
     include_examples 'respond_with_error', 422
   end
-
-  describe 'cache_collection' do
-    subject do
-      Class.new(ApplicationController) do
-        public :cache_collection
-      end
-    end
-
-    shared_examples 'receives :with_includes' do |fabricator, klass|
-      it 'uses raw if it is not an ActiveRecord::Relation' do
-        record = Fabricate(fabricator)
-        expect(subject.new.cache_collection([record], klass)).to eq [record]
-      end
-    end
-
-    shared_examples 'cacheable' do |fabricator, klass|
-      include_examples 'receives :with_includes', fabricator, klass
-
-      it 'calls cache_ids of raw if it is an ActiveRecord::Relation' do
-        record = Fabricate(fabricator)
-        relation = klass.none
-        allow(relation).to receive(:cache_ids).and_return([record])
-        expect(subject.new.cache_collection(relation, klass)).to eq [record]
-      end
-    end
-
-    it 'returns raw unless class responds to :with_includes' do
-      raw = Object.new
-      expect(subject.new.cache_collection(raw, Object)).to eq raw
-    end
-
-    context 'with a Status' do
-      include_examples 'cacheable', :status, Status
-    end
-  end
 end
diff --git a/spec/models/home_feed_spec.rb b/spec/models/home_feed_spec.rb
index bd649d826..06bb63b1a 100644
--- a/spec/models/home_feed_spec.rb
+++ b/spec/models/home_feed_spec.rb
@@ -27,7 +27,6 @@ RSpec.describe HomeFeed do
         results = subject.get(3)
 
         expect(results.map(&:id)).to eq [3, 2]
-        expect(results.first.attributes.keys).to eq %w(id updated_at)
       end
     end
 

From e69780ec59bbd04a649abc67cc41a824a635fc7b Mon Sep 17 00:00:00 2001
From: Tim Rogers <rogers.timothy.john@gmail.com>
Date: Mon, 22 Apr 2024 04:00:24 -0500
Subject: [PATCH 082/117] Fixed crash when supplying FFMPEG_BINARY environment
 variable (#30022)

---
 app/lib/video_metadata_extractor.rb | 2 +-
 config/initializers/ffmpeg.rb       | 5 +++--
 lib/paperclip/image_extractor.rb    | 2 +-
 lib/paperclip/transcoder.rb         | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/app/lib/video_metadata_extractor.rb b/app/lib/video_metadata_extractor.rb
index f27d34868..df5409375 100644
--- a/app/lib/video_metadata_extractor.rb
+++ b/app/lib/video_metadata_extractor.rb
@@ -22,7 +22,7 @@ class VideoMetadataExtractor
   private
 
   def ffmpeg_command_output
-    command = Terrapin::CommandLine.new('ffprobe', '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel')
+    command = Terrapin::CommandLine.new(Rails.configuration.x.ffprobe_binary, '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel')
     command.run(path: @path, format: 'json', loglevel: 'fatal')
   end
 
diff --git a/config/initializers/ffmpeg.rb b/config/initializers/ffmpeg.rb
index 30ea617fc..87f85eeec 100644
--- a/config/initializers/ffmpeg.rb
+++ b/config/initializers/ffmpeg.rb
@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
-if ENV['FFMPEG_BINARY'].present?
-  FFMPEG.ffmpeg_binary = ENV['FFMPEG_BINARY']
+Rails.application.configure do
+  config.x.ffmpeg_binary = ENV['FFMPEG_BINARY'] || 'ffmpeg'
+  config.x.ffprobe_binary = ENV['FFPROBE_BINARY'] || 'ffprobe'
 end
diff --git a/lib/paperclip/image_extractor.rb b/lib/paperclip/image_extractor.rb
index 17fe4326f..8a565d046 100644
--- a/lib/paperclip/image_extractor.rb
+++ b/lib/paperclip/image_extractor.rb
@@ -35,7 +35,7 @@ module Paperclip
       dst.binmode
 
       begin
-        command = Terrapin::CommandLine.new('ffmpeg', '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger)
+        command = Terrapin::CommandLine.new(Rails.configuration.x.ffmpeg_binary, '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger)
         command.run(source: @file.path, destination: dst.path, loglevel: 'fatal')
       rescue Terrapin::ExitStatusError
         dst.close(true)
diff --git a/lib/paperclip/transcoder.rb b/lib/paperclip/transcoder.rb
index d2d946d3a..3efffa355 100644
--- a/lib/paperclip/transcoder.rb
+++ b/lib/paperclip/transcoder.rb
@@ -61,7 +61,7 @@ module Paperclip
       command_arguments, interpolations = prepare_command(destination)
 
       begin
-        command = Terrapin::CommandLine.new('ffmpeg', command_arguments.join(' '), logger: Paperclip.logger)
+        command = Terrapin::CommandLine.new(Rails.configuration.x.ffmpeg_binary, command_arguments.join(' '), logger: Paperclip.logger)
         command.run(interpolations)
       rescue Terrapin::ExitStatusError => e
         raise Paperclip::Error, "Error while transcoding #{@basename}: #{e}"

From 51ef619140fc87c23352aced941e313e8aa1c699 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 26 Apr 2024 15:19:02 +0200
Subject: [PATCH 083/117] Fix Idempotency-Key ignored when scheduling a post
 (#30084)

---
 app/services/post_status_service.rb       | 4 ++--
 spec/services/post_status_service_spec.rb | 7 +++++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index ea27f374e..e4dd480f1 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -160,7 +160,7 @@ class PostStatusService < BaseService
 
   def idempotency_duplicate
     if scheduled?
-      @account.schedule_statuses.find(@idempotency_duplicate)
+      @account.scheduled_statuses.find(@idempotency_duplicate)
     else
       @account.statuses.find(@idempotency_duplicate)
     end
@@ -214,7 +214,7 @@ class PostStatusService < BaseService
   end
 
   def scheduled_options
-    @options.tap do |options_hash|
+    @options.dup.tap do |options_hash|
       options_hash[:in_reply_to_id]  = options_hash.delete(:thread)&.id
       options_hash[:application_id]  = options_hash.delete(:application)&.id
       options_hash[:scheduled_at]    = nil
diff --git a/spec/services/post_status_service_spec.rb b/spec/services/post_status_service_spec.rb
index 7d7679c88..a74e82626 100644
--- a/spec/services/post_status_service_spec.rb
+++ b/spec/services/post_status_service_spec.rb
@@ -54,6 +54,13 @@ RSpec.describe PostStatusService, type: :service do
     it 'does not change statuses count' do
       expect { subject.call(account, text: 'Hi future!', scheduled_at: future, thread: previous_status) }.to_not(change { [account.statuses_count, previous_status.replies_count] })
     end
+
+    it 'returns existing status when used twice with idempotency key' do
+      account = Fabricate(:account)
+      status1 = subject.call(account, text: 'test', idempotency: 'meepmeep', scheduled_at: future)
+      status2 = subject.call(account, text: 'test', idempotency: 'meepmeep', scheduled_at: future)
+      expect(status2.id).to eq status1.id
+    end
   end
 
   it 'creates response to the original status of boost' do

From 56b7d1a7b6c3beaacbcea21a849894d0a575cb13 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 2 May 2024 22:56:21 +0200
Subject: [PATCH 084/117] Fix not being able to block a subdomain of an
 already-blocked domain through the API (#30119)

---
 .../api/v1/admin/domain_blocks_controller.rb  |  9 +++-
 .../api/v1/admin/domain_blocks_spec.rb        | 45 ++++++++++++++++---
 2 files changed, 46 insertions(+), 8 deletions(-)

diff --git a/app/controllers/api/v1/admin/domain_blocks_controller.rb b/app/controllers/api/v1/admin/domain_blocks_controller.rb
index 2538c7c7c..8786431d9 100644
--- a/app/controllers/api/v1/admin/domain_blocks_controller.rb
+++ b/app/controllers/api/v1/admin/domain_blocks_controller.rb
@@ -29,10 +29,11 @@ class Api::V1::Admin::DomainBlocksController < Api::BaseController
   def create
     authorize :domain_block, :create?
 
+    @domain_block = DomainBlock.new(resource_params)
     existing_domain_block = resource_params[:domain].present? ? DomainBlock.rule_for(resource_params[:domain]) : nil
-    return render json: existing_domain_block, serializer: REST::Admin::ExistingDomainBlockErrorSerializer, status: 422 if existing_domain_block.present?
+    return render json: existing_domain_block, serializer: REST::Admin::ExistingDomainBlockErrorSerializer, status: 422 if conflicts_with_existing_block?(@domain_block, existing_domain_block)
 
-    @domain_block = DomainBlock.create!(resource_params)
+    @domain_block.save!
     DomainBlockWorker.perform_async(@domain_block.id)
     log_action :create, @domain_block
     render json: @domain_block, serializer: REST::Admin::DomainBlockSerializer
@@ -55,6 +56,10 @@ class Api::V1::Admin::DomainBlocksController < Api::BaseController
 
   private
 
+  def conflicts_with_existing_block?(domain_block, existing_domain_block)
+    existing_domain_block.present? && (existing_domain_block.domain == TagManager.instance.normalize_domain(domain_block.domain) || !domain_block.stricter_than?(existing_domain_block))
+  end
+
   def set_domain_blocks
     @domain_blocks = filtered_domain_blocks.order(id: :desc).to_a_paginated_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id))
   end
diff --git a/spec/requests/api/v1/admin/domain_blocks_spec.rb b/spec/requests/api/v1/admin/domain_blocks_spec.rb
index a6cfdb48d..58641c20d 100644
--- a/spec/requests/api/v1/admin/domain_blocks_spec.rb
+++ b/spec/requests/api/v1/admin/domain_blocks_spec.rb
@@ -135,14 +135,10 @@ RSpec.describe 'Domain Blocks' do
     it_behaves_like 'forbidden for wrong role', ''
     it_behaves_like 'forbidden for wrong role', 'Moderator'
 
-    it 'returns http success' do
+    it 'creates a domain block with the expected domain name and severity', :aggregate_failures do
       subject
 
       expect(response).to have_http_status(200)
-    end
-
-    it 'returns expected domain name and severity' do
-      subject
 
       body = body_as_json
 
@@ -160,7 +156,44 @@ RSpec.describe 'Domain Blocks' do
       expect(DomainBlock.find_by(domain: 'foo.bar.com')).to be_present
     end
 
-    context 'when a stricter domain block already exists' do
+    context 'when a looser domain block already exists on a higher level domain' do
+      let(:params) { { domain: 'foo.bar.com', severity: :suspend } }
+
+      before do
+        Fabricate(:domain_block, domain: 'bar.com', severity: :silence)
+      end
+
+      it 'creates a domain block with the expected domain name and severity', :aggregate_failures do
+        subject
+
+        body = body_as_json
+
+        expect(response).to have_http_status(200)
+        expect(body).to match a_hash_including(
+          {
+            domain: 'foo.bar.com',
+            severity: 'suspend',
+          }
+        )
+
+        expect(DomainBlock.find_by(domain: 'foo.bar.com')).to be_present
+      end
+    end
+
+    context 'when a domain block already exists on the same domain' do
+      before do
+        Fabricate(:domain_block, domain: 'foo.bar.com', severity: :silence)
+      end
+
+      it 'returns existing domain block in error', :aggregate_failures do
+        subject
+
+        expect(response).to have_http_status(422)
+        expect(body_as_json[:existing_domain_block][:domain]).to eq('foo.bar.com')
+      end
+    end
+
+    context 'when a stricter domain block already exists on a higher level domain' do
       before do
         Fabricate(:domain_block, domain: 'bar.com', severity: :suspend)
       end

From 67b2e62331f8459a3daa8ab06c8d18af4738e526 Mon Sep 17 00:00:00 2001
From: Emelia Smith <ThisIsMissEm@users.noreply.github.com>
Date: Tue, 30 Apr 2024 10:48:02 +0200
Subject: [PATCH 085/117] Fix missing destory audit logs for Domain Allows
 (#30125)

---
 app/controllers/admin/domain_allows_controller.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/controllers/admin/domain_allows_controller.rb b/app/controllers/admin/domain_allows_controller.rb
index 31be1978b..b0f139e3a 100644
--- a/app/controllers/admin/domain_allows_controller.rb
+++ b/app/controllers/admin/domain_allows_controller.rb
@@ -25,6 +25,8 @@ class Admin::DomainAllowsController < Admin::BaseController
   def destroy
     authorize @domain_allow, :destroy?
     UnallowDomainService.new.call(@domain_allow)
+    log_action :destroy, @domain_allow
+
     redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.destroyed_msg')
   end
 

From 8cf78825a23184333a4d9ac775e81add5bd5f0da Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 15 May 2024 15:11:13 +0200
Subject: [PATCH 086/117] Fix off-by-one in `tootctl media` commands (#30306)

---
 lib/mastodon/cli/media.rb | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/mastodon/cli/media.rb b/lib/mastodon/cli/media.rb
index c90616177..5879c532e 100644
--- a/lib/mastodon/cli/media.rb
+++ b/lib/mastodon/cli/media.rb
@@ -134,7 +134,7 @@ module Mastodon::CLI
 
             model_name      = path_segments.first.classify
             attachment_name = path_segments[1].singularize
-            record_id       = path_segments[2..-2].join.to_i
+            record_id       = path_segments[2...-2].join.to_i
             file_name       = path_segments.last
             record          = record_map.dig(model_name, record_id)
             attachment      = record&.public_send(attachment_name)
@@ -180,7 +180,7 @@ module Mastodon::CLI
           end
 
           model_name      = path_segments.first.classify
-          record_id       = path_segments[2..-2].join.to_i
+          record_id       = path_segments[2...-2].join.to_i
           attachment_name = path_segments[1].singularize
           file_name       = path_segments.last
 
@@ -311,7 +311,7 @@ module Mastodon::CLI
       end
 
       model_name = path_segments.first.classify
-      record_id  = path_segments[2..-2].join.to_i
+      record_id  = path_segments[2...-2].join.to_i
 
       unless PRELOAD_MODEL_WHITELIST.include?(model_name)
         say("Cannot find corresponding model: #{model_name}", :red)
@@ -361,7 +361,7 @@ module Mastodon::CLI
         next unless VALID_PATH_SEGMENTS_SIZE.include?(segments.size)
 
         model_name = segments.first.classify
-        record_id  = segments[2..-2].join.to_i
+        record_id  = segments[2...-2].join.to_i
 
         next unless PRELOAD_MODEL_WHITELIST.include?(model_name)
 

From 8c72e80019ea01d50bd0a868441083688028615b Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 10 May 2024 15:59:50 +0200
Subject: [PATCH 087/117] Update dependency rack-cors to 2.0.2

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 94d2d0ba4..2deab967c 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -537,7 +537,7 @@ GEM
     rack (2.2.8.1)
     rack-attack (6.7.0)
       rack (>= 1.0, < 4)
-    rack-cors (2.0.1)
+    rack-cors (2.0.2)
       rack (>= 2.0.0)
     rack-oauth2 (1.21.3)
       activesupport

From 2865bfadafc1f4472e9012ff4f03329b5b19d321 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 10 May 2024 16:01:07 +0200
Subject: [PATCH 088/117] Update dependency json-jwt to 1.15.3.1

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 2deab967c..d08f79ccc 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -368,7 +368,7 @@ GEM
     jmespath (1.6.2)
     json (2.6.3)
     json-canonicalization (1.0.0)
-    json-jwt (1.15.3)
+    json-jwt (1.15.3.1)
       activesupport (>= 4.2)
       aes_key_wrap
       bindata

From 997b021b69e4f8fe8750e27c28bc3ffd5fe6cee8 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 10 May 2024 16:03:48 +0200
Subject: [PATCH 089/117] Update dependency rotp to 6.3.0

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index d08f79ccc..bf438bd34 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -606,7 +606,7 @@ GEM
       actionpack (>= 5.2)
       railties (>= 5.2)
     rexml (3.2.6)
-    rotp (6.2.2)
+    rotp (6.3.0)
     rouge (4.1.2)
     rpam2 (4.0.2)
     rqrcode (2.2.0)

From 6fc07ff31f60a3c90c593c4c19ef9513f82a0124 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 10 May 2024 16:04:19 +0200
Subject: [PATCH 090/117] Update dependency fastimage to 2.3.1

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index bf438bd34..57783a522 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -288,7 +288,7 @@ GEM
     faraday_middleware (1.2.0)
       faraday (~> 1.0)
     fast_blank (1.0.1)
-    fastimage (2.2.7)
+    fastimage (2.3.1)
     ffi (1.15.5)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)

From a8dd32102fd5827c7a93df9ef6c2db21da43dde5 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 16 May 2024 10:11:21 +0200
Subject: [PATCH 091/117] Update dependency nokogiri to 1.16.5

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 57783a522..d9cb59c03 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -469,7 +469,7 @@ GEM
       net-protocol
     net-ssh (7.1.0)
     nio4r (2.7.0)
-    nokogiri (1.16.2)
+    nokogiri (1.16.5)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     nsa (0.3.0)

From 16213a678d391c18f283676648bbe235da3f6835 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 29 May 2024 11:09:54 +0200
Subject: [PATCH 092/117] Update dependency rexml to 3.2.8

---
 Gemfile.lock | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index d9cb59c03..891566e1a 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -605,7 +605,8 @@ GEM
     responders (3.1.0)
       actionpack (>= 5.2)
       railties (>= 5.2)
-    rexml (3.2.6)
+    rexml (3.2.8)
+      strscan (>= 3.0.9)
     rotp (6.3.0)
     rouge (4.1.2)
     rpam2 (4.0.2)
@@ -731,6 +732,7 @@ GEM
       redlock (~> 1.0)
     strong_migrations (0.8.0)
       activerecord (>= 5.2)
+    strscan (3.0.9)
     swd (1.3.0)
       activesupport (>= 3)
       attr_required (>= 0.0.5)

From b8edc95e8ad4ac07db311bc931cb09348b44820a Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 29 May 2024 10:15:06 +0200
Subject: [PATCH 093/117] Fix leaking Elasticsearch connections in Sidekiq
 processes (#30450)

---
 lib/mastodon/sidekiq_middleware.rb | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/lib/mastodon/sidekiq_middleware.rb b/lib/mastodon/sidekiq_middleware.rb
index 3a747afb6..ac63d4d38 100644
--- a/lib/mastodon/sidekiq_middleware.rb
+++ b/lib/mastodon/sidekiq_middleware.rb
@@ -8,6 +8,7 @@ class Mastodon::SidekiqMiddleware
   rescue Mastodon::HostValidationError
     # Do not retry
   rescue => e
+    clean_up_elasticsearch_connections!
     limit_backtrace_and_raise(e)
   ensure
     clean_up_sockets!
@@ -25,6 +26,32 @@ class Mastodon::SidekiqMiddleware
     clean_up_statsd_socket!
   end
 
+  # This is a hack to immediately free up unused Elasticsearch connections.
+  #
+  # Indeed, Chewy creates one `Elasticsearch::Client` instance per thread,
+  # and each such client manages its long-lasting connection to
+  # Elasticsearch.
+  #
+  # As far as I know, neither `chewy`,  `elasticsearch-transport` or even
+  # `faraday` provide a reliable way to immediately close a connection, and
+  # rely on the underlying object to be garbage-collected instead.
+  #
+  # Furthermore, `sidekiq` creates a new thread each time a job throws an
+  # exception, meaning that each failure will create a new connection, and
+  # the old one will only be closed on full garbage collection.
+  def clean_up_elasticsearch_connections!
+    return unless Chewy.enabled? && Chewy.current[:chewy_client].present?
+
+    Chewy.client.transport.connections.each do |connection|
+      # NOTE: This bit of code is tailored for the HTTPClient Faraday adapter
+      connection.connection.app.instance_variable_get(:@client)&.reset_all
+    end
+
+    Chewy.current.delete(:chewy_client)
+  rescue
+    nil
+  end
+
   def clean_up_redis_socket!
     RedisConfiguration.pool.checkin if Thread.current[:redis]
     Thread.current[:redis] = nil

From f9c41ae43b24b3561f122dcb1e07d4cbd25b1806 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 23 May 2024 19:28:18 +0200
Subject: [PATCH 094/117] Normalize language code of incoming posts (#30403)

---
 app/lib/activitypub/parser/status_parser.rb   | 11 +++-
 .../activitypub/parser/status_parser_spec.rb  | 50 +++++++++++++++++++
 2 files changed, 59 insertions(+), 2 deletions(-)
 create mode 100644 spec/lib/activitypub/parser/status_parser_spec.rb

diff --git a/app/lib/activitypub/parser/status_parser.rb b/app/lib/activitypub/parser/status_parser.rb
index 45f5fc5bf..6699dc21b 100644
--- a/app/lib/activitypub/parser/status_parser.rb
+++ b/app/lib/activitypub/parser/status_parser.rb
@@ -3,6 +3,8 @@
 class ActivityPub::Parser::StatusParser
   include JsonLdHelper
 
+  NORMALIZED_LOCALE_NAMES = LanguagesHelper::SUPPORTED_LOCALES.keys.index_by(&:downcase).freeze
+
   # @param [Hash] json
   # @param [Hash] magic_values
   # @option magic_values [String] :followers_collection
@@ -86,6 +88,13 @@ class ActivityPub::Parser::StatusParser
   end
 
   def language
+    lang = raw_language_code
+    lang.presence && NORMALIZED_LOCALE_NAMES.fetch(lang.downcase.to_sym, lang)
+  end
+
+  private
+
+  def raw_language_code
     if content_language_map?
       @object['contentMap'].keys.first
     elsif name_language_map?
@@ -95,8 +104,6 @@ class ActivityPub::Parser::StatusParser
     end
   end
 
-  private
-
   def audience_to
     as_array(@object['to'] || @json['to']).map { |x| value_or_id(x) }
   end
diff --git a/spec/lib/activitypub/parser/status_parser_spec.rb b/spec/lib/activitypub/parser/status_parser_spec.rb
new file mode 100644
index 000000000..5d9f008db
--- /dev/null
+++ b/spec/lib/activitypub/parser/status_parser_spec.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe ActivityPub::Parser::StatusParser do
+  subject { described_class.new(json) }
+
+  let(:sender) { Fabricate(:account, followers_url: 'http://example.com/followers', domain: 'example.com', uri: 'https://example.com/actor') }
+  let(:follower) { Fabricate(:account, username: 'bob') }
+
+  let(:json) do
+    {
+      '@context': 'https://www.w3.org/ns/activitystreams',
+      id: [ActivityPub::TagManager.instance.uri_for(sender), '#foo'].join,
+      type: 'Create',
+      actor: ActivityPub::TagManager.instance.uri_for(sender),
+      object: object_json,
+    }.with_indifferent_access
+  end
+
+  let(:object_json) do
+    {
+      id: [ActivityPub::TagManager.instance.uri_for(sender), 'post1'].join('/'),
+      type: 'Note',
+      to: [
+        'https://www.w3.org/ns/activitystreams#Public',
+        ActivityPub::TagManager.instance.uri_for(follower),
+      ],
+      content: '@bob lorem ipsum',
+      contentMap: {
+        EN: '@bob lorem ipsum',
+      },
+      published: 1.hour.ago.utc.iso8601,
+      updated: 1.hour.ago.utc.iso8601,
+      tag: {
+        type: 'Mention',
+        href: ActivityPub::TagManager.instance.uri_for(follower),
+      },
+    }
+  end
+
+  it 'correctly parses status' do
+    expect(subject).to have_attributes(
+      text: '@bob lorem ipsum',
+      uri: [ActivityPub::TagManager.instance.uri_for(sender), 'post1'].join('/'),
+      reply: false,
+      language: :en
+    )
+  end
+end

From 186f9161923f05cbc1999167caa9a2cf168749e9 Mon Sep 17 00:00:00 2001
From: Emelia Smith <ThisIsMissEm@users.noreply.github.com>
Date: Wed, 29 May 2024 16:00:05 +0200
Subject: [PATCH 095/117] Fix: remove broken OAuth Application vacuuming &
 throttle OAuth Application registrations (#30316)

Co-authored-by: Claire <claire.github-309c@sitedethib.com>
---
 app/lib/vacuum/applications_vacuum.rb        | 10 ----
 app/workers/scheduler/vacuum_scheduler.rb    |  5 --
 config/initializers/rack_attack.rb           |  4 ++
 spec/config/initializers/rack_attack_spec.rb | 18 ++++++++
 spec/lib/vacuum/applications_vacuum_spec.rb  | 48 --------------------
 5 files changed, 22 insertions(+), 63 deletions(-)
 delete mode 100644 app/lib/vacuum/applications_vacuum.rb
 delete mode 100644 spec/lib/vacuum/applications_vacuum_spec.rb

diff --git a/app/lib/vacuum/applications_vacuum.rb b/app/lib/vacuum/applications_vacuum.rb
deleted file mode 100644
index ba88655f1..000000000
--- a/app/lib/vacuum/applications_vacuum.rb
+++ /dev/null
@@ -1,10 +0,0 @@
-# frozen_string_literal: true
-
-class Vacuum::ApplicationsVacuum
-  def perform
-    Doorkeeper::Application.where(owner_id: nil)
-                           .where.missing(:created_users, :access_tokens, :access_grants)
-                           .where(created_at: ...1.day.ago)
-                           .in_batches.delete_all
-  end
-end
diff --git a/app/workers/scheduler/vacuum_scheduler.rb b/app/workers/scheduler/vacuum_scheduler.rb
index 1c9a2aabe..c22d6f5f8 100644
--- a/app/workers/scheduler/vacuum_scheduler.rb
+++ b/app/workers/scheduler/vacuum_scheduler.rb
@@ -22,7 +22,6 @@ class Scheduler::VacuumScheduler
       preview_cards_vacuum,
       backups_vacuum,
       access_tokens_vacuum,
-      applications_vacuum,
       feeds_vacuum,
       imports_vacuum,
     ]
@@ -56,10 +55,6 @@ class Scheduler::VacuumScheduler
     Vacuum::ImportsVacuum.new
   end
 
-  def applications_vacuum
-    Vacuum::ApplicationsVacuum.new
-  end
-
   def content_retention_policy
     ContentRetentionPolicy.current
   end
diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb
index d0af0fe94..482f4b95a 100644
--- a/config/initializers/rack_attack.rb
+++ b/config/initializers/rack_attack.rb
@@ -105,6 +105,10 @@ class Rack::Attack
     req.authenticated_user_id if (req.post? && req.path.match?(API_DELETE_REBLOG_REGEX)) || (req.delete? && req.path.match?(API_DELETE_STATUS_REGEX))
   end
 
+  throttle('throttle_oauth_application_registrations/ip', limit: 5, period: 10.minutes) do |req|
+    req.throttleable_remote_ip if req.post? && req.path == '/api/v1/apps'
+  end
+
   throttle('throttle_sign_up_attempts/ip', limit: 25, period: 5.minutes) do |req|
     req.throttleable_remote_ip if req.post? && req.path_matches?('/auth')
   end
diff --git a/spec/config/initializers/rack_attack_spec.rb b/spec/config/initializers/rack_attack_spec.rb
index 7cd4ac76b..ddbd74705 100644
--- a/spec/config/initializers/rack_attack_spec.rb
+++ b/spec/config/initializers/rack_attack_spec.rb
@@ -103,4 +103,22 @@ describe Rack::Attack, type: :request do
       it_behaves_like 'throttled endpoint'
     end
   end
+
+  describe 'throttle excessive oauth application registration requests by IP address' do
+    let(:throttle) { 'throttle_oauth_application_registrations/ip' }
+    let(:limit)  { 5 }
+    let(:period) { 10.minutes }
+    let(:path)   { '/api/v1/apps' }
+    let(:params) do
+      {
+        client_name: 'Throttle Test',
+        redirect_uris: 'urn:ietf:wg:oauth:2.0:oob',
+        scopes: 'read',
+      }
+    end
+
+    let(:request) { -> { post path, params: params, headers: { 'REMOTE_ADDR' => remote_ip } } }
+
+    it_behaves_like 'throttled endpoint'
+  end
 end
diff --git a/spec/lib/vacuum/applications_vacuum_spec.rb b/spec/lib/vacuum/applications_vacuum_spec.rb
deleted file mode 100644
index 57a222aaf..000000000
--- a/spec/lib/vacuum/applications_vacuum_spec.rb
+++ /dev/null
@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-RSpec.describe Vacuum::ApplicationsVacuum do
-  subject { described_class.new }
-
-  describe '#perform' do
-    let!(:app_with_token)  { Fabricate(:application, created_at: 1.month.ago) }
-    let!(:app_with_grant)  { Fabricate(:application, created_at: 1.month.ago) }
-    let!(:app_with_signup) { Fabricate(:application, created_at: 1.month.ago) }
-    let!(:app_with_owner)  { Fabricate(:application, created_at: 1.month.ago, owner: Fabricate(:user)) }
-    let!(:unused_app)      { Fabricate(:application, created_at: 1.month.ago) }
-    let!(:recent_app)      { Fabricate(:application, created_at: 1.hour.ago) }
-
-    let!(:active_access_token) { Fabricate(:access_token, application: app_with_token) }
-    let!(:active_access_grant) { Fabricate(:access_grant, application: app_with_grant) }
-    let!(:user) { Fabricate(:user, created_by_application: app_with_signup) }
-
-    before do
-      subject.perform
-    end
-
-    it 'does not delete applications with valid access tokens' do
-      expect { app_with_token.reload }.to_not raise_error
-    end
-
-    it 'does not delete applications with valid access grants' do
-      expect { app_with_grant.reload }.to_not raise_error
-    end
-
-    it 'does not delete applications that were used to create users' do
-      expect { app_with_signup.reload }.to_not raise_error
-    end
-
-    it 'does not delete owned applications' do
-      expect { app_with_owner.reload }.to_not raise_error
-    end
-
-    it 'does not delete applications registered less than a day ago' do
-      expect { recent_app.reload }.to_not raise_error
-    end
-
-    it 'deletes unused applications' do
-      expect { unused_app.reload }.to raise_error ActiveRecord::RecordNotFound
-    end
-  end
-end

From 943792c187392dc0a0f0b4e34dae5ddbc5c169b3 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 30 May 2024 14:03:13 +0200
Subject: [PATCH 096/117] Merge pull request from GHSA-5fq7-3p3j-9vrf

---
 app/services/notify_service.rb       | 17 +++++++++--------
 spec/services/notify_service_spec.rb |  8 ++++----
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/app/services/notify_service.rb b/app/services/notify_service.rb
index 125883b15..759d6e393 100644
--- a/app/services/notify_service.rb
+++ b/app/services/notify_service.rb
@@ -71,16 +71,17 @@ class NotifyService < BaseService
           LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
           WHERE s.id = :id
         UNION ALL
-          SELECT s.id, s.in_reply_to_id, m.id, st.path || s.id, st.depth + 1
-          FROM ancestors st
-          JOIN statuses s ON s.id = st.in_reply_to_id
-          LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
-          WHERE st.mention_id IS NULL AND NOT s.id = ANY(path) AND st.depth < :depth_limit
+          SELECT s.id, s.in_reply_to_id, m.id, ancestors.path || s.id, ancestors.depth + 1
+          FROM ancestors
+          JOIN statuses s ON s.id = ancestors.in_reply_to_id
+          /* early exit if we already have a mention matching our requirements */
+          LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id AND s.account_id = :recipient_id
+          WHERE ancestors.mention_id IS NULL AND NOT s.id = ANY(path) AND ancestors.depth < :depth_limit
       )
       SELECT COUNT(*)
-      FROM ancestors st
-      JOIN statuses s ON s.id = st.id
-      WHERE st.mention_id IS NOT NULL AND s.visibility = 3
+      FROM ancestors
+      JOIN statuses s ON s.id = ancestors.id
+      WHERE ancestors.mention_id IS NOT NULL AND s.account_id = :recipient_id AND s.visibility = 3
     SQL
   end
 
diff --git a/spec/services/notify_service_spec.rb b/spec/services/notify_service_spec.rb
index 8fcb58658..c2664e79c 100644
--- a/spec/services/notify_service_spec.rb
+++ b/spec/services/notify_service_spec.rb
@@ -76,10 +76,10 @@ RSpec.describe NotifyService, type: :service do
       end
 
       context 'when the message chain is initiated by recipient, but without a mention to the sender, even if the sender sends multiple messages in a row' do
-        let(:reply_to) { Fabricate(:status, account: recipient) }
-        let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) }
-        let(:dummy_reply) { Fabricate(:status, account: sender, visibility: :direct, thread: reply_to) }
-        let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: dummy_reply)) }
+        let(:public_status) { Fabricate(:status, account: recipient) }
+        let(:intermediate_reply) { Fabricate(:status, account: sender, thread: public_status, visibility: :direct) }
+        let!(:intermediate_mention) { Fabricate(:mention, account: sender, status: intermediate_reply) }
+        let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: intermediate_reply)) }
 
         it 'does not notify' do
           expect { subject }.to_not change(Notification, :count)

From 7920aa59e85bf21a4f6e2e9d485cd82dac89554c Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 30 May 2024 14:14:04 +0200
Subject: [PATCH 097/117] Merge pull request from GHSA-q3rg-xx5v-4mxh

---
 config/initializers/rack_attack.rb           | 10 +++++++-
 spec/config/initializers/rack_attack_spec.rb | 24 ++++++++++++++++++++
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb
index 482f4b95a..426b14b25 100644
--- a/config/initializers/rack_attack.rb
+++ b/config/initializers/rack_attack.rb
@@ -30,13 +30,17 @@ class Rack::Attack
     end
 
     def authenticated_user_id
-      authenticated_token&.resource_owner_id
+      authenticated_token&.resource_owner_id || warden_user_id
     end
 
     def authenticated_token_id
       authenticated_token&.id
     end
 
+    def warden_user_id
+      @env['warden']&.user&.id
+    end
+
     def unauthenticated?
       !authenticated_user_id
     end
@@ -141,6 +145,10 @@ class Rack::Attack
     req.session[:attempt_user_id] || req.params.dig('user', 'email').presence if req.post? && req.path_matches?('/auth/sign_in')
   end
 
+  throttle('throttle_password_change/account', limit: 10, period: 10.minutes) do |req|
+    req.authenticated_user_id if req.put? || (req.patch? && req.path_matches?('/auth'))
+  end
+
   self.throttled_responder = lambda do |request|
     now        = Time.now.utc
     match_data = request.env['rack.attack.match_data']
diff --git a/spec/config/initializers/rack_attack_spec.rb b/spec/config/initializers/rack_attack_spec.rb
index ddbd74705..78c4bf03a 100644
--- a/spec/config/initializers/rack_attack_spec.rb
+++ b/spec/config/initializers/rack_attack_spec.rb
@@ -121,4 +121,28 @@ describe Rack::Attack, type: :request do
 
     it_behaves_like 'throttled endpoint'
   end
+
+  describe 'throttle excessive password change requests by account' do
+    let(:user) { Fabricate(:user, email: 'user@host.example') }
+    let(:limit) { 10 }
+    let(:period) { 10.minutes }
+    let(:request) { -> { put path, headers: { 'REMOTE_ADDR' => remote_ip } } }
+    let(:path) { '/auth' }
+
+    before do
+      sign_in user, scope: :user
+
+      # Unfortunately, devise's `sign_in` helper causes the `session` to be
+      # loaded in the next request regardless of whether it's actually accessed
+      # by the client code.
+      #
+      # So, we make an extra query to clear issue a session cookie instead.
+      #
+      # A less resource-intensive way to deal with that would be to generate the
+      # session cookie manually, but this seems pretty involved.
+      get '/'
+    end
+
+    it_behaves_like 'throttled endpoint'
+  end
 end

From 8ab0ca7d64b9f7ec9d658d9a90eae1120ccef117 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 30 May 2024 14:24:29 +0200
Subject: [PATCH 098/117] Merge pull request from GHSA-c2r5-cfqr-c553

* Add hardening monkey-patch to prevent IP spoofing on misconfigured installations

* Remove rack-attack safelist
---
 config/application.rb                       |  1 +
 config/initializers/rack_attack.rb          |  4 --
 lib/action_dispatch/remote_ip_extensions.rb | 72 +++++++++++++++++++++
 3 files changed, 73 insertions(+), 4 deletions(-)
 create mode 100644 lib/action_dispatch/remote_ip_extensions.rb

diff --git a/config/application.rb b/config/application.rb
index 2a62c37e8..c2a63c869 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -48,6 +48,7 @@ require_relative '../lib/chewy/strategy/bypass_with_warning'
 require_relative '../lib/webpacker/manifest_extensions'
 require_relative '../lib/webpacker/helper_extensions'
 require_relative '../lib/rails/engine_extensions'
+require_relative '../lib/action_dispatch/remote_ip_extensions'
 require_relative '../lib/active_record/database_tasks_extensions'
 require_relative '../lib/active_record/batches'
 require_relative '../lib/simple_navigation/item_extensions'
diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb
index 426b14b25..962f02994 100644
--- a/config/initializers/rack_attack.rb
+++ b/config/initializers/rack_attack.rb
@@ -62,10 +62,6 @@ class Rack::Attack
     end
   end
 
-  Rack::Attack.safelist('allow from localhost') do |req|
-    req.remote_ip == '127.0.0.1' || req.remote_ip == '::1'
-  end
-
   Rack::Attack.blocklist('deny from blocklist') do |req|
     IpBlock.blocked?(req.remote_ip)
   end
diff --git a/lib/action_dispatch/remote_ip_extensions.rb b/lib/action_dispatch/remote_ip_extensions.rb
new file mode 100644
index 000000000..e5c48bf3c
--- /dev/null
+++ b/lib/action_dispatch/remote_ip_extensions.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# Mastodon is not made to be directly accessed without a reverse proxy.
+# This monkey-patch prevents remote IP address spoofing when being accessed
+# directly.
+#
+# See PR: https://github.com/rails/rails/pull/51610
+
+# In addition to the PR above, it also raises an error if a request with
+# `X-Forwarded-For` or `Client-Ip` comes directly from a client without
+# going through a trusted proxy.
+
+# rubocop:disable all -- This is a mostly vendored file
+
+module ActionDispatch
+  class RemoteIp
+    module GetIpExtensions
+      def calculate_ip
+        # Set by the Rack web server, this is a single value.
+        remote_addr = ips_from(@req.remote_addr).last
+
+        # Could be a CSV list and/or repeated headers that were concatenated.
+        client_ips    = ips_from(@req.client_ip).reverse!
+        forwarded_ips = ips_from(@req.x_forwarded_for).reverse!
+
+        # `Client-Ip` and `X-Forwarded-For` should not, generally, both be set. If they
+        # are both set, it means that either:
+        #
+        # 1) This request passed through two proxies with incompatible IP header
+        #     conventions.
+        #
+        # 2) The client passed one of `Client-Ip` or `X-Forwarded-For`
+        #     (whichever the proxy servers weren't using) themselves.
+        #
+        # Either way, there is no way for us to determine which header is the right one
+        # after the fact. Since we have no idea, if we are concerned about IP spoofing
+        # we need to give up and explode. (If you're not concerned about IP spoofing you
+        # can turn the `ip_spoofing_check` option off.)
+        should_check_ip = @check_ip && client_ips.last && forwarded_ips.last
+        if should_check_ip && !forwarded_ips.include?(client_ips.last)
+          # We don't know which came from the proxy, and which from the user
+          raise IpSpoofAttackError, "IP spoofing attack?! " \
+            "HTTP_CLIENT_IP=#{@req.client_ip.inspect} " \
+            "HTTP_X_FORWARDED_FOR=#{@req.x_forwarded_for.inspect}"
+        end
+
+        # NOTE: Mastodon addition to make sure we don't get requests from a non-trusted client
+        if @check_ip && (forwarded_ips.last || client_ips.last) && !@proxies.any? { |proxy| proxy === remote_addr }
+          raise IpSpoofAttackError, "IP spoofing attack?! client #{remote_addr} is not a trusted proxy " \
+            "HTTP_CLIENT_IP=#{@req.client_ip.inspect} " \
+            "HTTP_X_FORWARDED_FOR=#{@req.x_forwarded_for.inspect}"
+        end
+
+        # We assume these things about the IP headers:
+        #
+        #     - X-Forwarded-For will be a list of IPs, one per proxy, or blank
+        #     - Client-Ip is propagated from the outermost proxy, or is blank
+        #     - REMOTE_ADDR will be the IP that made the request to Rack
+        ips = forwarded_ips + client_ips
+        ips.compact!
+
+        # If every single IP option is in the trusted list, return the IP that's
+        # furthest away
+        filter_proxies([remote_addr] + ips).first || ips.last || remote_addr
+      end
+    end
+  end
+end
+
+ActionDispatch::RemoteIp::GetIp.prepend(ActionDispatch::RemoteIp::GetIpExtensions)
+
+# rubocop:enable all

From 9740c7eaea07629a4d6154a3ed26995110737a0c Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 30 May 2024 14:56:18 +0200
Subject: [PATCH 099/117] Fix rate-limiting incorrectly triggering a session
 cookie on most endpoints (#30483)

---
 config/initializers/rack_attack.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb
index 962f02994..6d8284e2b 100644
--- a/config/initializers/rack_attack.rb
+++ b/config/initializers/rack_attack.rb
@@ -30,7 +30,7 @@ class Rack::Attack
     end
 
     def authenticated_user_id
-      authenticated_token&.resource_owner_id || warden_user_id
+      authenticated_token&.resource_owner_id
     end
 
     def authenticated_token_id
@@ -142,7 +142,7 @@ class Rack::Attack
   end
 
   throttle('throttle_password_change/account', limit: 10, period: 10.minutes) do |req|
-    req.authenticated_user_id if req.put? || (req.patch? && req.path_matches?('/auth'))
+    req.warden_user_id if req.put? || (req.patch? && req.path_matches?('/auth'))
   end
 
   self.throttled_responder = lambda do |request|

From c93aacafdea188cac791b62a32e3117a7dc3e9cc Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 30 May 2024 15:34:50 +0200
Subject: [PATCH 100/117] Bump version to v4.2.9 (#30470)

---
 CHANGELOG.md            | 35 +++++++++++++++++++++++++++++++++++
 docker-compose.yml      |  6 +++---
 lib/mastodon/version.rb |  2 +-
 3 files changed, 39 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 636b3c161..8e579e148 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,41 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.2.9] - 2024-05-30
+
+### Security
+
+- Update dependencies
+- Fix private mention filtering ([GHSA-5fq7-3p3j-9vrf](https://github.com/mastodon/mastodon/security/advisories/GHSA-5fq7-3p3j-9vrf))
+- Fix password change endpoint not being rate-limited ([GHSA-q3rg-xx5v-4mxh](https://github.com/mastodon/mastodon/security/advisories/GHSA-q3rg-xx5v-4mxh))
+- Add hardening around rate-limit bypass ([GHSA-c2r5-cfqr-c553](https://github.com/mastodon/mastodon/security/advisories/GHSA-c2r5-cfqr-c553))
+
+### Added
+
+- Add rate-limit on OAuth application registration ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30316))
+- Add fallback redirection when getting a webfinger query `WEB_DOMAIN@WEB_DOMAIN` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28592))
+- Add `digest` attribute to `Admin::DomainBlock` entity in REST API ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/29092))
+
+### Removed
+
+- Remove superfluous application-level caching in some controllers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29862))
+- Remove aggressive OAuth application vacuuming ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30316))
+
+### Fixed
+
+- Fix leaking Elasticsearch connections in Sidekiq processes ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30450))
+- Fix language of remote posts not being recognized when using unusual casing ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30403))
+- Fix off-by-one in `tootctl media` commands ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30306))
+- Fix removal of allowed domains (in `LIMITED_FEDERATION_MODE`) not being recorded in the audit log ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30125))
+- Fix not being able to block a subdomain of an already-blocked domain through the API ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30119))
+- Fix `Idempotency-Key` being ignored when scheduling a post ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30084))
+- Fix crash when supplying the `FFMPEG_BINARY` environment variable ([timothyjrogers](https://github.com/mastodon/mastodon/pull/30022))
+- Fix improper email address validation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29838))
+- Fix results/query in `api/v1/featured_tags/suggestions` ([mjankowski](https://github.com/mastodon/mastodon/pull/29597))
+- Fix unblocking internationalized domain names under certain conditions ([tribela](https://github.com/mastodon/mastodon/pull/29530))
+- Fix admin account created by `mastodon:setup` not being auto-approved ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29379))
+- Fix reference to non-existent var in CLI maintenance command ([mjankowski](https://github.com/mastodon/mastodon/pull/28363))
+
 ## [4.2.8] - 2024-02-23
 
 ### Added
diff --git a/docker-compose.yml b/docker-compose.yml
index 004f5dbb0..10a57e31e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -56,7 +56,7 @@ services:
 
   web:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.8
+    image: ghcr.io/mastodon/mastodon:v4.2.9
     restart: always
     env_file: .env.production
     command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@@ -77,7 +77,7 @@ services:
 
   streaming:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.8
+    image: ghcr.io/mastodon/mastodon:v4.2.9
     restart: always
     env_file: .env.production
     command: node ./streaming
@@ -95,7 +95,7 @@ services:
 
   sidekiq:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.8
+    image: ghcr.io/mastodon/mastodon:v4.2.9
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq
diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb
index b28349dce..f9088382f 100644
--- a/lib/mastodon/version.rb
+++ b/lib/mastodon/version.rb
@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      8
+      9
     end
 
     def default_prerelease

From 520b2086af066d6434a6f2652870a6bb620f4e23 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 12 Oct 2023 16:47:18 +0200
Subject: [PATCH 101/117] Change PWA start URL from `/home` to `/` (#27377)

---
 app/serializers/manifest_serializer.rb | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/app/serializers/manifest_serializer.rb b/app/serializers/manifest_serializer.rb
index 48f3aa7a6..b6533815a 100644
--- a/app/serializers/manifest_serializer.rb
+++ b/app/serializers/manifest_serializer.rb
@@ -16,11 +16,18 @@ class ManifestSerializer < ActiveModel::Serializer
     512
   ).freeze
 
-  attributes :name, :short_name,
+  attributes :id, :name, :short_name,
              :icons, :theme_color, :background_color,
              :display, :start_url, :scope,
              :share_target, :shortcuts
 
+  def id
+    # This is set to `/home` because that was the old value of `start_url` and
+    # thus the fallback ID computed by Chrome:
+    # https://developer.chrome.com/blog/pwa-manifest-id/
+    '/home'
+  end
+
   def name
     object.title
   end
@@ -53,7 +60,7 @@ class ManifestSerializer < ActiveModel::Serializer
   end
 
   def start_url
-    '/home'
+    '/'
   end
 
   def scope

From 19ed22dc582e9fd922b221034953c428dd0de856 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 12 Jun 2024 09:28:28 +0200
Subject: [PATCH 102/117] Fix duplicate `@context` attribute in user export
 (#30653)

---
 app/services/backup_service.rb       |  4 ++--
 spec/services/backup_service_spec.rb | 10 ++++++++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/app/services/backup_service.rb b/app/services/backup_service.rb
index 3ef0366c3..b3282c409 100644
--- a/app/services/backup_service.rb
+++ b/app/services/backup_service.rb
@@ -19,8 +19,8 @@ class BackupService < BaseService
 
   def build_outbox_json!(file)
     skeleton = serialize(collection_presenter, ActivityPub::CollectionSerializer)
-    skeleton[:@context] = full_context
-    skeleton[:orderedItems] = ['!PLACEHOLDER!']
+    skeleton['@context'] = full_context
+    skeleton['orderedItems'] = ['!PLACEHOLDER!']
     skeleton = Oj.dump(skeleton)
     prepend, append = skeleton.split('"!PLACEHOLDER!"')
     add_comma = false
diff --git a/spec/services/backup_service_spec.rb b/spec/services/backup_service_spec.rb
index 806ba1832..1eb789d1f 100644
--- a/spec/services/backup_service_spec.rb
+++ b/spec/services/backup_service_spec.rb
@@ -55,9 +55,11 @@ RSpec.describe BackupService, type: :service do
   end
 
   def expect_outbox_export
-    json = export_json(:outbox)
+    body = export_json_raw(:outbox)
+    json = Oj.load(body)
 
     aggregate_failures do
+      expect(body.scan('@context').count).to eq 1
       expect(json['@context']).to_not be_nil
       expect(json['type']).to eq 'OrderedCollection'
       expect(json['totalItems']).to eq 2
@@ -85,8 +87,12 @@ RSpec.describe BackupService, type: :service do
     end
   end
 
+  def export_json_raw(type)
+    read_zip_file(backup, "#{type}.json")
+  end
+
   def export_json(type)
-    Oj.load(read_zip_file(backup, "#{type}.json"))
+    Oj.load(export_json_raw(type))
   end
 
   def include_create_item(status)

From bfc287fd6b83d9b32d0d7c34ed5d61f63c5bae35 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 20 Jun 2024 08:56:12 +0200
Subject: [PATCH 103/117] Remove dependency on posix-spawn (#18559)

---
 Gemfile      | 1 -
 Gemfile.lock | 2 --
 2 files changed, 3 deletions(-)

diff --git a/Gemfile b/Gemfile
index 1d038ad9f..7f7da8c57 100644
--- a/Gemfile
+++ b/Gemfile
@@ -65,7 +65,6 @@ gem 'nsa'
 gem 'oj', '~> 3.14'
 gem 'ox', '~> 2.14'
 gem 'parslet'
-gem 'posix-spawn'
 gem 'public_suffix', '~> 5.0'
 gem 'pundit', '~> 2.3'
 gem 'premailer-rails'
diff --git a/Gemfile.lock b/Gemfile.lock
index 891566e1a..2ef61e145 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -517,7 +517,6 @@ GEM
     pg (1.5.5)
     pghero (3.3.4)
       activerecord (>= 6)
-    posix-spawn (0.3.15)
     premailer (1.21.0)
       addressable
       css_parser (>= 1.12.0)
@@ -892,7 +891,6 @@ DEPENDENCIES
   parslet
   pg (~> 1.5)
   pghero
-  posix-spawn
   premailer-rails
   private_address_check (~> 0.5)
   public_suffix (~> 5.0)

From 3f75c6f048c75e5edd5749c0dfe4df1b935d0d3a Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 20 Jun 2024 08:59:41 +0200
Subject: [PATCH 104/117] Update dependency rails

---
 Gemfile.lock | 122 +++++++++++++++++++++++++--------------------------
 1 file changed, 61 insertions(+), 61 deletions(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index 2ef61e145..b3e19cd8f 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -28,47 +28,47 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (7.0.8.1)
-      actionpack (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    actioncable (7.0.8.4)
+      actionpack (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (7.0.8.1)
-      actionpack (= 7.0.8.1)
-      activejob (= 7.0.8.1)
-      activerecord (= 7.0.8.1)
-      activestorage (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    actionmailbox (7.0.8.4)
+      actionpack (= 7.0.8.4)
+      activejob (= 7.0.8.4)
+      activerecord (= 7.0.8.4)
+      activestorage (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
       mail (>= 2.7.1)
       net-imap
       net-pop
       net-smtp
-    actionmailer (7.0.8.1)
-      actionpack (= 7.0.8.1)
-      actionview (= 7.0.8.1)
-      activejob (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    actionmailer (7.0.8.4)
+      actionpack (= 7.0.8.4)
+      actionview (= 7.0.8.4)
+      activejob (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
       mail (~> 2.5, >= 2.5.4)
       net-imap
       net-pop
       net-smtp
       rails-dom-testing (~> 2.0)
-    actionpack (7.0.8.1)
-      actionview (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    actionpack (7.0.8.4)
+      actionview (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
       rack (~> 2.0, >= 2.2.4)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (7.0.8.1)
-      actionpack (= 7.0.8.1)
-      activerecord (= 7.0.8.1)
-      activestorage (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    actiontext (7.0.8.4)
+      actionpack (= 7.0.8.4)
+      activerecord (= 7.0.8.4)
+      activestorage (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
       globalid (>= 0.6.0)
       nokogiri (>= 1.8.5)
-    actionview (7.0.8.1)
-      activesupport (= 7.0.8.1)
+    actionview (7.0.8.4)
+      activesupport (= 7.0.8.4)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
@@ -78,22 +78,22 @@ GEM
       activemodel (>= 4.1, < 7.1)
       case_transform (>= 0.2)
       jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
-    activejob (7.0.8.1)
-      activesupport (= 7.0.8.1)
+    activejob (7.0.8.4)
+      activesupport (= 7.0.8.4)
       globalid (>= 0.3.6)
-    activemodel (7.0.8.1)
-      activesupport (= 7.0.8.1)
-    activerecord (7.0.8.1)
-      activemodel (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
-    activestorage (7.0.8.1)
-      actionpack (= 7.0.8.1)
-      activejob (= 7.0.8.1)
-      activerecord (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    activemodel (7.0.8.4)
+      activesupport (= 7.0.8.4)
+    activerecord (7.0.8.4)
+      activemodel (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
+    activestorage (7.0.8.4)
+      actionpack (= 7.0.8.4)
+      activejob (= 7.0.8.4)
+      activerecord (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (7.0.8.1)
+    activesupport (7.0.8.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -350,7 +350,7 @@ GEM
     httplog (1.6.2)
       rack (>= 2.0)
       rainbow (>= 2.0.0)
-    i18n (1.14.1)
+    i18n (1.14.5)
       concurrent-ruby (~> 1.0)
     i18n-tasks (1.0.12)
       activesupport (>= 4.0.2)
@@ -432,7 +432,7 @@ GEM
       net-imap
       net-pop
       net-smtp
-    marcel (1.0.2)
+    marcel (1.0.4)
     mario-redis-lock (1.2.1)
       redis (>= 3.0.5)
     matrix (0.4.2)
@@ -446,7 +446,7 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2023.0808)
     mini_mime (1.1.5)
-    mini_portile2 (2.8.5)
+    mini_portile2 (2.8.7)
     minitest (5.19.0)
     msgpack (1.7.1)
     multi_json (1.15.0)
@@ -468,8 +468,8 @@ GEM
     net-smtp (0.3.4)
       net-protocol
     net-ssh (7.1.0)
-    nio4r (2.7.0)
-    nokogiri (1.16.5)
+    nio4r (2.7.3)
+    nokogiri (1.16.6)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     nsa (0.3.0)
@@ -533,7 +533,7 @@ GEM
       activesupport (>= 3.0.0)
     raabro (1.4.0)
     racc (1.7.3)
-    rack (2.2.8.1)
+    rack (2.2.9)
     rack-attack (6.7.0)
       rack (>= 1.0, < 4)
     rack-cors (2.0.2)
@@ -550,20 +550,20 @@ GEM
       rack
     rack-test (2.1.0)
       rack (>= 1.3)
-    rails (7.0.8.1)
-      actioncable (= 7.0.8.1)
-      actionmailbox (= 7.0.8.1)
-      actionmailer (= 7.0.8.1)
-      actionpack (= 7.0.8.1)
-      actiontext (= 7.0.8.1)
-      actionview (= 7.0.8.1)
-      activejob (= 7.0.8.1)
-      activemodel (= 7.0.8.1)
-      activerecord (= 7.0.8.1)
-      activestorage (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    rails (7.0.8.4)
+      actioncable (= 7.0.8.4)
+      actionmailbox (= 7.0.8.4)
+      actionmailer (= 7.0.8.4)
+      actionpack (= 7.0.8.4)
+      actiontext (= 7.0.8.4)
+      actionview (= 7.0.8.4)
+      activejob (= 7.0.8.4)
+      activemodel (= 7.0.8.4)
+      activerecord (= 7.0.8.4)
+      activestorage (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
       bundler (>= 1.15.0)
-      railties (= 7.0.8.1)
+      railties (= 7.0.8.4)
     rails-controller-testing (1.0.5)
       actionpack (>= 5.0.1.rc1)
       actionview (>= 5.0.1.rc1)
@@ -578,9 +578,9 @@ GEM
     rails-i18n (7.0.7)
       i18n (>= 0.7, < 2)
       railties (>= 6.0.0, < 8)
-    railties (7.0.8.1)
-      actionpack (= 7.0.8.1)
-      activesupport (= 7.0.8.1)
+    railties (7.0.8.4)
+      actionpack (= 7.0.8.4)
+      activesupport (= 7.0.8.4)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
@@ -743,7 +743,7 @@ GEM
     terrapin (0.6.0)
       climate_control (>= 0.0.3, < 1.0)
     test-prof (1.2.3)
-    thor (1.3.0)
+    thor (1.3.1)
     tilt (2.2.0)
     timeout (0.4.1)
     tpm-key_attestation (0.12.0)
@@ -809,7 +809,7 @@ GEM
     xorcist (1.1.3)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.6.13)
+    zeitwerk (2.6.16)
 
 PLATFORMS
   ruby

From 55408f80853b69fb08d3ae8f9a7daeab78b21a54 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 20 Jun 2024 09:00:08 +0200
Subject: [PATCH 105/117] Update dependency cbor

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index b3e19cd8f..edddcbd73 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -191,7 +191,7 @@ GEM
       xpath (~> 3.2)
     case_transform (0.2)
       activesupport
-    cbor (0.5.9.6)
+    cbor (0.5.9.8)
     charlock_holmes (0.7.7)
     chewy (7.3.4)
       activesupport (>= 5.2)

From fcae9435ec6cb22e536c7b4eef351ce0b402e3ef Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 24 Jun 2024 15:11:10 +0200
Subject: [PATCH 106/117] Fix `/admin/accounts/:account_id/statuses/:id` for
 edited posts with media attachments (#30819)

---
 app/models/status_edit.rb                          | 2 +-
 spec/controllers/admin/statuses_controller_spec.rb | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/app/models/status_edit.rb b/app/models/status_edit.rb
index 2b3248bb2..7de509dda 100644
--- a/app/models/status_edit.rb
+++ b/app/models/status_edit.rb
@@ -42,7 +42,7 @@ class StatusEdit < ApplicationRecord
   default_scope { order(id: :asc) }
 
   delegate :local?, :application, :edited?, :edited_at,
-           :discarded?, :visibility, to: :status
+           :discarded?, :visibility, :language, to: :status
 
   def emojis
     return @emojis if defined?(@emojis)
diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb
index 9befdf978..5c8fc0a63 100644
--- a/spec/controllers/admin/statuses_controller_spec.rb
+++ b/spec/controllers/admin/statuses_controller_spec.rb
@@ -43,6 +43,11 @@ describe Admin::StatusesController do
 
   describe 'GET #show' do
     before do
+      status.media_attachments << Fabricate(:media_attachment, type: :image, account: status.account)
+      status.save!
+      status.snapshot!(at_time: status.created_at, rate_limit: false)
+      status.update!(text: 'Hello, this is an edited post')
+      status.snapshot!(rate_limit: false)
       get :show, params: { account_id: account.id, id: status.id }
     end
 

From 5fd7cd79e0734d0f8d7d1250d4c1629def1a1b48 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Mon, 24 Jun 2024 15:49:39 +0200
Subject: [PATCH 107/117] Specify yarn version to avoid confusion with main
 which uses Yarn 4

---
 package.json | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 227399e80..35a236c8c 100644
--- a/package.json
+++ b/package.json
@@ -226,5 +226,6 @@
     "Capfile|Gemfile|*.{rb,ruby,ru,rake}": "bundle exec rubocop --force-exclusion -a",
     "*.{js,jsx,ts,tsx}": "eslint --fix",
     "*.{css,scss}": "stylelint --fix"
-  }
+  },
+  "packageManager": "yarn@1.22.22"
 }

From 1e87634a43b19953f3e2956003d176967f2e7a8d Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 28 Jun 2024 23:23:19 +0200
Subject: [PATCH 108/117] Update dependency charlock_holmes

---
 Gemfile.lock | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Gemfile.lock b/Gemfile.lock
index edddcbd73..34958703a 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -192,7 +192,7 @@ GEM
     case_transform (0.2)
       activesupport
     cbor (0.5.9.8)
-    charlock_holmes (0.7.7)
+    charlock_holmes (0.7.8)
     chewy (7.3.4)
       activesupport (>= 5.2)
       elasticsearch (>= 7.12.0, < 7.14.0)

From 17f69c0002e68851293b158e1e7cda685fdfb2dc Mon Sep 17 00:00:00 2001
From: Tim Rogers <rogers.timothy.john@gmail.com>
Date: Mon, 24 Jun 2024 09:41:04 -0500
Subject: [PATCH 109/117] Added check for STATSD_ADDR setting to emit a warning
 and proceed rather than crashing if the address is unreachable (#30691)

---
 config/initializers/statsd.rb | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/config/initializers/statsd.rb b/config/initializers/statsd.rb
index a655c1071..f1628a9d1 100644
--- a/config/initializers/statsd.rb
+++ b/config/initializers/statsd.rb
@@ -3,13 +3,17 @@
 if ENV['STATSD_ADDR'].present?
   host, port = ENV['STATSD_ADDR'].split(':')
 
-  statsd = Statsd.new(host, port)
-  statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') }
+  begin
+    statsd = Statsd.new(host, port)
+    statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') }
 
-  NSA.inform_statsd(statsd) do |informant|
-    informant.collect(:action_controller, :web)
-    informant.collect(:active_record, :db)
-    informant.collect(:active_support_cache, :cache)
-    informant.collect(:sidekiq, :sidekiq) if ENV['STATSD_SIDEKIQ'] == 'true'
+    NSA.inform_statsd(statsd) do |informant|
+      informant.collect(:action_controller, :web)
+      informant.collect(:active_record, :db)
+      informant.collect(:active_support_cache, :cache)
+      informant.collect(:sidekiq, :sidekiq) if ENV['STATSD_SIDEKIQ'] == 'true'
+    end
+  rescue
+    Rails.logger.warn("statsd address #{ENV['STATSD_ADDR']} not reachable, proceeding without statsd")
   end
 end

From 846f59c6e95ae0f6ccd5714bd1a57c8dd0fed017 Mon Sep 17 00:00:00 2001
From: David Roetzel <david@roetzel.de>
Date: Thu, 27 Jun 2024 16:40:19 +0200
Subject: [PATCH 110/117] Add size limit for link preview URLs (#30854)

---
 app/services/fetch_link_card_service.rb       |  5 ++++-
 spec/services/fetch_link_card_service_spec.rb | 13 +++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 13775e63c..8302a9ef4 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -15,6 +15,9 @@ class FetchLinkCardService < BaseService
     )
   }iox
 
+  # URL size limit to safely store in PosgreSQL's unique indexes
+  BYTESIZE_LIMIT = 2692
+
   def call(status)
     @status       = status
     @original_url = parse_urls
@@ -85,7 +88,7 @@ class FetchLinkCardService < BaseService
 
   def bad_url?(uri)
     # Avoid local instance URLs and invalid URLs
-    uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme)
+    uri.host.blank? || TagManager.instance.local_url?(uri.to_s) || !%w(http https).include?(uri.scheme) || uri.to_s.bytesize > BYTESIZE_LIMIT
   end
 
   def mention_link?(anchor)
diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb
index f44cbb750..fa4d7bb36 100644
--- a/spec/services/fetch_link_card_service_spec.rb
+++ b/spec/services/fetch_link_card_service_spec.rb
@@ -184,6 +184,19 @@ RSpec.describe FetchLinkCardService, type: :service do
       end
     end
 
+    context 'with an URL too long for PostgreSQL unique indexes' do
+      let(:url) { "http://example.com/#{'a' * 2674}" }
+      let(:status) { Fabricate(:status, text: url) }
+
+      it 'does not fetch the URL' do
+        expect(a_request(:get, url)).to_not have_been_made
+      end
+
+      it 'does not create a preview card' do
+        expect(status.preview_card).to be_nil
+      end
+    end
+
     context 'with a URL of a page with oEmbed support' do
       let(:html) { '<!doctype html><title>Hello world</title><link rel="alternate" type="application/json+oembed" href="http://example.com/oembed?url=http://example.com/html">' }
       let(:status) { Fabricate(:status, text: 'http://example.com/html') }

From 88b2d6eca57e4f8d90062d9eb9d511213bb00a1c Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 27 Jun 2024 23:34:34 +0200
Subject: [PATCH 111/117] Change search modifiers to be case-insensitive
 (#30865)

---
 app/lib/search_query_transformer.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/lib/search_query_transformer.rb b/app/lib/search_query_transformer.rb
index a45ae3d09..8849acf12 100644
--- a/app/lib/search_query_transformer.rb
+++ b/app/lib/search_query_transformer.rb
@@ -223,7 +223,7 @@ class SearchQueryTransformer < Parslet::Transform
   end
 
   rule(clause: subtree(:clause)) do
-    prefix   = clause[:prefix][:term].to_s if clause[:prefix]
+    prefix   = clause[:prefix][:term].to_s.downcase if clause[:prefix]
     operator = clause[:operator]&.to_s
     term     = clause[:phrase] ? clause[:phrase].map { |term| term[:term].to_s }.join(' ') : clause[:term].to_s
 

From 9b6219c48f4fafacc1f31c0f8d89f98948a46aa9 Mon Sep 17 00:00:00 2001
From: David Roetzel <david@roetzel.de>
Date: Fri, 21 Jun 2024 14:51:10 +0200
Subject: [PATCH 112/117] Improve encoding detection for link cards (#30780)

---
 app/lib/link_details_extractor.rb               | 15 ++++++++++-----
 .../fixtures/requests/low_confidence_latin1.txt | 17 +++++++++++++++++
 spec/services/fetch_link_card_service_spec.rb   |  9 +++++++++
 3 files changed, 36 insertions(+), 5 deletions(-)
 create mode 100644 spec/fixtures/requests/low_confidence_latin1.txt

diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb
index bb031986d..48a592a40 100644
--- a/app/lib/link_details_extractor.rb
+++ b/app/lib/link_details_extractor.rb
@@ -265,16 +265,21 @@ class LinkDetailsExtractor
   end
 
   def document
-    @document ||= Nokogiri::HTML(@html, nil, encoding)
+    @document ||= detect_encoding_and_parse_document
   end
 
-  def encoding
-    @encoding ||= begin
-      guess = detector.detect(@html, @html_charset)
-      guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
+  def detect_encoding_and_parse_document
+    [detect_encoding, nil, @html_charset, 'UTF-8'].uniq.each do |encoding|
+      document = Nokogiri::HTML(@html, nil, encoding)
+      return document if document.to_s.valid_encoding?
     end
   end
 
+  def detect_encoding
+    guess = detector.detect(@html, @html_charset)
+    guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
+  end
+
   def detector
     @detector ||= CharlockHolmes::EncodingDetector.new.tap do |detector|
       detector.strip_tags = true
diff --git a/spec/fixtures/requests/low_confidence_latin1.txt b/spec/fixtures/requests/low_confidence_latin1.txt
new file mode 100644
index 000000000..39c3e23d6
--- /dev/null
+++ b/spec/fixtures/requests/low_confidence_latin1.txt
@@ -0,0 +1,17 @@
+HTTP/1.1 200 OK
+server: nginx
+date: Thu, 13 Jun 2024 14:33:13 GMT
+content-type: text/html; charset=ISO-8859-1
+content-length: 158
+accept-ranges: bytes
+
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>Tofu � l'orange</title>
+</head>
+<body>
+  <h2>Tofu � l'orange</h2>
+</body>
+</html>
diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb
index fa4d7bb36..592dce24d 100644
--- a/spec/services/fetch_link_card_service_spec.rb
+++ b/spec/services/fetch_link_card_service_spec.rb
@@ -26,6 +26,7 @@ RSpec.describe FetchLinkCardService, type: :service do
     stub_request(:get, 'http://example.com/sjis_with_wrong_charset').to_return(request_fixture('sjis_with_wrong_charset.txt'))
     stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt'))
     stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt'))
+    stub_request(:get, 'http://example.com/low_confidence_latin1').to_return(request_fixture('low_confidence_latin1.txt'))
 
     Rails.cache.write('oembed_endpoint:example.com', oembed_cache) if oembed_cache
 
@@ -148,6 +149,14 @@ RSpec.describe FetchLinkCardService, type: :service do
       end
     end
 
+    context 'with a URL of a page in ISO-8859-1 encoding, that charlock_holmes cannot detect' do
+      let(:status) { Fabricate(:status, text: 'Check out http://example.com/low_confidence_latin1') }
+
+      it 'decodes the HTML' do
+        expect(status.preview_card.title).to eq("Tofu á l'orange")
+      end
+    end
+
     context 'with a Japanese path URL' do
       let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') }
 

From 6cd9bd6ae154eee02c42121ffaae280de907edbb Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Wed, 3 Jul 2024 09:15:47 +0200
Subject: [PATCH 113/117] fix: Return HTTP 422 when scheduled status time is
 less than 5 minutes (#30584)

---
 app/services/post_status_service.rb           |  2 +-
 .../api/v1/statuses_controller_spec.rb        | 40 +++++++++++++++++++
 spec/services/post_status_service_spec.rb     | 10 +++++
 3 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index e4dd480f1..280666dcf 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -171,7 +171,7 @@ class PostStatusService < BaseService
   end
 
   def scheduled_in_the_past?
-    @scheduled_at.present? && @scheduled_at <= Time.now.utc + MIN_SCHEDULE_OFFSET
+    @scheduled_at.present? && @scheduled_at <= Time.now.utc
   end
 
   def bump_potential_friendship!
diff --git a/spec/controllers/api/v1/statuses_controller_spec.rb b/spec/controllers/api/v1/statuses_controller_spec.rb
index c2bdba9ac..591307fe2 100644
--- a/spec/controllers/api/v1/statuses_controller_spec.rb
+++ b/spec/controllers/api/v1/statuses_controller_spec.rb
@@ -182,6 +182,46 @@ RSpec.describe Api::V1::StatusesController do
           expect(response.headers['X-RateLimit-Remaining']).to eq '0'
         end
       end
+
+      context 'with missing thread' do
+        subject { post :create, params: params }
+
+        let(:params) { { status: 'Hello world', in_reply_to_id: 0 } }
+
+        it 'returns http not found' do
+          subject
+
+          expect(response).to have_http_status(404)
+        end
+      end
+
+      context 'when scheduling a status' do
+        subject { post :create, params: params }
+
+        let(:params) { { status: 'Hello world', scheduled_at: 10.minutes.from_now } }
+        let(:account) { user.account }
+
+        it 'returns HTTP 200' do
+          subject
+
+          expect(response).to have_http_status(200)
+        end
+
+        it 'creates a scheduled status' do
+          expect { subject }.to change { account.scheduled_statuses.count }.from(0).to(1)
+        end
+
+        context 'when the scheduling time is less than 5 minutes' do
+          let(:params) { { status: 'Hello world', scheduled_at: 4.minutes.from_now } }
+
+          it 'does not create a scheduled status', :aggregate_failures do
+            subject
+
+            expect(response).to have_http_status(422)
+            expect(account.scheduled_statuses).to be_empty
+          end
+        end
+      end
     end
 
     describe 'DELETE #destroy' do
diff --git a/spec/services/post_status_service_spec.rb b/spec/services/post_status_service_spec.rb
index a74e82626..826af5479 100644
--- a/spec/services/post_status_service_spec.rb
+++ b/spec/services/post_status_service_spec.rb
@@ -61,6 +61,16 @@ RSpec.describe PostStatusService, type: :service do
       status2 = subject.call(account, text: 'test', idempotency: 'meepmeep', scheduled_at: future)
       expect(status2.id).to eq status1.id
     end
+
+    context 'when scheduled_at is less than min offset' do
+      let(:invalid_scheduled_time) { 4.minutes.from_now }
+
+      it 'raises invalid record error' do
+        expect do
+          subject.call(account, text: 'Hi future!', scheduled_at: invalid_scheduled_time)
+        end.to raise_error(ActiveRecord::RecordInvalid)
+      end
+    end
   end
 
   it 'creates response to the original status of boost' do

From df974a912b59e276ad7bfa413e2b628633a776d4 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 4 Jul 2024 16:11:28 +0200
Subject: [PATCH 114/117] Merge pull request from GHSA-vp5r-5pgw-jwqx

* Fix streaming sessions not being closed when revoking access to an app

* Add tests for GHSA-7w3c-p9j8-mq3x
---
 .../oauth/authorized_applications_controller.rb        |  1 +
 app/lib/application_extension.rb                       |  8 +++++---
 .../oauth/authorized_applications_controller_spec.rb   | 10 ++++++++++
 .../settings/applications_controller_spec.rb           |  8 ++++++++
 4 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/app/controllers/oauth/authorized_applications_controller.rb b/app/controllers/oauth/authorized_applications_controller.rb
index 350ae2e90..17f1be23d 100644
--- a/app/controllers/oauth/authorized_applications_controller.rb
+++ b/app/controllers/oauth/authorized_applications_controller.rb
@@ -17,6 +17,7 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
 
   def destroy
     Web::PushSubscription.unsubscribe_for(params[:id], current_resource_owner)
+    Doorkeeper::Application.find_by(id: params[:id])&.close_streaming_sessions(current_resource_owner)
     super
   end
 
diff --git a/app/lib/application_extension.rb b/app/lib/application_extension.rb
index 400c51a02..f226b99cd 100644
--- a/app/lib/application_extension.rb
+++ b/app/lib/application_extension.rb
@@ -16,17 +16,19 @@ module ApplicationExtension
     # dependent: delete_all, which means the ActiveRecord callback in
     # AccessTokenExtension is not run, so instead we manually announce to
     # streaming that these tokens are being deleted.
-    before_destroy :push_to_streaming_api, prepend: true
+    before_destroy :close_streaming_sessions, prepend: true
   end
 
   def confirmation_redirect_uri
     redirect_uri.lines.first.strip
   end
 
-  def push_to_streaming_api
+  def close_streaming_sessions(resource_owner = nil)
     # TODO: #28793 Combine into a single topic
     payload = Oj.dump(event: :kill)
-    access_tokens.in_batches do |tokens|
+    scope = access_tokens
+    scope = scope.where(resource_owner_id: resource_owner.id) unless resource_owner.nil?
+    scope.in_batches do |tokens|
       redis.pipelined do |pipeline|
         tokens.ids.each do |id|
           pipeline.publish("timeline:access_token:#{id}", payload)
diff --git a/spec/controllers/oauth/authorized_applications_controller_spec.rb b/spec/controllers/oauth/authorized_applications_controller_spec.rb
index b54610604..3fd9f9499 100644
--- a/spec/controllers/oauth/authorized_applications_controller_spec.rb
+++ b/spec/controllers/oauth/authorized_applications_controller_spec.rb
@@ -50,9 +50,11 @@ describe Oauth::AuthorizedApplicationsController do
     let!(:application) { Fabricate(:application) }
     let!(:access_token) { Fabricate(:accessible_access_token, application: application, resource_owner_id: user.id) }
     let!(:web_push_subscription) { Fabricate(:web_push_subscription, user: user, access_token: access_token) }
+    let(:redis_pipeline_stub) { instance_double(Redis::Namespace, publish: nil) }
 
     before do
       sign_in user, scope: :user
+      allow(redis).to receive(:pipelined).and_yield(redis_pipeline_stub)
       post :destroy, params: { id: application.id }
     end
 
@@ -63,5 +65,13 @@ describe Oauth::AuthorizedApplicationsController do
     it 'removes subscriptions for the application\'s access tokens' do
       expect(Web::PushSubscription.where(user: user).count).to eq 0
     end
+
+    it 'removes the web_push_subscription' do
+      expect { web_push_subscription.reload }.to raise_error(ActiveRecord::RecordNotFound)
+    end
+
+    it 'sends a session kill payload to the streaming server' do
+      expect(redis_pipeline_stub).to have_received(:publish).with("timeline:access_token:#{access_token.id}", '{"event":"kill"}')
+    end
   end
 end
diff --git a/spec/controllers/settings/applications_controller_spec.rb b/spec/controllers/settings/applications_controller_spec.rb
index 169304b3e..88974e551 100644
--- a/spec/controllers/settings/applications_controller_spec.rb
+++ b/spec/controllers/settings/applications_controller_spec.rb
@@ -166,7 +166,11 @@ describe Settings::ApplicationsController do
   end
 
   describe 'destroy' do
+    let(:redis_pipeline_stub) { instance_double(Redis::Namespace, publish: nil) }
+    let!(:access_token) { Fabricate(:accessible_access_token, application: app) }
+
     before do
+      allow(redis).to receive(:pipelined).and_yield(redis_pipeline_stub)
       post :destroy, params: { id: app.id }
     end
 
@@ -177,6 +181,10 @@ describe Settings::ApplicationsController do
     it 'removes the app' do
       expect(Doorkeeper::Application.find_by(id: app.id)).to be_nil
     end
+
+    it 'sends a session kill payload to the streaming server' do
+      expect(redis_pipeline_stub).to have_received(:publish).with("timeline:access_token:#{access_token.id}", '{"event":"kill"}')
+    end
   end
 
   describe 'regenerate' do

From 4fb4721072e552dc2fa7541f7bdeb1737a6c113e Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 4 Jul 2024 16:26:49 +0200
Subject: [PATCH 115/117] Merge pull request from GHSA-58x8-3qxw-6hm7

* Fix insufficient permission checking for public timeline endpoints

Note that this changes unauthenticated access failure code from 401 to 422

* Add more tests for public timelines

* Require user token in `/api/v1/statuses/:id/translate` and `/api/v1/scheduled_statuses`
---
 .../api/v1/scheduled_statuses_controller.rb   |  1 +
 .../v1/statuses/translations_controller.rb    |  1 +
 .../api/v1/timelines/public_controller.rb     |  1 +
 .../api/v1/timelines/tag_controller.rb        |  3 ++-
 .../v1/scheduled_statuses_controller_spec.rb  | 11 ++++++++
 .../statuses/translations_controller_spec.rb  | 20 ++++++++++++++
 .../api/v1/timelines/tag_controller_spec.rb   | 19 +++++++++++---
 spec/requests/api/v1/timelines/public_spec.rb | 26 ++++++++++++++-----
 8 files changed, 71 insertions(+), 11 deletions(-)

diff --git a/app/controllers/api/v1/scheduled_statuses_controller.rb b/app/controllers/api/v1/scheduled_statuses_controller.rb
index 2220b6d22..b33b534eb 100644
--- a/app/controllers/api/v1/scheduled_statuses_controller.rb
+++ b/app/controllers/api/v1/scheduled_statuses_controller.rb
@@ -6,6 +6,7 @@ class Api::V1::ScheduledStatusesController < Api::BaseController
   before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, except: [:update, :destroy]
   before_action -> { doorkeeper_authorize! :write, :'write:statuses' }, only: [:update, :destroy]
 
+  before_action :require_user!
   before_action :set_statuses, only: :index
   before_action :set_status, except: :index
 
diff --git a/app/controllers/api/v1/statuses/translations_controller.rb b/app/controllers/api/v1/statuses/translations_controller.rb
index ec5ea5b85..5e5ee7d38 100644
--- a/app/controllers/api/v1/statuses/translations_controller.rb
+++ b/app/controllers/api/v1/statuses/translations_controller.rb
@@ -4,6 +4,7 @@ class Api::V1::Statuses::TranslationsController < Api::BaseController
   include Authorization
 
   before_action -> { doorkeeper_authorize! :read, :'read:statuses' }
+  before_action :require_user!
   before_action :set_status
   before_action :set_translation
 
diff --git a/app/controllers/api/v1/timelines/public_controller.rb b/app/controllers/api/v1/timelines/public_controller.rb
index 5bbd92b9e..0ff2c5aee 100644
--- a/app/controllers/api/v1/timelines/public_controller.rb
+++ b/app/controllers/api/v1/timelines/public_controller.rb
@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 class Api::V1::Timelines::PublicController < Api::BaseController
+  before_action -> { authorize_if_got_token! :read, :'read:statuses' }
   before_action :require_user!, only: [:show], if: :require_auth?
   after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
 
diff --git a/app/controllers/api/v1/timelines/tag_controller.rb b/app/controllers/api/v1/timelines/tag_controller.rb
index a79d65c12..ffea89c25 100644
--- a/app/controllers/api/v1/timelines/tag_controller.rb
+++ b/app/controllers/api/v1/timelines/tag_controller.rb
@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
 class Api::V1::Timelines::TagController < Api::BaseController
-  before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: :show, if: :require_auth?
+  before_action -> { authorize_if_got_token! :read, :'read:statuses' }
+  before_action :require_user!, if: :require_auth?
   before_action :load_tag
   after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
 
diff --git a/spec/controllers/api/v1/scheduled_statuses_controller_spec.rb b/spec/controllers/api/v1/scheduled_statuses_controller_spec.rb
index 256c4b272..cc3b65f37 100644
--- a/spec/controllers/api/v1/scheduled_statuses_controller_spec.rb
+++ b/spec/controllers/api/v1/scheduled_statuses_controller_spec.rb
@@ -13,6 +13,17 @@ describe Api::V1::ScheduledStatusesController do
     allow(controller).to receive(:doorkeeper_token) { token }
   end
 
+  context 'with an application token' do
+    let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: 'read:statuses') }
+
+    it 'returns http unprocessable entity' do
+      get :index
+
+      expect(response)
+        .to have_http_status(422)
+    end
+  end
+
   describe 'GET #index' do
     it 'returns http success' do
       get :index
diff --git a/spec/controllers/api/v1/statuses/translations_controller_spec.rb b/spec/controllers/api/v1/statuses/translations_controller_spec.rb
index 6257494ae..da152843b 100644
--- a/spec/controllers/api/v1/statuses/translations_controller_spec.rb
+++ b/spec/controllers/api/v1/statuses/translations_controller_spec.rb
@@ -9,6 +9,26 @@ describe Api::V1::Statuses::TranslationsController do
   let(:app)   { Fabricate(:application, name: 'Test app', website: 'http://testapp.com') }
   let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses', application: app) }
 
+  context 'with an application token' do
+    let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: 'read:statuses', application: app) }
+
+    before do
+      allow(controller).to receive(:doorkeeper_token) { token }
+    end
+
+    describe 'POST /api/v1/statuses/:status_id/translate' do
+      let(:status) { Fabricate(:status, account: user.account, text: 'Hola', language: 'es') }
+
+      before do
+        post :create, params: { status_id: status.id }
+      end
+
+      it 'returns http unprocessable entity' do
+        expect(response).to have_http_status(422)
+      end
+    end
+  end
+
   context 'with an oauth token' do
     before do
       allow(controller).to receive(:doorkeeper_token) { token }
diff --git a/spec/controllers/api/v1/timelines/tag_controller_spec.rb b/spec/controllers/api/v1/timelines/tag_controller_spec.rb
index 1c60798fc..89622a41a 100644
--- a/spec/controllers/api/v1/timelines/tag_controller_spec.rb
+++ b/spec/controllers/api/v1/timelines/tag_controller_spec.rb
@@ -6,7 +6,8 @@ describe Api::V1::Timelines::TagController do
   render_views
 
   let(:user)   { Fabricate(:user) }
-  let(:token)  { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses') }
+  let(:scopes) { 'read:statuses' }
+  let(:token)  { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: scopes) }
 
   before do
     allow(controller).to receive(:doorkeeper_token) { token }
@@ -48,13 +49,23 @@ describe Api::V1::Timelines::TagController do
         Form::AdminSettings.new(timeline_preview: false).save
       end
 
-      context 'when the user is not authenticated' do
+      context 'without an access token' do
         let(:token) { nil }
 
-        it 'returns http unauthorized' do
+        it 'returns http unprocessable entity' do
           subject
 
-          expect(response).to have_http_status(401)
+          expect(response).to have_http_status(422)
+        end
+      end
+
+      context 'with an application access token, not bound to a user' do
+        let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: scopes) }
+
+        it 'returns http unprocessable entity' do
+          subject
+
+          expect(response).to have_http_status(422)
         end
       end
 
diff --git a/spec/requests/api/v1/timelines/public_spec.rb b/spec/requests/api/v1/timelines/public_spec.rb
index c43626240..03bde3da8 100644
--- a/spec/requests/api/v1/timelines/public_spec.rb
+++ b/spec/requests/api/v1/timelines/public_spec.rb
@@ -32,6 +32,8 @@ describe 'Public' do
     context 'when the instance allows public preview' do
       let(:expected_statuses) { [local_status, remote_status, media_status] }
 
+      it_behaves_like 'forbidden for wrong scope', 'profile'
+
       context 'with an authorized user' do
         it_behaves_like 'a successful request to the public timeline'
       end
@@ -96,13 +98,9 @@ describe 'Public' do
         Form::AdminSettings.new(timeline_preview: false).save
       end
 
-      context 'with an authenticated user' do
-        let(:expected_statuses) { [local_status, remote_status, media_status] }
+      it_behaves_like 'forbidden for wrong scope', 'profile'
 
-        it_behaves_like 'a successful request to the public timeline'
-      end
-
-      context 'with an unauthenticated user' do
+      context 'without an authentication token' do
         let(:headers) { {} }
 
         it 'returns http unprocessable entity' do
@@ -111,6 +109,22 @@ describe 'Public' do
           expect(response).to have_http_status(422)
         end
       end
+
+      context 'with an application access token, not bound to a user' do
+        let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: scopes) }
+
+        it 'returns http unprocessable entity' do
+          subject
+
+          expect(response).to have_http_status(422)
+        end
+      end
+
+      context 'with an authenticated user' do
+        let(:expected_statuses) { [local_status, remote_status, media_status] }
+
+        it_behaves_like 'a successful request to the public timeline'
+      end
     end
   end
 end

From d4bf22b632ea8b1174375c4966a6768ab66393b6 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 4 Jul 2024 16:45:52 +0200
Subject: [PATCH 116/117] Merge pull request from GHSA-xjvf-fm67-4qc3

---
 app/lib/activitypub/activity/create.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index fedfa39de..1581555bd 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -104,7 +104,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def find_existing_status
     status   = status_from_uri(object_uri)
     status ||= Status.find_by(uri: @object['atomUri']) if @object['atomUri'].present?
-    status
+    status if status&.account_id == @account.id
   end
 
   def process_status_params

From a5b4a2b7e71aedd2f04bc7a90f79dfe234fa7f89 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 4 Jul 2024 16:46:35 +0200
Subject: [PATCH 117/117] Bump version to v4.2.10 (#30910)

---
 CHANGELOG.md            | 31 +++++++++++++++++++++++++++++++
 docker-compose.yml      |  6 +++---
 lib/mastodon/version.rb |  2 +-
 3 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8e579e148..e57d5f23b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,37 @@
 
 All notable changes to this project will be documented in this file.
 
+## [4.2.10] - 2024-07-04
+
+### Security
+
+- Fix incorrect permission checking on multiple API endpoints ([GHSA-58x8-3qxw-6hm7](https://github.com/mastodon/mastodon/security/advisories/GHSA-58x8-3qxw-6hm7))
+- Fix incorrect authorship checking when processing some activities (CVE-2024-37903, [GHSA-xjvf-fm67-4qc3](https://github.com/mastodon/mastodon/security/advisories/GHSA-xjvf-fm67-4qc3))
+- Fix ongoing streaming sessions not being invalidated when application tokens get revoked ([GHSA-vp5r-5pgw-jwqx](https://github.com/mastodon/mastodon/security/advisories/GHSA-vp5r-5pgw-jwqx))
+- Update dependencies
+
+### Added
+
+- Add yarn version specification to avoid confusion with Yarn 3 and Yarn 4
+
+### Changed
+
+- Change preview cards generation to skip unusually long URLs ([oneiros](https://github.com/mastodon/mastodon/pull/30854))
+- Change search modifiers to be case-insensitive ([Gargron](https://github.com/mastodon/mastodon/pull/30865))
+- Change `STATSD_ADDR` handling to emit a warning rather than crashing if the address is unreachable ([timothyjrogers](https://github.com/mastodon/mastodon/pull/30691))
+- Change PWA start URL from `/home` to `/` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/27377))
+
+### Removed
+
+- Removed dependency on `posix-spawn` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/18559))
+
+### Fixed
+
+- Fix scheduled statuses scheduled in less than 5 minutes being immediately published ([danielmbrasil](https://github.com/mastodon/mastodon/pull/30584))
+- Fix encoding detection for link cards ([oneiros](https://github.com/mastodon/mastodon/pull/30780))
+- Fix `/admin/accounts/:account_id/statuses/:id` for edited posts with media attachments ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30819))
+- Fix duplicate `@context` attribute in user archive export ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30653))
+
 ## [4.2.9] - 2024-05-30
 
 ### Security
diff --git a/docker-compose.yml b/docker-compose.yml
index 10a57e31e..d61aa0558 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -56,7 +56,7 @@ services:
 
   web:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.9
+    image: ghcr.io/mastodon/mastodon:v4.2.10
     restart: always
     env_file: .env.production
     command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@@ -77,7 +77,7 @@ services:
 
   streaming:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.9
+    image: ghcr.io/mastodon/mastodon:v4.2.10
     restart: always
     env_file: .env.production
     command: node ./streaming
@@ -95,7 +95,7 @@ services:
 
   sidekiq:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.2.9
+    image: ghcr.io/mastodon/mastodon:v4.2.10
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq
diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb
index f9088382f..77e85a165 100644
--- a/lib/mastodon/version.rb
+++ b/lib/mastodon/version.rb
@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      9
+      10
     end
 
     def default_prerelease