Merge tag 'v4.1.17'

* Add hardening monkey-patch to prevent IP spoofing on misconfigured installations

* Remove rack-attack safelist

.github/workflows/test-ruby.yml

@@ -0,0 +1,153 @@
+name: Ruby Testing
+
+on:
+  push:
+    branches-ignore:
+      - 'dependabot/**'
+      - 'renovate/**'
+  pull_request:
+
+env:
+  BUNDLE_CLEAN: true
+  BUNDLE_FROZEN: true
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: true
+      matrix:
+        mode:
+          - production
+          - test
+    env:
+      RAILS_ENV: ${{ matrix.mode }}
+      BUNDLE_WITH: ${{ matrix.mode }}
+      OTP_SECRET: precompile_placeholder
+      SECRET_KEY_BASE: precompile_placeholder
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          cache: yarn
+          node-version-file: '.nvmrc'
+
+      - name: Install native Ruby dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libicu-dev libidn11-dev
+
+      - name: Set up bundler cache
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: .ruby-version
+          bundler-cache: true
+
+      - run: yarn --frozen-lockfile --production
+      - name: Precompile assets
+        # Previously had set this, but it's not supported
+        # export NODE_OPTIONS=--openssl-legacy-provider
+        run: |-
+          ./bin/rails assets:precompile
+
+      - uses: actions/upload-artifact@v3
+        if: matrix.mode == 'test'
+        with:
+          path: |-
+            ./public/assets
+            ./public/packs-test
+          name: ${{ github.sha }}
+          retention-days: 0
+
+  test:
+    runs-on: ubuntu-latest
+
+    needs:
+      - build
+
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 5432:5432
+
+      redis:
+        image: redis:7-alpine
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          - 6379:6379
+
+    env:
+      DB_HOST: localhost
+      DB_USER: postgres
+      DB_PASS: postgres
+      DISABLE_SIMPLECOV: true
+      RAILS_ENV: test
+      ALLOW_NOPAM: true
+      PAM_ENABLED: true
+      PAM_DEFAULT_SERVICE: pam_test
+      PAM_CONTROLLED_SERVICE: pam_test_controlled
+      OIDC_ENABLED: true
+      OIDC_SCOPE: read
+      SAML_ENABLED: true
+      CAS_ENABLED: true
+      BUNDLE_WITH: 'pam_authentication test'
+      CI_JOBS: ${{ matrix.ci_job }}/4
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version:
+          - '.ruby-version'
+        ci_job:
+          - 1
+          - 2
+          - 3
+          - 4
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v3
+        with:
+          path: './public'
+          name: ${{ github.sha }}
+
+      - name: Update package index
+        run: sudo apt-get update
+
+      - name: Install native Ruby dependencies
+        run: sudo apt-get install -y libicu-dev libidn11-dev
+
+      - name: Install additional system dependencies
+        run: sudo apt-get install -y ffmpeg imagemagick libpam-dev
+
+      - name: Set up bundler cache
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version}}
+          bundler-cache: true
+
+      - name: Load database schema
+        run: './bin/rails db:create db:schema:load db:seed'
+
+      - run: bin/rspec

CHANGELOG.md

@@ -3,6 +3,59 @@ Changelog
 
 All notable changes to this project will be documented in this file.
 
+## [4.1.17] - 2024-05-30
+
+### Security
+
+- Update dependencies
+- Fix private mention filtering ([GHSA-5fq7-3p3j-9vrf](https://github.com/mastodon/mastodon/security/advisories/GHSA-5fq7-3p3j-9vrf))
+- Fix password change endpoint not being rate-limited ([GHSA-q3rg-xx5v-4mxh](https://github.com/mastodon/mastodon/security/advisories/GHSA-q3rg-xx5v-4mxh))
+- Add hardening around rate-limit bypass ([GHSA-c2r5-cfqr-c553](https://github.com/mastodon/mastodon/security/advisories/GHSA-c2r5-cfqr-c553))
+
+### Added
+
+- Add fallback redirection when getting a webfinger query `WEB_DOMAIN@WEB_DOMAIN` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/28592))
+- Add `digest` attribute to `Admin::DomainBlock` entity in REST API ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/29092))
+
+### Removed
+
+- Remove superfluous application-level caching in some controllers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29862))
+
+### Fixed
+
+- Fix leaking Elasticsearch connections in Sidekiq processes ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30450))
+- Fix language of remote posts not being recognized when using unusual casing ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30403))
+- Fix off-by-one in `tootctl media` commands ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30306))
+- Fix removal of allowed domains (in `LIMITED_FEDERATION_MODE`) not being recorded in the audit log ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/30125))
+- Fix not being able to block a subdomain of an already-blocked domain through the API ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30119))
+- Fix `Idempotency-Key` being ignored when scheduling a post ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/30084))
+- Fix crash when supplying the `FFMPEG_BINARY` environment variable ([timothyjrogers](https://github.com/mastodon/mastodon/pull/30022))
+- Fix improper email address validation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29838))
+- Fix results/query in `api/v1/featured_tags/suggestions` ([mjankowski](https://github.com/mastodon/mastodon/pull/29597))
+- Fix unblocking internationalized domain names under certain conditions ([tribela](https://github.com/mastodon/mastodon/pull/29530))
+- Fix admin account created by `mastodon:setup` not being auto-approved ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29379))
+- Fix reference to non-existent var in CLI maintenance command ([mjankowski](https://github.com/mastodon/mastodon/pull/28363))
+
+## [4.1.16] - 2024-02-23
+
+### Added
+
+- Add hourly task to automatically require approval for new registrations in the absence of moderators ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29318), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/29355))
+  In order to prevent future abandoned Mastodon servers from being used for spam, harassment and other malicious activity, Mastodon will now automatically switch new user registrations to require moderator approval whenever they are left open and no activity (including non-moderation actions from apps) from any logged-in user with permission to access moderation reports has been detected in a full week.
+  When this happens, users with the permission to change server settings will receive an email notification.
+  This feature is disabled when `EMAIL_DOMAIN_ALLOWLIST` is used, and can also be disabled with `DISABLE_AUTOMATIC_SWITCHING_TO_APPROVED_REGISTRATIONS=true`.
+
+### Changed
+
+- Change registrations to be closed by default on new installations ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29280))
+  If you are running a server and never changed your registrations mode from the default, updating will automatically close your registrations.
+  Simply re-enable them through the administration interface or using `tootctl settings registrations open` if you want to enable them again.
+
+### Fixed
+
+- Fix processing of remote ActivityPub actors making use of `Link` objects as `Image` `url` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29335))
+- Fix link verifications when page size exceeds 1MB ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29358))
+
 ## [4.1.15] - 2024-02-16
 
 ### Fixed

Gemfile

@@ -158,3 +158,4 @@ gem 'concurrent-ruby', require: false
 gem 'connection_pool', require: false
 gem 'xorcist', '~> 1.1'
 gem 'cocoon', '~> 1.2'
+gem 'mail', '~> 2.8'

Gemfile.lock

@@ -10,40 +10,40 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.1.7.6)
+    actioncable (6.1.7.7)
-      actionpack (= 6.1.7.6)
+      actionpack (= 6.1.7.7)
-      activesupport (= 6.1.7.6)
+      activesupport (= 6.1.7.7)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.1.7.6)
+    actionmailbox (6.1.7.7)
-      actionpack (= 6.1.7.6)
+      actionpack (= 6.1.7.7)
-      activejob (= 6.1.7.6)
+      activejob (= 6.1.7.7)
-      activerecord (= 6.1.7.6)
+      activerecord (= 6.1.7.7)
-      activestorage (= 6.1.7.6)
+      activestorage (= 6.1.7.7)
-      activesupport (= 6.1.7.6)
+      activesupport (= 6.1.7.7)
       mail (>= 2.7.1)
-    actionmailer (6.1.7.6)
+    actionmailer (6.1.7.7)
-      actionpack (= 6.1.7.6)
+      actionpack (= 6.1.7.7)
-      actionview (= 6.1.7.6)
+      actionview (= 6.1.7.7)
-      activejob (= 6.1.7.6)
+      activejob (= 6.1.7.7)
-      activesupport (= 6.1.7.6)
+      activesupport (= 6.1.7.7)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.1.7.6)
+    actionpack (6.1.7.7)
-      actionview (= 6.1.7.6)
+      actionview (= 6.1.7.7)
-      activesupport (= 6.1.7.6)
+      activesupport (= 6.1.7.7)
       rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.1.7.6)
+    actiontext (6.1.7.7)
-      actionpack (= 6.1.7.6)
+      actionpack (= 6.1.7.7)
-      activerecord (= 6.1.7.6)
+      activerecord (= 6.1.7.7)
-      activestorage (= 6.1.7.6)
+      activestorage (= 6.1.7.7)
-      activesupport (= 6.1.7.6)
+      activesupport (= 6.1.7.7)
       nokogiri (>= 1.8.5)
-    actionview (6.1.7.6)
+    actionview (6.1.7.7)
-      activesupport (= 6.1.7.6)
+      activesupport (= 6.1.7.7)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
@@ -54,22 +54,22 @@ GEM
       case_transform (>= 0.2)
       jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
     active_record_query_trace (1.8)
-    activejob (6.1.7.6)
+    activejob (6.1.7.7)
-      activesupport (= 6.1.7.6)
+      activesupport (= 6.1.7.7)
       globalid (>= 0.3.6)
-    activemodel (6.1.7.6)
+    activemodel (6.1.7.7)
-      activesupport (= 6.1.7.6)
+      activesupport (= 6.1.7.7)
-    activerecord (6.1.7.6)
+    activerecord (6.1.7.7)
-      activemodel (= 6.1.7.6)
+      activemodel (= 6.1.7.7)
-      activesupport (= 6.1.7.6)
+      activesupport (= 6.1.7.7)
-    activestorage (6.1.7.6)
+    activestorage (6.1.7.7)
-      actionpack (= 6.1.7.6)
+      actionpack (= 6.1.7.7)
-      activejob (= 6.1.7.6)
+      activejob (= 6.1.7.7)
-      activerecord (= 6.1.7.6)
+      activerecord (= 6.1.7.7)
-      activesupport (= 6.1.7.6)
+      activesupport (= 6.1.7.7)
       marcel (~> 1.0)
       mini_mime (>= 1.1.0)
-    activesupport (6.1.7.6)
+    activesupport (6.1.7.7)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -173,7 +173,7 @@ GEM
     cocoon (1.2.15)
     coderay (1.1.3)
     color_diff (0.1)
-    concurrent-ruby (1.2.2)
+    concurrent-ruby (1.2.3)
     connection_pool (2.3.0)
     cose (1.2.1)
       cbor (~> 0.5.9)
@@ -183,7 +183,7 @@ GEM
     crass (1.0.6)
     css_parser (1.12.0)
       addressable
-    date (3.3.3)
+    date (3.3.4)
     debug_inspector (1.0.0)
     devise (4.8.1)
       bcrypt (~> 3.0)
@@ -330,7 +330,7 @@ GEM
     jmespath (1.6.2)
     json (2.6.3)
     json-canonicalization (0.3.0)
-    json-jwt (1.15.3)
+    json-jwt (1.15.3.1)
       activesupport (>= 4.2)
       aes_key_wrap
       bindata
@@ -416,15 +416,15 @@ GEM
     net-ldap (0.17.1)
     net-pop (0.1.2)
       net-protocol
-    net-protocol (0.2.1)
+    net-protocol (0.2.2)
       timeout
     net-scp (4.0.0.rc1)
       net-ssh (>= 2.6.5, < 8.0.0)
-    net-smtp (0.3.3)
+    net-smtp (0.3.4)
       net-protocol
     net-ssh (7.0.1)
     nio4r (2.5.9)
-    nokogiri (1.16.2)
+    nokogiri (1.16.5)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     nsa (0.2.8)
@@ -491,13 +491,13 @@ GEM
     pry-rails (0.3.9)
       pry (>= 0.10.4)
     public_suffix (5.0.1)
-    puma (5.6.7)
+    puma (5.6.8)
       nio4r (~> 2.0)
     pundit (2.3.0)
       activesupport (>= 3.0.0)
     raabro (1.4.0)
     racc (1.7.3)
-    rack (2.2.8)
+    rack (2.2.8.1)
     rack-attack (6.6.1)
       rack (>= 1.0, < 3)
     rack-cors (1.1.1)
@@ -512,20 +512,20 @@ GEM
       rack
     rack-test (2.0.2)
       rack (>= 1.3)
-    rails (6.1.7.6)
+    rails (6.1.7.7)
-      actioncable (= 6.1.7.6)
+      actioncable (= 6.1.7.7)
-      actionmailbox (= 6.1.7.6)
+      actionmailbox (= 6.1.7.7)
-      actionmailer (= 6.1.7.6)
+      actionmailer (= 6.1.7.7)
-      actionpack (= 6.1.7.6)
+      actionpack (= 6.1.7.7)
-      actiontext (= 6.1.7.6)
+      actiontext (= 6.1.7.7)
-      actionview (= 6.1.7.6)
+      actionview (= 6.1.7.7)
-      activejob (= 6.1.7.6)
+      activejob (= 6.1.7.7)
-      activemodel (= 6.1.7.6)
+      activemodel (= 6.1.7.7)
-      activerecord (= 6.1.7.6)
+      activerecord (= 6.1.7.7)
-      activestorage (= 6.1.7.6)
+      activestorage (= 6.1.7.7)
-      activesupport (= 6.1.7.6)
+      activesupport (= 6.1.7.7)
       bundler (>= 1.15.0)
-      railties (= 6.1.7.6)
+      railties (= 6.1.7.7)
       sprockets-rails (>= 2.0.0)
     rails-controller-testing (1.0.5)
       actionpack (>= 5.0.1.rc1)
@@ -541,9 +541,9 @@ GEM
       railties (>= 6.0.0, < 7)
     rails-settings-cached (0.6.6)
       rails (>= 4.2.0)
-    railties (6.1.7.6)
+    railties (6.1.7.7)
-      actionpack (= 6.1.7.6)
+      actionpack (= 6.1.7.7)
-      activesupport (= 6.1.7.6)
+      activesupport (= 6.1.7.7)
       method_source
       rake (>= 12.2)
       thor (~> 1.0)
@@ -565,7 +565,8 @@ GEM
     responders (3.0.1)
       actionpack (>= 5.0)
       railties (>= 5.0)
-    rexml (3.2.5)
+    rexml (3.2.8)
+      strscan (>= 3.0.9)
     rotp (6.2.0)
     rpam2 (4.0.2)
     rqrcode (2.1.2)
@@ -679,6 +680,7 @@ GEM
       redlock (~> 1.0)
     strong_migrations (0.7.9)
       activerecord (>= 5)
+    strscan (3.1.0)
     swd (1.3.0)
       activesupport (>= 3)
       attr_required (>= 0.0.5)
@@ -753,7 +755,7 @@ GEM
     xorcist (1.1.3)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.6.12)
+    zeitwerk (2.6.13)
 
 PLATFORMS
   ruby
@@ -816,6 +818,7 @@ DEPENDENCIES
   letter_opener_web (~> 2.0)
   link_header (~> 0.0)
   lograge (~> 0.12)
+  mail (~> 2.8)
   makara (~> 0.5)
   mario-redis-lock (~> 1.2)
   memory_profiler

app/controllers/admin/domain_allows_controller.rb

@@ -25,6 +25,8 @@ class Admin::DomainAllowsController < Admin::BaseController
   def destroy
     authorize @domain_allow, :destroy?
     UnallowDomainService.new.call(@domain_allow)
+    log_action :destroy, @domain_allow
+
     redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.destroyed_msg')
   end
 

app/controllers/api/v1/admin/domain_blocks_controller.rb

@@ -19,10 +19,11 @@ class Api::V1::Admin::DomainBlocksController < Api::BaseController
   def create
     authorize :domain_block, :create?
 
+    @domain_block = DomainBlock.new(resource_params)
     existing_domain_block = resource_params[:domain].present? ? DomainBlock.rule_for(resource_params[:domain]) : nil
-    return render json: existing_domain_block, serializer: REST::Admin::ExistingDomainBlockErrorSerializer, status: 422 if existing_domain_block.present?
+    return render json: existing_domain_block, serializer: REST::Admin::ExistingDomainBlockErrorSerializer, status: 422 if conflicts_with_existing_block?(@domain_block, existing_domain_block)
 
-    @domain_block = DomainBlock.create!(resource_params)
+    @domain_block.save!
     DomainBlockWorker.perform_async(@domain_block.id)
     log_action :create, @domain_block
     render json: @domain_block, serializer: REST::Admin::DomainBlockSerializer
@@ -55,6 +56,10 @@ class Api::V1::Admin::DomainBlocksController < Api::BaseController
 
   private
 
+  def conflicts_with_existing_block?(domain_block, existing_domain_block)
+    existing_domain_block.present? && (existing_domain_block.domain == TagManager.instance.normalize_domain(domain_block.domain) || !domain_block.stricter_than?(existing_domain_block))
+  end
+
   def set_domain_blocks
     @domain_blocks = filtered_domain_blocks.order(id: :desc).to_a_paginated_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id))
   end

app/controllers/api/v1/featured_tags/suggestions_controller.rb

@@ -12,6 +12,10 @@ class Api::V1::FeaturedTags::SuggestionsController < Api::BaseController
   private
 
   def set_recently_used_tags
-    @recently_used_tags = Tag.recently_used(current_account).where.not(id: current_account.featured_tags).limit(10)
+    @recently_used_tags = Tag.recently_used(current_account).where.not(id: featured_tag_ids).limit(10)
+  end
+
+  def featured_tag_ids
+    current_account.featured_tags.pluck(:tag_id)
   end
 end

app/controllers/concerns/cache_concern.rb

@@ -28,29 +28,19 @@ module CacheConcern
     response.headers['Vary'] = public_fetch_mode? ? 'Accept' : 'Accept, Signature'
   end
 
+  # TODO: Rename this method, as it does not perform any caching anymore.
   def cache_collection(raw, klass)
-    return raw unless klass.respond_to?(:with_includes)
+    return raw unless klass.respond_to?(:preload_cacheable_associations)
 
-    raw = raw.cache_ids.to_a if raw.is_a?(ActiveRecord::Relation)
+    records = raw.to_a
-    return [] if raw.empty?
 
-    cached_keys_with_value = Rails.cache.read_multi(*raw).transform_keys(&:id)
+    klass.preload_cacheable_associations(records)
-    uncached_ids           = raw.map(&:id) - cached_keys_with_value.keys
 
-    klass.reload_stale_associations!(cached_keys_with_value.values) if klass.respond_to?(:reload_stale_associations!)
+    records
-
-    unless uncached_ids.empty?
-      uncached = klass.where(id: uncached_ids).with_includes.index_by(&:id)
-
-      uncached.each_value do |item|
-        Rails.cache.write(item, item)
-      end
-    end
-
-    raw.filter_map { |item| cached_keys_with_value[item.id] || uncached[item.id] }
   end
 
+  # TODO: Rename this method, as it does not perform any caching anymore.
   def cache_collection_paginated_by_id(raw, klass, limit, options)
-    cache_collection raw.cache_ids.to_a_paginated_by_id(limit, options), klass
+    cache_collection raw.to_a_paginated_by_id(limit, options), klass
   end
 end

app/controllers/well_known/webfinger_controller.rb

@@ -20,7 +20,7 @@ module WellKnown
     def set_account
       username = username_from_resource
       @account = begin
-        if username == Rails.configuration.x.local_domain
+        if username == Rails.configuration.x.local_domain || username == Rails.configuration.x.web_domain
           Account.representative
         else
           Account.find_local!(username)

app/lib/activitypub/parser/status_parser.rb

@@ -3,6 +3,8 @@
 class ActivityPub::Parser::StatusParser
   include JsonLdHelper
 
+  NORMALIZED_LOCALE_NAMES = LanguagesHelper::SUPPORTED_LOCALES.keys.index_by(&:downcase).freeze
+
   # @param [Hash] json
   # @param [Hash] magic_values
   # @option magic_values [String] :followers_collection
@@ -86,6 +88,13 @@ class ActivityPub::Parser::StatusParser
   end
 
   def language
+    lang = raw_language_code
+    lang.presence && NORMALIZED_LOCALE_NAMES.fetch(lang.downcase.to_sym, lang)
+  end
+
+  private
+
+  def raw_language_code
     if content_language_map?
       @object['contentMap'].keys.first
     elsif name_language_map?
@@ -95,8 +104,6 @@ class ActivityPub::Parser::StatusParser
     end
   end
 
-  private
-
   def audience_to
     as_array(@object['to'] || @json['to']).map { |x| value_or_id(x) }
   end

app/lib/video_metadata_extractor.rb

@@ -22,7 +22,7 @@ class VideoMetadataExtractor
   private
 
   def ffmpeg_command_output
-    command = Terrapin::CommandLine.new('ffprobe', '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel')
+    command = Terrapin::CommandLine.new(Rails.configuration.x.ffprobe_binary, '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel')
     command.run(path: @path, format: 'json', loglevel: 'fatal')
   end
 

app/mailers/admin_mailer.rb

@@ -47,4 +47,13 @@ class AdminMailer < ApplicationMailer
       mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trends.subject', instance: @instance)
     end
   end
+
+  def auto_close_registrations(recipient)
+    @me       = recipient
+    @instance = Rails.configuration.x.local_domain
+
+    locale_for_account(@me) do
+      mail to: @me.user_email, subject: I18n.t('admin_mailer.auto_close_registrations.subject', instance: @instance)
+    end
+  end
 end

app/models/concerns/account_interactions.rb

@@ -187,7 +187,7 @@ module AccountInteractions
   end
 
   def unblock_domain!(other_domain)
-    block = domain_blocks.find_by(domain: other_domain)
+    block = domain_blocks.find_by(domain: normalized_domain(other_domain))
     block&.destroy
   end
 
@@ -299,4 +299,8 @@ module AccountInteractions
   def remove_potential_friendship(other_account)
     PotentialFriendshipTracker.remove(id, other_account.id)
   end
+
+  def normalized_domain(domain)
+    TagManager.instance.normalize_domain(domain)
+  end
 end

app/models/concerns/cacheable.rb

@@ -14,6 +14,10 @@ module Cacheable
       includes(@cache_associated)
     end
 
+    def preload_cacheable_associations(records)
+      ActiveRecord::Associations::Preloader.new.preload(records, @cache_associated)
+    end
+
     def cache_ids
       select(:id, :updated_at)
     end

app/models/feed.rb

@@ -28,7 +28,7 @@ class Feed
       unhydrated = redis.zrangebyscore(key, "(#{min_id}", "(#{max_id}", limit: [0, limit], with_scores: true).map(&:first).map(&:to_i)
     end
 
-    Status.where(id: unhydrated).cache_ids
+    Status.where(id: unhydrated)
   end
 
   def key

app/models/public_feed.rb

@@ -29,7 +29,7 @@ class PublicFeed
     scope.merge!(media_only_scope) if media_only?
     scope.merge!(language_scope) if account&.chosen_languages.present?
 
-    scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
+    scope.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
   end
 
   private

app/models/status.rb

@@ -344,38 +344,6 @@ class Status < ApplicationRecord
       StatusPin.select('status_id').where(status_id: status_ids).where(account_id: account_id).each_with_object({}) { |p, h| h[p.status_id] = true }
     end
 
-    def reload_stale_associations!(cached_items)
-      account_ids = []
-
-      cached_items.each do |item|
-        account_ids << item.account_id
-        account_ids << item.reblog.account_id if item.reblog?
-      end
-
-      account_ids.uniq!
-
-      status_ids = cached_items.map { |item| item.reblog? ? item.reblog_of_id : item.id }.uniq
-
-      return if account_ids.empty?
-
-      accounts = Account.where(id: account_ids).includes(:account_stat, :user).index_by(&:id)
-
-      status_stats = StatusStat.where(status_id: status_ids).index_by(&:status_id)
-
-      cached_items.each do |item|
-        item.account = accounts[item.account_id]
-        item.reblog.account = accounts[item.reblog.account_id] if item.reblog?
-
-        if item.reblog?
-          status_stat = status_stats[item.reblog.id]
-          item.reblog.status_stat = status_stat if status_stat.present?
-        else
-          status_stat = status_stats[item.id]
-          item.status_stat = status_stat if status_stat.present?
-        end
-      end
-    end
-
     def from_text(text)
       return [] if text.blank?
 

app/models/tag_feed.rb

@@ -33,7 +33,7 @@ class TagFeed < PublicFeed
     scope.merge!(account_filters_scope) if account?
     scope.merge!(media_only_scope) if media_only?
 
-    scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
+    scope.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
   end
 
   private

app/models/user.rb

@@ -94,6 +94,9 @@ class User < ApplicationRecord
   validates :invite_request, presence: true, on: :create, if: :invite_text_required?
 
   validates :locale, inclusion: I18n.available_locales.map(&:to_s), if: :locale?
+
+  validates :email, presence: true, email_address: true
+
   validates_with BlacklistedEmailValidator, if: -> { ENV['EMAIL_DOMAIN_LISTS_APPLY_AFTER_CONFIRMATION'] == 'true' || !confirmed? }
   validates_with EmailMxValidator, if: :validate_email_dns?
   validates :agreement, acceptance: { allow_nil: false, accept: [true, 'true', '1'] }, on: :create

app/serializers/rest/admin/domain_block_serializer.rb

@@ -1,11 +1,15 @@
 # frozen_string_literal: true
 
 class REST::Admin::DomainBlockSerializer < ActiveModel::Serializer
-  attributes :id, :domain, :created_at, :severity,
+  attributes :id, :domain, :digest, :created_at, :severity,
              :reject_media, :reject_reports,
              :private_comment, :public_comment, :obfuscate
 
   def id
     object.id.to_s
   end
+
+  def digest
+    object.domain_digest
+  end
 end

app/services/activitypub/process_account_service.rb

@@ -199,10 +199,15 @@ class ActivityPub::ProcessAccountService < BaseService
     value = first_of_value(@json[key])
 
     return if value.nil?
-    return value['url'] if value.is_a?(Hash)
 
-    image = fetch_resource_without_id_validation(value)
+    if value.is_a?(String)
-    image['url'] if image
+      value = fetch_resource_without_id_validation(value)
+      return if value.nil?
+    end
+
+    value = first_of_value(value['url']) if value.is_a?(Hash) && value['type'] == 'Image'
+    value = value['href'] if value.is_a?(Hash)
+    value if value.is_a?(String)
   end
 
   def public_key

app/services/notify_service.rb

@@ -62,16 +62,17 @@ class NotifyService < BaseService
           LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
           WHERE s.id = :id
         UNION ALL
-          SELECT s.id, s.in_reply_to_id, m.id, st.path || s.id, st.depth + 1
+          SELECT s.id, s.in_reply_to_id, m.id, ancestors.path || s.id, ancestors.depth + 1
-          FROM ancestors st
+          FROM ancestors
-          JOIN statuses s ON s.id = st.in_reply_to_id
+          JOIN statuses s ON s.id = ancestors.in_reply_to_id
-          LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id
+          /* early exit if we already have a mention matching our requirements */
-          WHERE st.mention_id IS NULL AND NOT s.id = ANY(path) AND st.depth < :depth_limit
+          LEFT JOIN mentions m ON m.silent = FALSE AND m.account_id = :sender_id AND m.status_id = s.id AND s.account_id = :recipient_id
+          WHERE ancestors.mention_id IS NULL AND NOT s.id = ANY(path) AND ancestors.depth < :depth_limit
       )
       SELECT COUNT(*)
-      FROM ancestors st
+      FROM ancestors
-      JOIN statuses s ON s.id = st.id
+      JOIN statuses s ON s.id = ancestors.id
-      WHERE st.mention_id IS NOT NULL AND s.visibility = 3
+      WHERE ancestors.mention_id IS NOT NULL AND s.account_id = :recipient_id AND s.visibility = 3
     SQL
   end
 

app/services/post_status_service.rb

@@ -137,7 +137,7 @@ class PostStatusService < BaseService
 
   def idempotency_duplicate
     if scheduled?
-      @account.schedule_statuses.find(@idempotency_duplicate)
+      @account.scheduled_statuses.find(@idempotency_duplicate)
     else
       @account.statuses.find(@idempotency_duplicate)
     end
@@ -189,7 +189,7 @@ class PostStatusService < BaseService
   end
 
   def scheduled_options
-    @options.tap do |options_hash|
+    @options.dup.tap do |options_hash|
       options_hash[:in_reply_to_id]  = options_hash.delete(:thread)&.id
       options_hash[:application_id]  = options_hash.delete(:application)&.id
       options_hash[:scheduled_at]    = nil

app/services/verify_link_service.rb

@@ -19,7 +19,7 @@ class VerifyLinkService < BaseService
 
   def perform_request!
     @body = Request.new(:get, @url).add_headers('Accept' => 'text/html').perform do |res|
-      res.code == 200 ? res.body_with_limit : nil
+      res.code == 200 ? res.truncated_body : nil
     end
   end
 

app/validators/email_address_validator.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# NOTE: I initially wrote this as `EmailValidator` but it ended up clashing
+# with an indirect dependency of ours, `validate_email`, which, turns out,
+# has the same approach as we do, but with an extra check disallowing
+# single-label domains. Decided to not switch to `validate_email` because
+# we do want to allow at least `localhost`.
+
+class EmailAddressValidator < ActiveModel::EachValidator
+  def validate_each(record, attribute, value)
+    value = value.strip
+
+    address = Mail::Address.new(value)
+    record.errors.add(attribute, :invalid) if address.address != value
+  rescue Mail::Field::FieldError
+    record.errors.add(attribute, :invalid)
+  end
+end

app/views/admin/settings/registrations/show.html.haml

@@ -13,6 +13,8 @@
 
   %p.lead= t('admin.settings.registrations.preamble')
 
+  .flash-message= t('admin.settings.registrations.moderation_recommandation')
+
   .fields-row
     .fields-row__column.fields-row__column-6.fields-group
       = f.input :registrations_mode, collection: %w(open approved none), wrapper: :with_label, include_blank: false, label_method: lambda { |mode| I18n.t("admin.settings.registrations_mode.modes.#{mode}") }

app/views/admin_mailer/auto_close_registrations.text.erb

@@ -0,0 +1,3 @@
+<%= raw t('admin_mailer.auto_close_registrations.body', instance: @instance) %>
+
+<%= raw t('application_mailer.view')%> <%= admin_settings_registrations_url %>

app/workers/scheduler/auto_close_registrations_scheduler.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+class Scheduler::AutoCloseRegistrationsScheduler
+  include Sidekiq::Worker
+  include Redisable
+
+  sidekiq_options retry: 0
+
+  # Automatically switch away from open registrations if no
+  # moderator had any activity in that period of time
+  OPEN_REGISTRATIONS_MODERATOR_THRESHOLD = 1.week + UserTrackingConcern::SIGN_IN_UPDATE_FREQUENCY
+
+  def perform
+    return if Rails.configuration.x.email_domains_whitelist.present? || ENV['DISABLE_AUTOMATIC_SWITCHING_TO_APPROVED_REGISTRATIONS'] == 'true'
+    return unless Setting.registrations_mode == 'open'
+
+    switch_to_approval_mode! unless active_moderators?
+  end
+
+  private
+
+  def active_moderators?
+    User.those_who_can(:manage_reports).exists?(current_sign_in_at: OPEN_REGISTRATIONS_MODERATOR_THRESHOLD.ago...)
+  end
+
+  def switch_to_approval_mode!
+    Setting.registrations_mode = 'approved'
+
+    User.those_who_can(:manage_settings).includes(:account).find_each do |user|
+      AdminMailer.auto_close_registrations(user.account).deliver_later
+    end
+  end
+end

config/application.rb

@@ -44,6 +44,7 @@ require_relative '../lib/chewy/strategy/bypass_with_warning'
 require_relative '../lib/webpacker/manifest_extensions'
 require_relative '../lib/webpacker/helper_extensions'
 require_relative '../lib/rails/engine_extensions'
+require_relative '../lib/action_dispatch/remote_ip_extensions'
 require_relative '../lib/active_record/database_tasks_extensions'
 require_relative '../lib/active_record/batches'
 require_relative '../lib/simple_navigation/item_extensions'

config/initializers/ffmpeg.rb

@@ -1,3 +1,6 @@
-if ENV['FFMPEG_BINARY'].present?
+# frozen_string_literal: true
-    FFMPEG.ffmpeg_binary = ENV['FFMPEG_BINARY']
+
+Rails.application.configure do
+  config.x.ffmpeg_binary = ENV['FFMPEG_BINARY'] || 'ffmpeg'
+  config.x.ffprobe_binary = ENV['FFPROBE_BINARY'] || 'ffprobe'
 end

config/initializers/rack_attack.rb

@@ -37,6 +37,10 @@ class Rack::Attack
       authenticated_token&.id
     end
 
+    def warden_user_id
+      @env['warden']&.user&.id
+    end
+
     def unauthenticated?
       !authenticated_user_id
     end
@@ -58,10 +62,6 @@ class Rack::Attack
     end
   end
 
-  Rack::Attack.safelist('allow from localhost') do |req|
-    req.remote_ip == '127.0.0.1' || req.remote_ip == '::1'
-  end
-
   Rack::Attack.blocklist('deny from blocklist') do |req|
     IpBlock.blocked?(req.remote_ip)
   end
@@ -137,6 +137,10 @@ class Rack::Attack
     req.session[:attempt_user_id] || req.params.dig('user', 'email').presence if req.post? && req.path_matches?('/auth/sign_in')
   end
 
+  throttle('throttle_password_change/account', limit: 10, period: 10.minutes) do |req|
+    req.warden_user_id if req.put? || (req.patch? && req.path_matches?('/auth'))
+  end
+
   self.throttled_responder = lambda do |request|
     now        = Time.now.utc
     match_data = request.env['rack.attack.match_data']

config/locales/en.yml

@@ -745,6 +745,7 @@ en:
         disabled: To no one
         users: To logged-in local users
       registrations:
+        moderation_recommandation: Please make sure you have an adequate and reactive moderation team before you open registrations to everyone!
         preamble: Control who can create an account on your server.
         title: Registrations
       registrations_mode:
@@ -905,6 +906,9 @@ en:
       title: Webhooks
       webhook: Webhook
   admin_mailer:
+    auto_close_registrations:
+      body: Due to a lack of recent moderator activity, registrations on %{instance} have been automatically switched to requiring manual review, to prevent %{instance} from being used as a platform for potential bad actors. You can switch it back to open registrations at any time.
+      subject: Registrations for %{instance} have been automatically switched to requiring approval
     new_appeal:
       actions:
         delete_statuses: to delete their posts

config/settings.yml

@@ -9,7 +9,7 @@ defaults: &defaults
   site_terms: ''
   site_contact_username: ''
   site_contact_email: ''
-  registrations_mode: 'open'
+  registrations_mode: 'none'
   profile_directory: true
   closed_registrations_message: ''
   open_deletion: true

config/sidekiq.yml

@@ -58,3 +58,7 @@
     interval: 1 minute
     class: Scheduler::SuspendedUserCleanupScheduler
     queue: scheduler
+  auto_close_registrations_scheduler:
+    interval: 1 hour
+    class: Scheduler::AutoCloseRegistrationsScheduler
+    queue: scheduler

docker-compose.yml

@@ -56,7 +56,7 @@ services:
 
   web:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.1.15
+    image: ghcr.io/mastodon/mastodon:v4.1.17
     restart: always
     env_file: .env.production
     command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000"
@@ -77,7 +77,7 @@ services:
 
   streaming:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.1.15
+    image: ghcr.io/mastodon/mastodon:v4.1.17
     restart: always
     env_file: .env.production
     command: node ./streaming
@@ -95,7 +95,7 @@ services:
 
   sidekiq:
     build: .
-    image: ghcr.io/mastodon/mastodon:v4.1.15
+    image: ghcr.io/mastodon/mastodon:v4.1.17
     restart: always
     env_file: .env.production
     command: bundle exec sidekiq

lib/action_dispatch/remote_ip_extensions.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# Mastodon is not made to be directly accessed without a reverse proxy.
+# This monkey-patch prevents remote IP address spoofing when being accessed
+# directly.
+#
+# See PR: https://github.com/rails/rails/pull/51610
+
+# In addition to the PR above, it also raises an error if a request with
+# `X-Forwarded-For` or `Client-Ip` comes directly from a client without
+# going through a trusted proxy.
+
+# rubocop:disable all -- This is a mostly vendored file
+
+module ActionDispatch
+  class RemoteIp
+    module GetIpExtensions
+      def calculate_ip
+        # Set by the Rack web server, this is a single value.
+        remote_addr = ips_from(@req.remote_addr).last
+
+        # Could be a CSV list and/or repeated headers that were concatenated.
+        client_ips    = ips_from(@req.client_ip).reverse!
+        forwarded_ips = ips_from(@req.x_forwarded_for).reverse!
+
+        # `Client-Ip` and `X-Forwarded-For` should not, generally, both be set. If they
+        # are both set, it means that either:
+        #
+        # 1) This request passed through two proxies with incompatible IP header
+        #     conventions.
+        #
+        # 2) The client passed one of `Client-Ip` or `X-Forwarded-For`
+        #     (whichever the proxy servers weren't using) themselves.
+        #
+        # Either way, there is no way for us to determine which header is the right one
+        # after the fact. Since we have no idea, if we are concerned about IP spoofing
+        # we need to give up and explode. (If you're not concerned about IP spoofing you
+        # can turn the `ip_spoofing_check` option off.)
+        should_check_ip = @check_ip && client_ips.last && forwarded_ips.last
+        if should_check_ip && !forwarded_ips.include?(client_ips.last)
+          # We don't know which came from the proxy, and which from the user
+          raise IpSpoofAttackError, "IP spoofing attack?! " \
+            "HTTP_CLIENT_IP=#{@req.client_ip.inspect} " \
+            "HTTP_X_FORWARDED_FOR=#{@req.x_forwarded_for.inspect}"
+        end
+
+        # NOTE: Mastodon addition to make sure we don't get requests from a non-trusted client
+        if @check_ip && (forwarded_ips.last || client_ips.last) && !@proxies.any? { |proxy| proxy === remote_addr }
+          raise IpSpoofAttackError, "IP spoofing attack?! client #{remote_addr} is not a trusted proxy " \
+            "HTTP_CLIENT_IP=#{@req.client_ip.inspect} " \
+            "HTTP_X_FORWARDED_FOR=#{@req.x_forwarded_for.inspect}"
+        end
+
+        # We assume these things about the IP headers:
+        #
+        #     - X-Forwarded-For will be a list of IPs, one per proxy, or blank
+        #     - Client-Ip is propagated from the outermost proxy, or is blank
+        #     - REMOTE_ADDR will be the IP that made the request to Rack
+        ips = forwarded_ips + client_ips
+        ips.compact!
+
+        # If every single IP option is in the trusted list, return the IP that's
+        # furthest away
+        filter_proxies([remote_addr] + ips).first || ips.last || remote_addr
+      end
+    end
+  end
+end
+
+ActionDispatch::RemoteIp::GetIp.prepend(ActionDispatch::RemoteIp::GetIpExtensions)
+
+# rubocop:enable all

lib/mastodon/maintenance_cli.rb

@@ -234,7 +234,7 @@ module Mastodon
         users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse
         ref_user = users.shift
         @prompt.warn "Multiple users registered with e-mail address #{ref_user.email}."
-        @prompt.warn "e-mail will be disabled for the following accounts: #{user.map(&:account).map(&:acct).join(', ')}"
+        @prompt.warn "e-mail will be disabled for the following accounts: #{users.map(&:account).map(&:acct).join(', ')}"
         @prompt.warn 'Please reach out to them and set another address with `tootctl account modify` or delete them.'
 
         i = 0

lib/mastodon/media_cli.rb

@@ -143,7 +143,7 @@ module Mastodon
 
             model_name      = path_segments.first.classify
             attachment_name = path_segments[1].singularize
-            record_id       = path_segments[2..-2].join.to_i
+            record_id       = path_segments[2...-2].join.to_i
             file_name       = path_segments.last
             record          = record_map.dig(model_name, record_id)
             attachment      = record&.public_send(attachment_name)
@@ -186,7 +186,7 @@ module Mastodon
           end
 
           model_name      = path_segments.first.classify
-          record_id       = path_segments[2..-2].join.to_i
+          record_id       = path_segments[2...-2].join.to_i
           attachment_name = path_segments[1].singularize
           file_name       = path_segments.last
 
@@ -322,7 +322,7 @@ module Mastodon
       end
 
       model_name = path_segments.first.classify
-      record_id  = path_segments[2..-2].join.to_i
+      record_id  = path_segments[2...-2].join.to_i
 
       unless PRELOAD_MODEL_WHITELIST.include?(model_name)
         say("Cannot find corresponding model: #{model_name}", :red)
@@ -372,7 +372,7 @@ module Mastodon
         next unless [7, 10].include?(segments.size)
 
         model_name = segments.first.classify
-        record_id  = segments[2..-2].join.to_i
+        record_id  = segments[2...-2].join.to_i
 
         next unless PRELOAD_MODEL_WHITELIST.include?(model_name)
 

lib/mastodon/sidekiq_middleware.rb

@@ -8,6 +8,7 @@ class Mastodon::SidekiqMiddleware
   rescue Mastodon::HostValidationError
     # Do not retry
   rescue => e
+    clean_up_elasticsearch_connections!
     limit_backtrace_and_raise(e)
   ensure
     clean_up_sockets!
@@ -25,6 +26,32 @@ class Mastodon::SidekiqMiddleware
     clean_up_statsd_socket!
   end
 
+  # This is a hack to immediately free up unused Elasticsearch connections.
+  #
+  # Indeed, Chewy creates one `Elasticsearch::Client` instance per thread,
+  # and each such client manages its long-lasting connection to
+  # Elasticsearch.
+  #
+  # As far as I know, neither `chewy`,  `elasticsearch-transport` or even
+  # `faraday` provide a reliable way to immediately close a connection, and
+  # rely on the underlying object to be garbage-collected instead.
+  #
+  # Furthermore, `sidekiq` creates a new thread each time a job throws an
+  # exception, meaning that each failure will create a new connection, and
+  # the old one will only be closed on full garbage collection.
+  def clean_up_elasticsearch_connections!
+    return unless Chewy.enabled? && Chewy.current[:chewy_client].present?
+
+    Chewy.client.transport.connections.each do |connection|
+      # NOTE: This bit of code is tailored for the HTTPClient Faraday adapter
+      connection.connection.app.instance_variable_get(:@client)&.reset_all
+    end
+
+    Chewy.current.delete(:chewy_client)
+  rescue
+    nil
+  end
+
   def clean_up_redis_socket!
     RedisConfiguration.pool.checkin if Thread.current[:redis]
     Thread.current[:redis] = nil

lib/mastodon/version.rb

@@ -13,7 +13,7 @@ module Mastodon
     end
 
     def patch
-      15
+      17
     end
 
     def flags

lib/paperclip/image_extractor.rb

@@ -35,7 +35,7 @@ module Paperclip
       dst.binmode
 
       begin
-        command = Terrapin::CommandLine.new('ffmpeg', '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger)
+        command = Terrapin::CommandLine.new(Rails.configuration.x.ffmpeg_binary, '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger)
         command.run(source: @file.path, destination: dst.path, loglevel: 'fatal')
       rescue Terrapin::ExitStatusError
         dst.close(true)

lib/paperclip/transcoder.rb

@@ -51,7 +51,7 @@ module Paperclip
       command_arguments, interpolations = prepare_command(destination)
 
       begin
-        command = Terrapin::CommandLine.new('ffmpeg', command_arguments.join(' '), logger: Paperclip.logger)
+        command = Terrapin::CommandLine.new(Rails.configuration.x.ffmpeg_binary, command_arguments.join(' '), logger: Paperclip.logger)
         command.run(interpolations)
       rescue Terrapin::ExitStatusError => e
         raise Paperclip::Error, "Error while transcoding #{@basename}: #{e}"

lib/tasks/mastodon.rake

@@ -516,6 +516,7 @@ namespace :mastodon do
           owner_role = UserRole.find_by(name: 'Owner')
           user = User.new(email: email, password: password, confirmed_at: Time.now.utc, account_attributes: { username: username }, bypass_invite_request_check: true, role: owner_role)
           user.save(validate: false)
+          user.approve!
 
           Setting.site_contact_username = username
 

spec/config/initializers/rack_attack_spec.rb

@@ -1,8 +1,6 @@
 require 'rails_helper'
 
-describe Rack::Attack do
+describe Rack::Attack, type: :request do
-  include Rack::Test::Methods
-
   def app
     Rails.application
   end
@@ -12,7 +10,7 @@ describe Rack::Attack do
       it 'does not change the request status' do
         limit.times do
           request.call
-          expect(last_response.status).to_not eq(429)
+          expect(response.status).to_not eq(429)
         end
       end
     end
@@ -21,7 +19,7 @@ describe Rack::Attack do
       it 'returns http too many requests' do
         (limit * 2).times do |i|
           request.call
-          expect(last_response.status).to eq(429) if i > limit
+          expect(response.status).to eq(429) if i > limit
         end
       end
     end
@@ -32,7 +30,7 @@ describe Rack::Attack do
   describe 'throttle excessive sign-up requests by IP address' do
     context 'through the website' do
       let(:limit) { 25 }
-      let(:request) { ->() { post path, {}, 'REMOTE_ADDR' => remote_ip } }
+      let(:request) { ->() { post path, headers: { 'REMOTE_ADDR' => remote_ip } } }
 
       context 'for exact path' do
         let(:path)  { '/auth' }
@@ -47,7 +45,7 @@ describe Rack::Attack do
 
     context 'through the API' do
       let(:limit) { 5 }
-      let(:request) { ->() { post path, {}, 'REMOTE_ADDR' => remote_ip } }
+      let(:request) { ->() { post path, headers: { 'REMOTE_ADDR' => remote_ip } } }
 
       context 'for exact path' do
         let(:path)  { '/api/v1/accounts' }
@@ -59,7 +57,7 @@ describe Rack::Attack do
 
         it 'returns http not found' do
           request.call
-          expect(last_response.status).to eq(404)
+          expect(response.status).to eq(404)
         end
       end
     end
@@ -67,7 +65,7 @@ describe Rack::Attack do
 
   describe 'throttle excessive sign-in requests by IP address' do
     let(:limit) { 25 }
-    let(:request) { ->() { post path, {}, 'REMOTE_ADDR' => remote_ip } }
+    let(:request) { ->() { post path, headers: { 'REMOTE_ADDR' => remote_ip } } }
 
     context 'for exact path' do
       let(:path)  { '/auth/sign_in' }
@@ -79,4 +77,28 @@ describe Rack::Attack do
       it_behaves_like 'throttled endpoint'
     end
   end
+
+  describe 'throttle excessive password change requests by account' do
+    let(:user) { Fabricate(:user, email: 'user@host.example') }
+    let(:limit) { 10 }
+    let(:period) { 10.minutes }
+    let(:request) { -> { put path, headers: { 'REMOTE_ADDR' => remote_ip } } }
+    let(:path) { '/auth' }
+
+    before do
+      sign_in user, scope: :user
+
+      # Unfortunately, devise's `sign_in` helper causes the `session` to be
+      # loaded in the next request regardless of whether it's actually accessed
+      # by the client code.
+      #
+      # So, we make an extra query to clear issue a session cookie instead.
+      #
+      # A less resource-intensive way to deal with that would be to generate the
+      # session cookie manually, but this seems pretty involved.
+      get '/'
+    end
+
+    it_behaves_like 'throttled endpoint'
+  end
 end

spec/controllers/api/v1/admin/domain_blocks_controller_spec.rb

@@ -16,6 +16,8 @@ RSpec.describe Api::V1::Admin::DomainBlocksController, type: :controller do
     let(:scopes) { wrong_scope }
 
     it 'returns http forbidden' do
+      subject
+
       expect(response).to have_http_status(403)
     end
   end
@@ -24,6 +26,8 @@ RSpec.describe Api::V1::Admin::DomainBlocksController, type: :controller do
     let(:role) { UserRole.find_by(name: wrong_role) }
 
     it 'returns http forbidden' do
+      subject
+
       expect(response).to have_http_status(403)
     end
   end
@@ -140,39 +144,70 @@ RSpec.describe Api::V1::Admin::DomainBlocksController, type: :controller do
 
   describe 'POST #create' do
     let(:existing_block_domain) { 'example.com' }
+    let(:params) { { domain: 'foo.bar.com', severity: :silence } }
     let!(:block) { Fabricate(:domain_block, domain: existing_block_domain, severity: :suspend) }
 
-    before do
+    subject do
-      post :create, params: { domain: 'foo.bar.com', severity: :silence }
+      post :create, params: params
     end
 
     it_behaves_like 'forbidden for wrong scope', 'write:statuses'
     it_behaves_like 'forbidden for wrong role', ''
     it_behaves_like 'forbidden for wrong role', 'Moderator'
 
-    it 'returns http success' do
+    it 'creates a domain block and returns expected domain name', :aggregate_failures do
+      subject
+
       expect(response).to have_http_status(200)
-    end
+      expect(body_as_json[:domain]).to eq 'foo.bar.com'
-
-    it 'returns expected domain name' do
-      json = body_as_json
-      expect(json[:domain]).to eq 'foo.bar.com'
-    end
-
-    it 'creates a domain block' do
       expect(DomainBlock.find_by(domain: 'foo.bar.com')).to_not be_nil
     end
 
-    context 'when a stricter domain block already exists' do
+    context 'when a looser domain block already exists on a higher level domain' do
-      let(:existing_block_domain) { 'bar.com' }
+      let(:params) { { domain: 'foo.bar.com', severity: :suspend } }
 
-      it 'returns http unprocessable entity' do
+      before do
-        expect(response).to have_http_status(422)
+        Fabricate(:domain_block, domain: 'bar.com', severity: :silence)
       end
 
-      it 'renders existing domain block in error' do
+      it 'creates a domain block with the expected domain name and severity', :aggregate_failures do
-        json = body_as_json
+        subject
-        expect(json[:existing_domain_block][:domain]).to eq existing_block_domain
+
+        body = body_as_json
+
+        expect(response).to have_http_status(200)
+        expect(body).to match a_hash_including(
+          {
+            domain: 'foo.bar.com',
+            severity: 'suspend',
+          }
+        )
+
+        expect(DomainBlock.find_by(domain: 'foo.bar.com')).to be_present
+      end
+    end
+
+    context 'when a domain block already exists on the same domain' do
+      before do
+        Fabricate(:domain_block, domain: 'foo.bar.com', severity: :silence)
+      end
+
+      it 'returns existing domain block in error', :aggregate_failures do
+        subject
+
+        expect(response).to have_http_status(422)
+        expect(body_as_json[:existing_domain_block][:domain]).to eq('foo.bar.com')
+      end
+    end
+
+    context 'when a stricter domain block already exists on a higher level domain' do
+      let(:existing_block_domain) { 'bar.com' }
+
+      it 'returns http unprocessable entity with existing domain block in error', :aggregate_reblogs do
+        subject
+
+        expect(response).to have_http_status(422)
+        expect(body_as_json[:existing_domain_block][:domain]).to eq existing_block_domain
       end
     end
   end

spec/controllers/api/v1/featured_tags/suggestions_controller_spec.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Api::V1::FeaturedTags::SuggestionsController do
+  render_views
+
+  let(:user)    { Fabricate(:user) }
+  let(:token)   { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:accounts') }
+  let(:account) { Fabricate(:account, user: user) }
+
+  before do
+    allow(controller).to receive(:doorkeeper_token) { token }
+  end
+
+  describe 'GET #index' do
+    let!(:unused_featured_tag) { Fabricate(:tag, name: 'unused_featured_tag') }
+    let!(:used_tag) { Fabricate(:tag, name: 'used_tag') }
+    let!(:used_featured_tag) { Fabricate(:tag, name: 'used_featured_tag') }
+
+    before do
+      _unused_tag = Fabricate(:tag, name: 'unused_tag')
+
+      # Make relevant tags used by account
+      status = Fabricate(:status, account: account)
+      status.tags << used_tag
+      status.tags << used_featured_tag
+
+      # Feature the relevant tags
+      Fabricate :featured_tag, account: account, name: unused_featured_tag.name
+      Fabricate :featured_tag, account: account, name: used_featured_tag.name
+    end
+
+    it 'returns http success and recently used but not featured tags', :aggregate_failures do
+      get :index, params: { account_id: account.id, limit: 2 }
+
+      expect(response)
+        .to have_http_status(200)
+      expect(body_as_json)
+        .to contain_exactly(
+          include(name: used_tag.name)
+        )
+    end
+  end
+end

spec/controllers/application_controller_spec.rb

@@ -242,37 +242,4 @@ describe ApplicationController, type: :controller do
 
     include_examples 'respond_with_error', 422
   end
-
-  describe 'cache_collection' do
-    class C < ApplicationController
-      public :cache_collection
-    end
-
-    shared_examples 'receives :with_includes' do |fabricator, klass|
-      it 'uses raw if it is not an ActiveRecord::Relation' do
-        record = Fabricate(fabricator)
-        expect(C.new.cache_collection([record], klass)).to eq [record]
-      end
-    end
-
-    shared_examples 'cacheable' do |fabricator, klass|
-      include_examples 'receives :with_includes', fabricator, klass
-
-      it 'calls cache_ids of raw if it is an ActiveRecord::Relation' do
-        record = Fabricate(fabricator)
-        relation = klass.none
-        allow(relation).to receive(:cache_ids).and_return([record])
-        expect(C.new.cache_collection(relation, klass)).to eq [record]
-      end
-    end
-
-    it 'returns raw unless class responds to :with_includes' do
-      raw = Object.new
-      expect(C.new.cache_collection(raw, Object)).to eq raw
-    end
-
-    context 'Status' do
-      include_examples 'cacheable', :status, Status
-    end
-  end
 end

spec/lib/activitypub/parser/status_parser_spec.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe ActivityPub::Parser::StatusParser do
+  subject { described_class.new(json) }
+
+  let(:sender) { Fabricate(:account, followers_url: 'http://example.com/followers', domain: 'example.com', uri: 'https://example.com/actor') }
+  let(:follower) { Fabricate(:account, username: 'bob') }
+
+  let(:json) do
+    {
+      '@context': 'https://www.w3.org/ns/activitystreams',
+      id: [ActivityPub::TagManager.instance.uri_for(sender), '#foo'].join,
+      type: 'Create',
+      actor: ActivityPub::TagManager.instance.uri_for(sender),
+      object: object_json,
+    }.with_indifferent_access
+  end
+
+  let(:object_json) do
+    {
+      id: [ActivityPub::TagManager.instance.uri_for(sender), 'post1'].join('/'),
+      type: 'Note',
+      to: [
+        'https://www.w3.org/ns/activitystreams#Public',
+        ActivityPub::TagManager.instance.uri_for(follower),
+      ],
+      content: '@bob lorem ipsum',
+      contentMap: {
+        EN: '@bob lorem ipsum',
+      },
+      published: 1.hour.ago.utc.iso8601,
+      updated: 1.hour.ago.utc.iso8601,
+      tag: {
+        type: 'Mention',
+        href: ActivityPub::TagManager.instance.uri_for(follower),
+      },
+    }
+  end
+
+  it 'correctly parses status' do
+    expect(subject).to have_attributes(
+      text: '@bob lorem ipsum',
+      uri: [ActivityPub::TagManager.instance.uri_for(sender), 'post1'].join('/'),
+      reply: false,
+      language: :en
+    )
+  end
+end

spec/models/concerns/account_interactions_spec.rb

@@ -248,6 +248,24 @@ describe AccountInteractions do
     end
   end
 
+  describe '#block_idna_domain!' do
+    subject do
+      [
+        account.block_domain!(idna_domain),
+        account.block_domain!(punycode_domain),
+      ]
+    end
+
+    let(:idna_domain) { '대한민국.한국' }
+    let(:punycode_domain) { 'xn--3e0bs9hfvinn1a.xn--3e0b707e' }
+
+    it 'creates single AccountDomainBlock' do
+      expect do
+        expect(subject).to all(be_a AccountDomainBlock)
+      end.to change { account.domain_blocks.count }.by 1
+    end
+  end
+
   describe '#unfollow!' do
     subject { account.unfollow!(target_account) }
 
@@ -343,6 +361,28 @@ describe AccountInteractions do
     end
   end
 
+  describe '#unblock_idna_domain!' do
+    subject { account.unblock_domain!(punycode_domain) }
+
+    let(:idna_domain) { '대한민국.한국' }
+    let(:punycode_domain) { 'xn--3e0bs9hfvinn1a.xn--3e0b707e' }
+
+    context 'when blocking the domain' do
+      it 'returns destroyed AccountDomainBlock' do
+        account_domain_block = Fabricate(:account_domain_block, domain: idna_domain)
+        account.domain_blocks << account_domain_block
+        expect(subject).to be_a AccountDomainBlock
+        expect(subject).to be_destroyed
+      end
+    end
+
+    context 'when unblocking idna domain' do
+      it 'returns nil' do
+        expect(subject).to be_nil
+      end
+    end
+  end
+
   describe '#following?' do
     subject { account.following?(target_account) }
 

spec/models/home_feed_spec.rb

@@ -25,7 +25,6 @@ RSpec.describe HomeFeed, type: :model do
         results = subject.get(3)
 
         expect(results.map(&:id)).to eq [3, 2]
-        expect(results.first.attributes.keys).to eq %w(id updated_at)
       end
     end
 

spec/models/setting_spec.rb

@@ -142,22 +142,12 @@ RSpec.describe Setting, type: :model do
     context 'records includes nothing' do
       let(:records) { [] }
 
-      context 'default_value is not a Hash' do
+      it 'includes Setting with value of default_value' do
-        it 'includes Setting with value of default_value' do
+        setting = described_class.all_as_records[key]
-          setting = described_class.all_as_records[key]
 
-          expect(setting).to be_kind_of Setting
+        expect(setting).to be_a described_class
-          expect(setting).to have_attributes(var: key)
+        expect(setting).to have_attributes(var: key)
-          expect(setting).to have_attributes(value: 'default_value')
+        expect(setting).to have_attributes(value: default_value)
-        end
-      end
-
-      context 'default_value is a Hash' do
-        let(:default_value) { { 'foo' => 'fuga' } }
-
-        it 'returns {}' do
-          expect(described_class.all_as_records).to eq({})
-        end
       end
     end
   end

spec/models/user_spec.rb

@@ -40,6 +40,12 @@ RSpec.describe User, type: :model do
       expect(user.valid?).to be true
     end
 
+    it 'is valid with a localhost e-mail address' do
+      user = Fabricate.build(:user, email: 'admin@localhost')
+      user.valid?
+      expect(user.valid?).to be true
+    end
+
     it 'cleans out empty string from languages' do
       user = Fabricate.build(:user, chosen_languages: [''])
       user.valid?

spec/rails_helper.rb

@@ -41,6 +41,7 @@ RSpec.configure do |config|
 
   config.include Devise::Test::ControllerHelpers, type: :controller
   config.include Devise::Test::ControllerHelpers, type: :view
+  config.include Devise::Test::IntegrationHelpers, type: :request
   config.include Paperclip::Shoulda::Matchers
   config.include ActiveSupport::Testing::TimeHelpers
   config.include Redisable

spec/services/activitypub/process_account_service_spec.rb

@@ -3,7 +3,7 @@ require 'rails_helper'
 RSpec.describe ActivityPub::ProcessAccountService, type: :service do
   subject { described_class.new }
 
-  context 'property values' do
+  context 'with property values, an avatar, and a profile header' do
     let(:payload) do
       {
         id: 'https://foo.test',
@@ -14,19 +14,50 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do
           { type: 'PropertyValue', name: 'Occupation', value: 'Unit test' },
           { type: 'PropertyValue', name: 'non-string', value: ['foo', 'bar'] },
         ],
+        image: {
+          type: 'Image',
+          mediaType: 'image/png',
+          url: 'https://foo.test/image.png',
+        },
+        icon: {
+          type: 'Image',
+          url: [
+            {
+              mediaType: 'image/png',
+              href: 'https://foo.test/icon.png',
+            },
+          ],
+        },
       }.with_indifferent_access
     end
 
-    it 'parses out of attachment' do
+    before do
+      stub_request(:get, 'https://foo.test/image.png').to_return(request_fixture('avatar.txt'))
+      stub_request(:get, 'https://foo.test/icon.png').to_return(request_fixture('avatar.txt'))
+    end
+
+    it 'parses property values, avatar and profile header as expected' do
       account = subject.call('alice', 'example.com', payload)
-      expect(account.fields).to be_a Array
+
-      expect(account.fields.size).to eq 2
+      expect(account.fields)
-      expect(account.fields[0]).to be_a Account::Field
+        .to be_an(Array)
-      expect(account.fields[0].name).to eq 'Pronouns'
+        .and have_attributes(size: 2)
-      expect(account.fields[0].value).to eq 'They/them'
+      expect(account.fields.first)
-      expect(account.fields[1]).to be_a Account::Field
+        .to be_an(Account::Field)
-      expect(account.fields[1].name).to eq 'Occupation'
+        .and have_attributes(
-      expect(account.fields[1].value).to eq 'Unit test'
+          name: eq('Pronouns'),
+          value: eq('They/them')
+        )
+      expect(account.fields.last)
+        .to be_an(Account::Field)
+        .and have_attributes(
+          name: eq('Occupation'),
+          value: eq('Unit test')
+        )
+      expect(account).to have_attributes(
+        avatar_remote_url: 'https://foo.test/icon.png',
+        header_remote_url: 'https://foo.test/image.png'
+      )
     end
   end
 

spec/services/notify_service_spec.rb

@@ -72,11 +72,11 @@ RSpec.describe NotifyService, type: :service do
         end
       end
 
-      context 'if the message chain is initiated by recipient, but without a mention to the sender, even if the sender sends multiple messages in a row' do
+      context 'when the message chain is initiated by recipient, but without a mention to the sender, even if the sender sends multiple messages in a row' do
-        let(:reply_to) { Fabricate(:status, account: recipient) }
+        let(:public_status) { Fabricate(:status, account: recipient) }
-        let!(:mention) { Fabricate(:mention, account: sender, status: reply_to) }
+        let(:intermediate_reply) { Fabricate(:status, account: sender, thread: public_status, visibility: :direct) }
-        let(:dummy_reply) { Fabricate(:status, account: sender, visibility: :direct, thread: reply_to) }
+        let!(:intermediate_mention) { Fabricate(:mention, account: sender, status: intermediate_reply) }
-        let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: dummy_reply)) }
+        let(:activity) { Fabricate(:mention, account: recipient, status: Fabricate(:status, account: sender, visibility: :direct, thread: intermediate_reply)) }
 
         it 'does not notify' do
           expect { subject }.to_not change(Notification, :count)

spec/services/post_status_service_spec.rb

@@ -52,6 +52,13 @@ RSpec.describe PostStatusService, type: :service do
     it 'does not change statuses count' do
       expect { subject.call(account, text: 'Hi future!', scheduled_at: future, thread: previous_status) }.not_to change { [account.statuses_count, previous_status.replies_count] }
     end
+
+    it 'returns existing status when used twice with idempotency key' do
+      account = Fabricate(:account)
+      status1 = subject.call(account, text: 'test', idempotency: 'meepmeep', scheduled_at: future)
+      status2 = subject.call(account, text: 'test', idempotency: 'meepmeep', scheduled_at: future)
+      expect(status2.id).to eq status1.id
+    end
   end
 
   it 'creates response to the original status of boost' do

spec/services/verify_link_service_spec.rb

@@ -74,6 +74,20 @@ RSpec.describe VerifyLinkService, type: :service do
     end
 
     context 'when a document is truncated but the link back is valid' do
+      let(:html) do
+        "
+          <!doctype html>
+          <body>
+            <a rel=\"me\" href=\"#{ActivityPub::TagManager.instance.url_for(account)}\">
+        "
+      end
+
+      it 'marks the field as verified' do
+        expect(field.verified?).to be true
+      end
+    end
+
+    context 'when a link tag might be truncated' do
       let(:html) do
         "
           <!doctype html>
@@ -87,19 +101,6 @@ RSpec.describe VerifyLinkService, type: :service do
       end
     end
 
-    context 'when a link back might be truncated' do
-      let(:html) do
-        "
-          <!doctype html>
-          <body>
-            <a rel=\"me\" href=\"#{ActivityPub::TagManager.instance.url_for(account)}"
-      end
-
-      it 'does not mark the field as verified' do
-        expect(field.verified?).to be false
-      end
-    end
-
     context 'when a link does not contain a link back' do
       let(:html) { '' }
 

spec/spec_helper.rb

@@ -30,6 +30,12 @@ RSpec.configure do |config|
   config.before :suite do
     Rails.application.load_seed
     Chewy.strategy(:bypass)
+
+    # NOTE: we switched registrations mode to closed by default, but the specs
+    # very heavily rely on having it enabled by default, as it relies on users
+    # being approved by default except in select cases where explicitly testing
+    # other registration modes
+    Setting.registrations_mode = 'open'
   end
 
   config.after :suite do

spec/workers/scheduler/auto_close_registrations_scheduler_spec.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Scheduler::AutoCloseRegistrationsScheduler do
+  subject { described_class.new }
+
+  describe '#perform' do
+    let(:moderator_activity_date) { Time.now.utc }
+
+    before do
+      Fabricate(:user, role: UserRole.find_by(name: 'Owner'), current_sign_in_at: 10.years.ago)
+      Fabricate(:user, role: UserRole.find_by(name: 'Moderator'), current_sign_in_at: moderator_activity_date)
+    end
+
+    context 'when registrations are open' do
+      before do
+        Setting.registrations_mode = 'open'
+      end
+
+      context 'when a moderator has logged in recently' do
+        let(:moderator_activity_date) { Time.now.utc }
+
+        it 'does not change registrations mode' do
+          expect { subject.perform }.to_not change(Setting, :registrations_mode)
+        end
+      end
+
+      context 'when a moderator has not recently signed in' do
+        let(:moderator_activity_date) { 1.year.ago }
+
+        it 'changes registrations mode from open to approved' do
+          expect { subject.perform }.to change(Setting, :registrations_mode).from('open').to('approved')
+        end
+      end
+    end
+
+    context 'when registrations are closed' do
+      before do
+        Setting.registrations_mode = 'none'
+      end
+
+      context 'when a moderator has logged in recently' do
+        let(:moderator_activity_date) { Time.now.utc }
+
+        it 'does not change registrations mode' do
+          expect { subject.perform }.to_not change(Setting, :registrations_mode)
+        end
+      end
+
+      context 'when a moderator has not recently signed in' do
+        let(:moderator_activity_date) { 1.year.ago }
+
+        it 'does not change registrations mode' do
+          expect { subject.perform }.to_not change(Setting, :registrations_mode)
+        end
+      end
+    end
+  end
+end