chinwag/chinwagsocial
chinwag/chinwagsocial
2
0
Fork 0
Code Issues Pull requests Releases 14 Wiki Activity

Merge branch 'chinwag-next'

Browse source
This commit is contained in:
Mike Barnes 2025-09-17 20:31:48 +10:00
commit 095ce4fb34
2690 changed files with 96825 additions and 58372 deletions
Download patch file Download diff file Expand all files Collapse all files

4
lib/active_record/batches.rb
View file

@ -13,7 +13,9 @@ module ActiveRecord
column_names.unshift(primary_key)
relation = relation.reorder(build_batch_orders(order).to_h).limit(batch_limit)
cursor = Array(primary_key)
relation = relation.reorder(build_batch_orders(cursor, order).to_h).limit(batch_limit)
relation.skip_query_cache!
batch_relation = relation

3
lib/active_record/database_tasks_extensions.rb
View file

@ -1,5 +1,6 @@
# frozen_string_literal: true
require_relative '../mastodon/database'
require_relative '../mastodon/snowflake'
module ActiveRecord
@ -8,6 +9,8 @@ module ActiveRecord
original_load_schema = instance_method(:load_schema)
define_method(:load_schema) do |db_config, *args|
Mastodon::Database.add_post_migrate_path_to_rails(force: true)
ActiveRecord::Base.establish_connection(db_config)
Mastodon::Snowflake.define_timestamp_id

65
lib/active_record/with_recursive.rb
View file

@ -1,65 +0,0 @@
# frozen_string_literal: true
# Add support for writing recursive CTEs in ActiveRecord
# Initially from Lorin Thwaits (https://github.com/lorint) as per comment:
# https://github.com/vlado/activerecord-cte/issues/16#issuecomment-1433043310
# Modified from the above code to change the signature to
# `with_recursive(hash)` and extending CTE hash values to also includes arrays
# of values that get turned into UNION ALL expressions.
# This implementation has been merged in Rails: https://github.com/rails/rails/pull/51601
module ActiveRecord
module QueryMethodsExtensions
def with_recursive(*args)
@with_is_recursive = true
check_if_method_has_arguments!(__callee__, args)
spawn.with_recursive!(*args)
end
# Like #with_recursive but modifies the relation in place.
def with_recursive!(*args) # :nodoc:
self.with_values += args
@with_is_recursive = true
self
end
private
def build_with(arel)
return if with_values.empty?
with_statements = with_values.map do |with_value|
raise ArgumentError, "Unsupported argument type: #{with_value} #{with_value.class}" unless with_value.is_a?(Hash)
build_with_value_from_hash(with_value)
end
# Was: arel.with(with_statements)
@with_is_recursive ? arel.with(:recursive, with_statements) : arel.with(with_statements)
end
def build_with_value_from_hash(hash)
hash.map do |name, value|
Arel::Nodes::TableAlias.new(build_with_expression_from_value(value), name)
end
end
def build_with_expression_from_value(value)
case value
when Arel::Nodes::SqlLiteral then Arel::Nodes::Grouping.new(value)
when ActiveRecord::Relation then value.arel
when Arel::SelectManager then value
when Array then value.map { |e| build_with_expression_from_value(e) }.reduce { |result, value| Arel::Nodes::UnionAll.new(result, value) }
else
raise ArgumentError, "Unsupported argument type: `#{value}` #{value.class}"
end
end
end
end
ActiveSupport.on_load(:active_record) do
ActiveRecord::QueryMethods.prepend(ActiveRecord::QueryMethodsExtensions)
end

51
lib/arel/union_parenthesizing.rb
View file

@ -1,51 +0,0 @@
# frozen_string_literal: true
# Fix an issue with `LIMIT` ocurring on the left side of a `UNION` causing syntax errors.
# See https://github.com/rails/rails/issues/40181
# The fix has been merged in ActiveRecord: https://github.com/rails/rails/pull/51549
# TODO: drop this when available in ActiveRecord
# rubocop:disable all -- This is a mostly vendored file
module Arel
module Visitors
class ToSql
private
def infix_value_with_paren(o, collector, value, suppress_parens = false)
collector << "( " unless suppress_parens
collector = if o.left.class == o.class
infix_value_with_paren(o.left, collector, value, true)
else
select_parentheses o.left, collector, false # Changed from `visit o.left, collector`
end
collector << value
collector = if o.right.class == o.class
infix_value_with_paren(o.right, collector, value, true)
else
select_parentheses o.right, collector, false # Changed from `visit o.right, collector`
end
collector << " )" unless suppress_parens
collector
end
def select_parentheses(o, collector, always_wrap_selects = true)
if o.is_a?(Nodes::SelectStatement) && (always_wrap_selects || require_parentheses?(o))
collector << "("
visit o, collector
collector << ")"
collector
else
visit o, collector
end
end
def require_parentheses?(o)
!o.orders.empty? || o.limit || o.offset
end
end
end
end
# rubocop:enable all

7
lib/chewy/index_extensions.rb
View file

@ -12,6 +12,13 @@ module Chewy
base_options.merge(number_of_replicas: 1, number_of_shards: (base_options[:number_of_shards] || 1) * 2)
end
end
def update_specification
client.indices.close index: index_name
client.indices.put_settings index: index_name, body: { settings: { analysis: settings_hash[:settings][:analysis] } }
client.indices.put_mapping index: index_name, body: root.mappings_hash
client.indices.open index: index_name
end
end
end

2
lib/devise/strategies/two_factor_ldap_authenticatable.rb
View file

@ -23,7 +23,7 @@ module Devise
protected
def valid_params?
params[scope] && params[scope][:password].present?
params[scope].is_a?(Hash) && params[scope][:password].present?
end
end
end

2
lib/devise/strategies/two_factor_pam_authenticatable.rb
View file

@ -22,7 +22,7 @@ module Devise
protected
def valid_params?
params[scope] && params[scope][:password].present?
params[scope].is_a?(Hash) && params[scope][:password].present?
end
end
end

10
lib/exceptions.rb
View file

@ -12,6 +12,9 @@ module Mastodon
class RateLimitExceededError < Error; end
class SyntaxError < Error; end
class InvalidParameterError < Error; end
class SignatureVerificationError < Error; end
class MalformedHeaderError < Error; end
class RecursionLimitExceededError < Error; end
class UnexpectedResponseError < Error
attr_reader :response
@ -35,4 +38,11 @@ module Mastodon
super()
end
end
HTTP_CONNECTION_ERRORS = [
HTTP::ConnectionError,
HTTP::Error,
HTTP::TimeoutError,
OpenSSL::SSL::SSLError,
].freeze
end

18
lib/mastodon/cli/accounts.rb
View file

@ -1,6 +1,5 @@
# frozen_string_literal: true
require 'set'
require_relative 'base'
module Mastodon::CLI
@ -80,7 +79,14 @@ module Mastodon::CLI
account = Account.new(username: username)
password = SecureRandom.hex
user = User.new(email: options[:email], password: password, agreement: true, role_id: role_id, confirmed_at: options[:confirmed] ? Time.now.utc : nil, bypass_invite_request_check: true)
user = User.new(
email: options[:email],
password: password,
agreement: true,
role_id: role_id,
confirmed_at: options[:confirmed] ? Time.now.utc : nil,
bypass_registration_checks: true
)
if options[:reattach]
account = Account.find_local(username) || Account.new(username: username)
@ -165,7 +171,7 @@ module Mastodon::CLI
user.disabled = false if options[:enable]
user.disabled = true if options[:disable]
user.approved = true if options[:approve]
user.otp_required_for_login = false if options[:disable_2fa]
user.disable_two_factor! if options[:disable_2fa]
if user.save
user.confirm if options[:confirm]
@ -305,7 +311,7 @@ module Mastodon::CLI
begin
code = Request.new(:head, account.uri).perform(&:code)
rescue HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError, Mastodon::PrivateNetworkAddressError
rescue *Mastodon::HTTP_CONNECTION_ERRORS, Mastodon::PrivateNetworkAddressError
skip_domains << account.domain
end
@ -322,7 +328,9 @@ module Mastodon::CLI
unless skip_domains.empty?
say('The following domains were not available during the check:', :yellow)
skip_domains.each { |domain| say(" #{domain}") }
shell.indent(2) do
skip_domains.each { |domain| say(domain) }
end
end
end

4
lib/mastodon/cli/cache.rb
View file

@ -52,7 +52,7 @@ module Mastodon::CLI
account.account_stat.tap do |account_stat|
account_stat.following_count = account.active_relationships.count
account_stat.followers_count = account.passive_relationships.count
account_stat.statuses_count = account.statuses.where.not(visibility: :direct).count
account_stat.statuses_count = account.statuses.not_direct_visibility.count
account_stat.save if account_stat.changed?
end
@ -60,7 +60,7 @@ module Mastodon::CLI
def recount_status_stats(status)
status.status_stat.tap do |status_stat|
status_stat.replies_count = status.replies.where.not(visibility: :direct).count
status_stat.replies_count = status.replies.not_direct_visibility.count
status_stat.reblogs_count = status.reblogs.count
status_stat.favourites_count = status.favourites.count

17
lib/mastodon/cli/email_domain_blocks.rb
View file

@ -7,11 +7,13 @@ module Mastodon::CLI
class EmailDomainBlocks < Base
desc 'list', 'List blocked e-mail domains'
def list
EmailDomainBlock.where(parent_id: nil).find_each do |entry|
say(entry.domain.to_s, :white)
EmailDomainBlock.parents.find_each do |parent|
say(parent.domain.to_s, :white)
EmailDomainBlock.where(parent_id: entry.id).find_each do |child|
say(" #{child.domain}", :cyan)
shell.indent do
EmailDomainBlock.where(parent_id: parent.id).find_each do |child|
say(child.domain, :cyan)
end
end
end
end
@ -43,12 +45,7 @@ module Mastodon::CLI
end
other_domains = []
if options[:with_dns_records]
Resolv::DNS.open do |dns|
dns.timeouts = 5
other_domains = dns.getresources(domain, Resolv::DNS::Resource::IN::MX).to_a.map { |e| e.exchange.to_s }.compact_blank
end
end
other_domains = DomainResource.new(domain).mx if options[:with_dns_records]
email_domain_block = EmailDomainBlock.new(domain: domain, other_domains: other_domains)
email_domain_block.save!

4
lib/mastodon/cli/emoji.rb
View file

@ -62,7 +62,9 @@ module Mastodon::CLI
failed += 1
say('Failure/Error: ', :red)
say(entry.full_name)
say(" #{custom_emoji.errors[:image].join(', ')}", :red)
shell.indent(2) do
say(custom_emoji.errors[:image].join(', '), :red)
end
end
end
end

2
lib/mastodon/cli/federation.rb
View file

@ -76,7 +76,7 @@ module Mastodon::CLI
def self_destruct_value
Rails
.application
.message_verifier('self-destruct')
.message_verifier(SelfDestructHelper::VERIFY_PURPOSE)
.generate(Rails.configuration.x.local_domain)
end
end

8
lib/mastodon/cli/feeds.rb
View file

@ -11,17 +11,21 @@ module Mastodon::CLI
option :concurrency, type: :numeric, default: 5, aliases: [:c]
option :verbose, type: :boolean, aliases: [:v]
option :dry_run, type: :boolean, default: false
option :skip_filled_timelines
desc 'build [USERNAME]', 'Build home and list feeds for one or all users'
long_desc <<-LONG_DESC
Build home and list feeds that are stored in Redis from the database.
With the --skip-filled-timelines, timelines which contain more than half
the maximum number of posts will be skipped.
With the --all option, all active users will be processed.
Otherwise, a single user specified by USERNAME.
LONG_DESC
def build(username = nil)
if options[:all] || username.nil?
processed, = parallelize_with_progress(active_user_accounts) do |account|
PrecomputeFeedService.new.call(account) unless dry_run?
PrecomputeFeedService.new.call(account, skip_filled_timelines: options[:skip_filled_timelines]) unless dry_run?
end
say("Regenerated feeds for #{processed} accounts #{dry_run_mode_suffix}", :green, true)
@ -30,7 +34,7 @@ module Mastodon::CLI
fail_with_message 'No such account' if account.nil?
PrecomputeFeedService.new.call(account) unless dry_run?
PrecomputeFeedService.new.call(account, skip_filled_timelines: options[:skip_filled_timelines]) unless dry_run?
say("OK #{dry_run_mode_suffix}", :green, true)
else

4
lib/mastodon/cli/ip_blocks.rb
View file

@ -80,9 +80,9 @@ module Mastodon::CLI
end
ip_blocks = if options[:force]
IpBlock.where('ip >>= ?', address)
IpBlock.containing(address)
else
IpBlock.where('ip <<= ?', address)
IpBlock.contained_by(address)
end
if ip_blocks.empty?

8
lib/mastodon/cli/maintenance.rb
View file

@ -192,6 +192,7 @@ module Mastodon::CLI
verify_schema_version!
verify_sidekiq_not_active!
verify_backup_warning!
disable_timeout!
end
def process_deduplications
@ -251,6 +252,13 @@ module Mastodon::CLI
fail_with_message 'Maintenance process stopped.' unless yes?('Continue? (Yes/No)')
end
def disable_timeout!
# Remove server-configured timeout if present
database_connection.execute(<<~SQL.squish)
SET statement_timeout = 0
SQL
end
def deduplicate_accounts!
remove_index_if_exists!(:accounts, 'index_accounts_on_username_and_domain_lower')

114
lib/mastodon/cli/media.rb
View file

@ -6,6 +6,8 @@ module Mastodon::CLI
class Media < Base
include ActionView::Helpers::NumberHelper
class UnrecognizedOrphanType < StandardError; end
VALID_PATH_SEGMENTS_SIZE = [7, 10].freeze
option :days, type: :numeric, default: 7, aliases: [:d]
@ -120,23 +122,10 @@ module Mastodon::CLI
object.acl.put(acl: s3_permissions) if options[:fix_permissions] && !dry_run?
path_segments = object.key.split('/')
path_segments.delete('cache')
unless VALID_PATH_SEGMENTS_SIZE.include?(path_segments.size)
progress.log(pastel.yellow("Unrecognized file found: #{object.key}"))
next
end
model_name = path_segments.first.classify
attachment_name = path_segments[1].singularize
record_id = path_segments[2...-2].join.to_i
file_name = path_segments.last
record = record_map.dig(model_name, record_id)
attachment = record&.public_send(attachment_name)
progress.increment
next unless attachment.blank? || !attachment.variant?(file_name)
next unless orphaned_file?(path_segments, record_map)
begin
object.delete unless dry_run?
@ -148,6 +137,8 @@ module Mastodon::CLI
rescue => e
progress.log(pastel.red("Error processing #{object.key}: #{e}"))
end
rescue UnrecognizedOrphanType
progress.log(pastel.yellow("Unrecognized file found: #{object.key}"))
end
end
when :fog
@ -165,26 +156,10 @@ module Mastodon::CLI
key = path.gsub("#{root_path}#{File::SEPARATOR}", '')
path_segments = key.split(File::SEPARATOR)
path_segments.delete('cache')
unless VALID_PATH_SEGMENTS_SIZE.include?(path_segments.size)
progress.log(pastel.yellow("Unrecognized file found: #{key}"))
next
end
model_name = path_segments.first.classify
record_id = path_segments[2...-2].join.to_i
attachment_name = path_segments[1].singularize
file_name = path_segments.last
next unless PRELOAD_MODEL_WHITELIST.include?(model_name)
record = model_name.constantize.find_by(id: record_id)
attachment = record&.public_send(attachment_name)
progress.increment
next unless attachment.blank? || !attachment.variant?(file_name)
next unless orphaned_file?(path_segments)
begin
size = File.size(path)
@ -205,6 +180,8 @@ module Mastodon::CLI
rescue => e
progress.log(pastel.red("Error processing #{key}: #{e}"))
end
rescue UnrecognizedOrphanType
progress.log(pastel.yellow("Unrecognized file found: #{path}"))
end
end
@ -278,14 +255,10 @@ module Mastodon::CLI
desc 'usage', 'Calculate disk space consumed by Mastodon'
def usage
say("Attachments:\t#{number_to_human_size(media_attachment_storage_size)} (#{number_to_human_size(local_media_attachment_storage_size)} local)")
say("Custom emoji:\t#{number_to_human_size(CustomEmoji.sum(:image_file_size))} (#{number_to_human_size(CustomEmoji.local.sum(:image_file_size))} local)")
say("Preview cards:\t#{number_to_human_size(PreviewCard.sum(:image_file_size))}")
say("Avatars:\t#{number_to_human_size(Account.sum(:avatar_file_size))} (#{number_to_human_size(Account.local.sum(:avatar_file_size))} local)")
say("Headers:\t#{number_to_human_size(Account.sum(:header_file_size))} (#{number_to_human_size(Account.local.sum(:header_file_size))} local)")
say("Backups:\t#{number_to_human_size(Backup.sum(:dump_file_size))}")
say("Imports:\t#{number_to_human_size(Import.sum(:data_file_size))}")
say("Settings:\t#{number_to_human_size(SiteUpload.sum(:file_file_size))}")
print_table [
%w(Object Total Local),
*object_storage_summary,
]
end
desc 'lookup URL', 'Lookup where media is displayed by passing a media URL'
@ -300,7 +273,7 @@ module Mastodon::CLI
model_name = path_segments.first.classify
record_id = path_segments[2...-2].join.to_i
fail_with_message "Cannot find corresponding model: #{model_name}" unless PRELOAD_MODEL_WHITELIST.include?(model_name)
fail_with_message "Cannot find corresponding model: #{model_name}" unless PRELOADED_MODELS.include?(model_name)
record = model_name.constantize.find_by(id: record_id)
record = record.status if record.respond_to?(:status)
@ -316,34 +289,35 @@ module Mastodon::CLI
fail_with_message 'Invalid URL'
end
private
def media_attachment_storage_size
MediaAttachment.sum(file_and_thumbnail_size_sql)
end
def local_media_attachment_storage_size
MediaAttachment.where(account: Account.local).sum(file_and_thumbnail_size_sql)
end
def file_and_thumbnail_size_sql
Arel.sql(
<<~SQL.squish
COALESCE(file_file_size, 0) + COALESCE(thumbnail_file_size, 0)
SQL
)
end
PRELOAD_MODEL_WHITELIST = %w(
PRELOADED_MODELS = %w(
Account
Backup
CustomEmoji
Import
MediaAttachment
PreviewCard
SiteUpload
).freeze
private
def object_storage_summary
[
[:attachments, MediaAttachment.sum(combined_media_sum), MediaAttachment.where(account: Account.local).sum(combined_media_sum)],
[:custom_emoji, CustomEmoji.sum(:image_file_size), CustomEmoji.local.sum(:image_file_size)],
[:avatars, Account.sum(:avatar_file_size), Account.local.sum(:avatar_file_size)],
[:headers, Account.sum(:header_file_size), Account.local.sum(:header_file_size)],
[:preview_cards, PreviewCard.sum(:image_file_size), nil],
[:backups, Backup.sum(:dump_file_size), nil],
[:settings, SiteUpload.sum(:file_file_size), nil],
].map { |label, total, local| [label.to_s.titleize, number_to_human_size(total), local.present? ? number_to_human_size(local) : nil] }
end
def combined_media_sum
Arel.sql(<<~SQL.squish)
COALESCE(file_file_size, 0) + COALESCE(thumbnail_file_size, 0)
SQL
end
def preload_records_from_mixed_objects(objects)
preload_map = Hash.new { |hash, key| hash[key] = [] }
@ -356,7 +330,7 @@ module Mastodon::CLI
model_name = segments.first.classify
record_id = segments[2...-2].join.to_i
next unless PRELOAD_MODEL_WHITELIST.include?(model_name)
next unless PRELOADED_MODELS.include?(model_name)
preload_map[model_name] << record_id
end
@ -365,5 +339,23 @@ module Mastodon::CLI
model_map[model_name] = model_name.constantize.where(id: record_ids).index_by(&:id)
end
end
def orphaned_file?(path_segments, record_map = nil)
path_segments.delete('cache')
raise UnrecognizedOrphanType unless VALID_PATH_SEGMENTS_SIZE.include?(path_segments.size)
model_name = path_segments.first.classify
record_id = path_segments[2...-2].join.to_i
attachment_name = path_segments[1].singularize
file_name = path_segments.last
raise UnrecognizedOrphanType unless PRELOADED_MODELS.include?(model_name)
record = record_map.present? ? record_map.dig(model_name, record_id) : model_name.constantize.find_by(id: record_id)
attachment = record&.public_send(attachment_name)
attachment.blank? || !attachment.variant?(file_name)
end
end
end

4
lib/mastodon/cli/progress_helper.rb
View file

@ -1,11 +1,11 @@
# frozen_string_literal: true
dev_null = Logger.new('/dev/null')
dev_null = Logger.new(File::NULL)
Rails.logger = dev_null
ActiveRecord::Base.logger = dev_null
ActiveJob::Base.logger = dev_null
HttpLog.configuration.logger = dev_null
HttpLog.configuration.logger = dev_null if defined?(HttpLog)
Paperclip.options[:log] = false
Chewy.logger = dev_null

15
lib/mastodon/cli/search.rb
View file

@ -20,6 +20,7 @@ module Mastodon::CLI
option :import, type: :boolean, default: true, desc: 'Import data from the database to the index'
option :clean, type: :boolean, default: true, desc: 'Remove outdated documents from the index'
option :reset_chewy, type: :boolean, default: false, desc: "Reset Chewy's internal index"
option :only_mapping, type: :boolean, default: false, desc: 'Update the index specification without re-index'
desc 'deploy', 'Create or upgrade Elasticsearch indices and populate them'
long_desc <<~LONG_DESC
If Elasticsearch is empty, this command will create the necessary indices
@ -52,6 +53,20 @@ module Mastodon::CLI
Chewy::Stash::Specification.reset! if options[:reset_chewy]
if options[:only_mapping]
indices.select { |index| index.specification.changed? }.each do |index|
progress.title = "Updating mapping for #{index} "
index.update_specification
index.specification.lock!
end
progress.title = 'Done! '
progress.finish
say('Updated index mappings', :green, true)
return
end
# First, ensure all indices are created and have the correct
# structure, so that live data can already be written
indices.select { |index| index.specification.changed? }.each do |index|

19
lib/mastodon/cli/statuses.rb
View file

@ -40,17 +40,11 @@ module Mastodon::CLI
def remove_statuses
return if options[:skip_status_remove]
say('Creating temporary database indices...')
ActiveRecord::Base.connection.add_index(:media_attachments, :remote_url, name: :index_media_attachments_remote_url, where: 'remote_url is not null', algorithm: :concurrently, if_not_exists: true)
max_id = Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false)
start_at = Time.now.to_f
unless options[:continue] && ActiveRecord::Base.connection.table_exists?('statuses_to_be_deleted')
ActiveRecord::Base.connection.add_index(:accounts, :id, name: :index_accounts_local, where: 'domain is null', algorithm: :concurrently, if_not_exists: true)
ActiveRecord::Base.connection.add_index(:status_pins, :status_id, name: :index_status_pins_status_id, algorithm: :concurrently, if_not_exists: true)
max_id = Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false)
unless options[:continue] && ActiveRecord::Base.connection.table_exists?('statuses_to_be_deleted')
say('Extract the deletion target from statuses... This might take a while...')
ActiveRecord::Base.connection.create_table('statuses_to_be_deleted', force: true)
@ -72,9 +66,6 @@ module Mastodon::CLI
SQL
say('Removing temporary database indices to restore write performance...')
ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local, if_exists: true)
ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id, if_exists: true)
end
say('Beginning statuses removal... This might take a while...')
@ -102,12 +93,6 @@ module Mastodon::CLI
ActiveRecord::Base.connection.drop_table('statuses_to_be_deleted')
say("Done after #{Time.now.to_f - start_at}s, removed #{removed} out of #{processed} statuses.", :green)
ensure
say('Removing temporary database indices to restore write performance...')
ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local, if_exists: true)
ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id, if_exists: true)
ActiveRecord::Base.connection.remove_index(:media_attachments, name: :index_media_attachments_remote_url, if_exists: true)
end
def remove_orphans_media_attachments

46
lib/mastodon/database.rb Normal file
View file

@ -0,0 +1,46 @@
# frozen_string_literal: true
# This file is entirely lifted from GitLab.
# The original file:
# https://gitlab.com/gitlab-org/gitlab/-/blob/69127d59467185cf4ff1109d88ceec1f499f354f/lib/gitlab/database.rb#L244-258
# Copyright (c) 2011-present GitLab B.V.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
module Mastodon
module Database
def self.add_post_migrate_path_to_rails(force: false)
return if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && !force
Rails.application.config.paths['db'].each do |db_path|
path = Rails.root.join(db_path, 'post_migrate').to_s
next if Rails.application.config.paths['db/migrate'].include?(path)
Rails.application.config.paths['db/migrate'] << path
# Rails memoizes migrations at certain points where it won't read the above
# path just yet. As such we must also update the following list of paths.
ActiveRecord::Migrator.migrations_paths << path
end
end
end
end

33
lib/mastodon/email_configuration_helper.rb Normal file
View file

@ -0,0 +1,33 @@
# frozen_string_literal: true
module Mastodon
module EmailConfigurationHelper
module_function
# Convert smtp settings from environment variables (or defaults in
# `config/email.yml`) into the format that `ActionMailer` understands
def convert_smtp_settings(config)
enable_starttls = nil
enable_starttls_auto = nil
case config[:enable_starttls]
when 'always'
enable_starttls = true
when 'never'
enable_starttls = false
when 'auto'
enable_starttls_auto = true
else
enable_starttls_auto = config[:enable_starttls_auto] != 'false'
end
authentication = config[:authentication] == 'none' ? nil : (config[:authentication] || 'plain')
config.merge(
authentication:,
enable_starttls:,
enable_starttls_auto:
)
end
end
end

26
lib/mastodon/feature.rb Normal file
View file

@ -0,0 +1,26 @@
# frozen_string_literal: true
module Mastodon::Feature
class << self
def enabled_features
@enabled_features ||=
(Rails.configuration.x.mastodon.experimental_features || '').split(',').map(&:strip)
end
def method_missing(name)
if respond_to_missing?(name)
feature = name.to_s.delete_suffix('_enabled?')
enabled = enabled_features.include?(feature)
define_singleton_method(name) { enabled }
return enabled
end
super
end
def respond_to_missing?(name, include_all = false)
name.to_s.end_with?('_enabled?') || super
end
end
end

24
lib/mastodon/middleware/prometheus_queue_time.rb Normal file
View file

@ -0,0 +1,24 @@
# frozen_string_literal: true
module Mastodon
module Middleware
class PrometheusQueueTime < ::PrometheusExporter::Middleware
# Overwrite to only collect the queue time metric
def call(env)
queue_time = measure_queue_time(env)
result = @app.call(env)
result
ensure
obj = {
type: 'web',
queue_time: queue_time,
default_labels: {},
}
@client.send_json(obj)
end
end
end
end

51
lib/mastodon/middleware/public_file_server.rb Normal file
View file

@ -0,0 +1,51 @@
# frozen_string_literal: true
require 'action_dispatch/middleware/static'
module Mastodon
module Middleware
class PublicFileServer
SERVICE_WORKER_TTL = 7.days.to_i
CACHE_TTL = 28.days.to_i
def initialize(app)
@app = app
@file_handler = ActionDispatch::FileHandler.new(Rails.application.paths['public'].first)
end
def call(env)
file = @file_handler.attempt(env)
# If the request is not a static file, move on!
return @app.call(env) if file.nil?
status, headers, response = file
# Set cache headers on static files. Some paths require different cache headers
request = Rack::Request.new env
headers['cache-control'] = begin
if request.path.start_with?('/sw.js')
"public, max-age=#{SERVICE_WORKER_TTL}, must-revalidate"
elsif request.path.start_with?(paperclip_root_url)
"public, max-age=#{CACHE_TTL}, immutable"
else
"public, max-age=#{CACHE_TTL}, must-revalidate"
end
end
# Override the default CSP header set by the CSP middleware
headers['content-security-policy'] = "default-src 'none'; form-action 'none'" if request.path.start_with?(paperclip_root_url)
headers['x-content-type-options'] = 'nosniff'
[status, headers, response]
end
private
def paperclip_root_url
ENV.fetch('PAPERCLIP_ROOT_URL', '/system')
end
end
end
end

34
lib/mastodon/middleware/socket_cleanup.rb Normal file
View file

@ -0,0 +1,34 @@
# frozen_string_literal: true
module Mastodon
module Middleware
class SocketCleanup
def initialize(app)
@app = app
end
def call(env)
@app.call(env)
ensure
clean_up_sockets!
end
private
def clean_up_sockets!
clean_up_redis_socket!
clean_up_statsd_socket!
end
def clean_up_redis_socket!
RedisConnection.pool.checkin if Thread.current[:redis]
Thread.current[:redis] = nil
end
def clean_up_statsd_socket!
Thread.current[:statsd_socket]&.close
Thread.current[:statsd_socket] = nil
end
end
end
end

4
lib/mastodon/migration_warning.rb
View file

@ -4,7 +4,7 @@ module Mastodon
module MigrationWarning
WARNING_SECONDS = 10
DEFAULT_WARNING = <<~WARNING_MESSAGE
DEFAULT_WARNING = <<~WARNING_MESSAGE.freeze
WARNING: This migration may take a *long* time for large instances.
It will *not* lock tables for any significant time, but it may run
for a very long time. We will pause for #{WARNING_SECONDS} seconds to allow you to
@ -23,7 +23,7 @@ module Mastodon
def announce_countdown
WARNING_SECONDS.downto(1) do |i|
say "Continuing in #{i} second#{i == 1 ? '' : 's'}...", true
say "Continuing in #{i} second#{'s' unless i == 1}...", true
sleep 1
end
end

18
lib/mastodon/prometheus_exporter/local_server.rb Normal file
View file

@ -0,0 +1,18 @@
# frozen_string_literal: true
require 'prometheus_exporter/server'
require 'prometheus_exporter/client'
module Mastodon::PrometheusExporter
module LocalServer
mattr_accessor :bind, :port
def self.setup!
server = PrometheusExporter::Server::WebServer.new(bind:, port:)
server.start
# wire up a default local client
PrometheusExporter::Client.default = PrometheusExporter::LocalClient.new(collector: server.collector)
end
end
end

30
lib/mastodon/rack_middleware.rb
View file

@ -1,30 +0,0 @@
# frozen_string_literal: true
class Mastodon::RackMiddleware
def initialize(app)
@app = app
end
def call(env)
@app.call(env)
ensure
clean_up_sockets!
end
private
def clean_up_sockets!
clean_up_redis_socket!
clean_up_statsd_socket!
end
def clean_up_redis_socket!
RedisConnection.pool.checkin if Thread.current[:redis]
Thread.current[:redis] = nil
end
def clean_up_statsd_socket!
Thread.current[:statsd_socket]&.close
Thread.current[:statsd_socket] = nil
end
end

26
lib/mastodon/redis_configuration.rb
View file

@ -9,22 +9,20 @@ class Mastodon::RedisConfiguration
def base
@base ||= setup_config(prefix: nil, defaults: DEFAULTS)
.merge(namespace: base_namespace)
end
def sidekiq
@sidekiq ||= setup_config(prefix: 'SIDEKIQ_')
.merge(namespace: sidekiq_namespace)
end
def cache
@cache ||= setup_config(prefix: 'CACHE_')
.merge({
namespace: cache_namespace,
namespace: 'cache',
expires_in: 10.minutes,
connect_timeout: 5,
pool: {
size: Sidekiq.server? ? Sidekiq[:concurrency] : Integer(ENV['MAX_THREADS'] || 5),
size: Sidekiq.server? ? Sidekiq.default_configuration[:concurrency] : Integer(ENV['MAX_THREADS'] || 5),
timeout: 5,
},
})
@ -36,24 +34,6 @@ class Mastodon::RedisConfiguration
ENV['REDIS_DRIVER'] == 'ruby' ? :ruby : :hiredis
end
def namespace
@namespace ||= ENV.fetch('REDIS_NAMESPACE', nil)
end
def base_namespace
return "mastodon_test#{ENV.fetch('TEST_ENV_NUMBER', nil)}" if Rails.env.test?
namespace
end
def sidekiq_namespace
namespace
end
def cache_namespace
namespace ? "#{namespace}_cache" : 'cache'
end
def setup_config(prefix: nil, defaults: {})
prefix = "#{prefix}REDIS_"
@ -62,7 +42,7 @@ class Mastodon::RedisConfiguration
password = ENV.fetch("#{prefix}PASSWORD", nil)
host = ENV.fetch("#{prefix}HOST", defaults[:host])
port = ENV.fetch("#{prefix}PORT", defaults[:port])
db = ENV.fetch("#{prefix}DB", defaults[:db])
db = Rails.env.test? ? ENV.fetch('TEST_ENV_NUMBER', defaults[:db]).to_i + 1 : ENV.fetch("#{prefix}DB", defaults[:db])
return { url:, driver: } if url

16
lib/mastodon/sidekiq_middleware.rb
View file

@ -3,8 +3,10 @@
class Mastodon::SidekiqMiddleware
BACKTRACE_LIMIT = 3
def call(*, &block)
Chewy.strategy(:mastodon, &block)
def call(_worker_class, job, _queue, &block)
setup_query_log_tags(job) do
Chewy.strategy(:mastodon, &block)
end
rescue Mastodon::HostValidationError
# Do not retry
rescue => e
@ -61,4 +63,14 @@ class Mastodon::SidekiqMiddleware
Thread.current[:statsd_socket]&.close
Thread.current[:statsd_socket] = nil
end
def setup_query_log_tags(job, &block)
if Rails.configuration.active_record.query_log_tags_enabled
# If `wrapped` is set, this is an `ActiveJob` which is already in the execution context
sidekiq_job_class = job['wrapped'].present? ? nil : job['class'].to_s
ActiveSupport::ExecutionContext.set(sidekiq_job_class: sidekiq_job_class, &block)
else
yield
end
end
end

28
lib/mastodon/version.rb
View file

@ -9,11 +9,11 @@ module Mastodon
end
def minor
3
4
end
def patch
8
4
end
def default_prerelease
@ -21,7 +21,7 @@ module Mastodon
end
def prerelease
ENV['MASTODON_VERSION_PRERELEASE'].presence || default_prerelease
version_configuration[:prerelease].presence || default_prerelease
end
def build_metadata
@ -45,7 +45,7 @@ module Mastodon
def api_versions
{
mastodon: 2,
mastodon: 6,
}
end
@ -59,7 +59,7 @@ module Mastodon
# specify git tag or commit hash here
def source_tag
ENV.fetch('SOURCE_TAG', nil)
source_configuration[:tag]
end
def source_url
@ -70,8 +70,24 @@ module Mastodon
end
end
def source_commit
ENV.fetch('SOURCE_COMMIT', nil)
end
def user_agent
@user_agent ||= "Mastodon/#{Version} (#{HTTP::Request::USER_AGENT}; +http#{Rails.configuration.x.use_https ? 's' : ''}://#{Rails.configuration.x.web_domain}/)"
@user_agent ||= "Mastodon/#{Version} (#{HTTP::Request::USER_AGENT}; +http#{'s' if Rails.configuration.x.use_https}://#{Rails.configuration.x.web_domain}/)"
end
def version_configuration
mastodon_configuration.version
end
def source_configuration
mastodon_configuration.source
end
def mastodon_configuration
Rails.configuration.x.mastodon
end
end
end

2
lib/paperclip/color_extractor.rb
View file

@ -239,7 +239,7 @@ module Paperclip
end
def rgb_to_hex(rgb)
format('#%02x%02x%02x', rgb.r, rgb.g, rgb.b)
format('#%02x%02x%02x', rgb.r, rgb.g, rgb.b) # rubocop:disable Style/FormatStringToken
end
end
end

2
lib/paperclip/transcoder.rb
View file

@ -52,8 +52,8 @@ module Paperclip
@output_options['bufsize'] = bitrate * 5
if high_vfr?(metadata)
# TODO: change to `fps_mode` in the future, as `vsync` is being deprecated
@output_options['vsync'] = 'vfr'
@output_options['r'] = @vfr_threshold
end
end
end

33
lib/premailer_bundled_asset_strategy.rb
View file

@ -2,21 +2,28 @@
module PremailerBundledAssetStrategy
def load(url)
asset_host = ENV['CDN_HOST'] || ENV['WEB_DOMAIN'] || ENV.fetch('LOCAL_DOMAIN', nil)
if ViteRuby.instance.dev_server_running?
# Request from the dev server
return unless url.start_with?("/#{ViteRuby.config.public_output_dir}/")
if Webpacker.dev_server.running?
asset_host = "#{Webpacker.dev_server.protocol}://#{Webpacker.dev_server.host_with_port}"
url = File.join(asset_host, url)
headers = {}
# Vite dev server wants this header for CSS files, otherwise it will respond with a JS file that inserts the CSS (to support hot reloading)
headers['Accept'] = 'text/css' if url.end_with?('.scss', '.css')
Net::HTTP.get(
URI("#{ViteRuby.config.origin}#{url}"),
headers
).presence
else
url = url.delete_prefix(Rails.configuration.action_controller.asset_host) if Rails.configuration.action_controller.asset_host.present?
url = url.delete_prefix('/')
path = Rails.public_path.join(url)
return unless path.exist?
path.read
end
css = if url.start_with?('http')
HTTP.get(url).to_s
else
url = url[1..] if url.start_with?('/')
Rails.public_path.join(url).read
end
css.gsub(%r{url\(/}, "url(#{asset_host}/")
rescue ViteRuby::MissingEntrypointError
# If the path is not in the manifest, ignore it
end
module_function :load

48
lib/public_file_server_middleware.rb
View file

@ -1,48 +0,0 @@
# frozen_string_literal: true
require 'action_dispatch/middleware/static'
class PublicFileServerMiddleware
SERVICE_WORKER_TTL = 7.days.to_i
CACHE_TTL = 28.days.to_i
def initialize(app)
@app = app
@file_handler = ActionDispatch::FileHandler.new(Rails.application.paths['public'].first)
end
def call(env)
file = @file_handler.attempt(env)
# If the request is not a static file, move on!
return @app.call(env) if file.nil?
status, headers, response = file
# Set cache headers on static files. Some paths require different cache headers
headers['Cache-Control'] = begin
request_path = env['REQUEST_PATH']
if request_path.start_with?('/sw.js')
"public, max-age=#{SERVICE_WORKER_TTL}, must-revalidate"
elsif request_path.start_with?(paperclip_root_url)
"public, max-age=#{CACHE_TTL}, immutable"
else
"public, max-age=#{CACHE_TTL}, must-revalidate"
end
end
# Override the default CSP header set by the CSP middleware
headers['Content-Security-Policy'] = "default-src 'none'; form-action 'none'" if request_path.start_with?(paperclip_root_url)
headers['X-Content-Type-Options'] = 'nosniff'
[status, headers, response]
end
private
def paperclip_root_url
ENV.fetch('PAPERCLIP_ROOT_URL', '/system')
end
end

16
lib/redis/namespace_extensions.rb
View file

@ -1,16 +0,0 @@
# frozen_string_literal: true
class Redis
module NamespaceExtensions
def exists?(...)
call_with_namespace('exists?', ...)
end
def with
yield self
end
end
end
Redis::Namespace::COMMANDS['exists?'] = [:first]
Redis::Namespace.prepend(Redis::NamespaceExtensions)

47
lib/sanitize_ext/sanitize_config.rb
View file

@ -21,7 +21,7 @@ class Sanitize
gemini
).freeze
CLASS_WHITELIST_TRANSFORMER = lambda do |env|
ALLOWED_CLASS_TRANSFORMER = lambda do |env|
node = env[:node]
class_list = node['class']&.split(/[\t\n\f\r ]/)
@ -31,6 +31,7 @@ class Sanitize
next true if /^(h|p|u|dt|e)-/.match?(e) # microformats classes
next true if /^(mention|hashtag)$/.match?(e) # semantic classes
next true if /^(ellipsis|invisible)$/.match?(e) # link formatting classes
next true if e == 'quote-inline'
end
node['class'] = class_list.join(' ')
@ -64,6 +65,44 @@ class Sanitize
current_node.wrap('<p></p>')
end
# We assume that incomming <math> nodes are of the form
# <math><semantics>...<annotation>...</annotation></semantics></math>
# according to the [FEP]. We try to grab the most relevant plain-text
# annotation from the semantics node, and use it to display a representation
# of the mathematics.
#
# FEP: https://codeberg.org/fediverse/fep/src/branch/main/fep/dc88/fep-dc88.md
MATH_TRANSFORMER = lambda do |env|
math = env[:node]
return if env[:is_allowlisted]
return unless math.element? && env[:node_name] == 'math'
semantics = math.element_children[0]
return if semantics.nil? || semantics.name != 'semantics'
# next, we find the plain-text description
is_annotation_with_encoding = lambda do |encoding, node|
return false unless node.name == 'annotation'
node.attributes['encoding'].value == encoding
end
annotation = semantics.children.find(&is_annotation_with_encoding.curry['application/x-tex'])
if annotation
text = if math.attributes['display']&.value == 'block'
"$$#{annotation.text}$$"
else
"$#{annotation.text}$"
end
math.replace(math.document.create_text_node(text))
return
end
# Don't bother surrounding 'text/plain' annotations with dollar signs,
# since it isn't LaTeX
annotation = semantics.children.find(&is_annotation_with_encoding.curry['text/plain'])
math.replace(math.document.create_text_node(annotation.text)) unless annotation.nil?
end
MASTODON_STRICT = freeze_config(
elements: %w(p br span a del s pre blockquote code b strong u i em ul ol li ruby rt rp),
@ -72,11 +111,12 @@ class Sanitize
'span' => %w(class translate),
'ol' => %w(start reversed),
'li' => %w(value),
'p' => %w(class),
},
add_attributes: {
'a' => {
'rel' => 'nofollow noopener noreferrer',
'rel' => 'nofollow noopener',
'target' => '_blank',
},
},
@ -84,8 +124,9 @@ class Sanitize
protocols: {},
transformers: [
CLASS_WHITELIST_TRANSFORMER,
ALLOWED_CLASS_TRANSFORMER,
TRANSLATE_TRANSFORMER,
MATH_TRANSFORMER,
UNSUPPORTED_ELEMENTS_TRANSFORMER,
UNSUPPORTED_HREF_TRANSFORMER,
]

17
lib/stoplight/redis_data_store_extensions.rb
View file

@ -1,17 +0,0 @@
# frozen_string_literal: true
# Restore compatibility with Redis < 6.2
module Stoplight
module DataStore
module RedisExtensions
def query_failures(light, transaction: @redis)
window_start = Time.now.to_i - light.window_size
transaction.zrevrangebyscore(failures_key(light), Float::INFINITY, window_start)
end
end
end
end
Stoplight::DataStore::Redis.prepend(Stoplight::DataStore::RedisExtensions)

10
lib/tasks/annotate_rb.rake Normal file
View file

@ -0,0 +1,10 @@
# frozen_string_literal: true
# This rake task was added by annotate_rb gem.
# Can set `ANNOTATERB_SKIP_ON_DB_TASKS` to be anything to skip this
if Rails.env.development? && ENV['ANNOTATERB_SKIP_ON_DB_TASKS'].nil?
require 'annotate_rb'
AnnotateRb::Core.load_rake_tasks
end

4
lib/tasks/assets.rake
View file

@ -14,7 +14,9 @@ end
if Rake::Task.task_defined?('assets:precompile')
Rake::Task['assets:precompile'].enhance do
Webpacker.manifest.refresh
Rake::Task['assets:generate_static_pages'].invoke
end
end
# We don't want vite_ruby to run yarn, we do that in a separate step
Rake::Task['vite:install_dependencies'].clear

46
lib/tasks/auto_annotate_models.rake
View file

@ -1,46 +0,0 @@
# frozen_string_literal: true
if Rails.env.development?
task :set_annotation_options do
Annotate.set_defaults(
'routes' => 'false',
'models' => 'true',
'position_in_routes' => 'before',
'position_in_class' => 'before',
'position_in_test' => 'before',
'position_in_fixture' => 'before',
'position_in_factory' => 'before',
'position_in_serializer' => 'before',
'show_foreign_keys' => 'false',
'show_indexes' => 'false',
'simple_indexes' => 'false',
'model_dir' => 'app/models',
'root_dir' => '',
'include_version' => 'false',
'require' => '',
'exclude_tests' => 'true',
'exclude_fixtures' => 'true',
'exclude_factories' => 'true',
'exclude_serializers' => 'true',
'exclude_scaffolds' => 'true',
'exclude_controllers' => 'true',
'exclude_helpers' => 'true',
'ignore_model_sub_dir' => 'false',
'ignore_columns' => nil,
'ignore_routes' => nil,
'ignore_unknown_models' => 'false',
'hide_limit_column_types' => 'integer,boolean',
'skip_on_db_migrate' => 'false',
'format_bare' => 'true',
'format_rdoc' => 'false',
'format_markdown' => 'false',
'sort' => 'false',
'force' => 'false',
'trace' => 'false',
'wrapper_open' => nil,
'wrapper_close' => nil
)
end
Annotate.load_tasks
end

10
lib/tasks/db.rake
View file

@ -62,8 +62,14 @@ namespace :db do
end
task pre_migration_check: :environment do
version = ActiveRecord::Base.connection.database_version
abort 'This version of Mastodon requires PostgreSQL 12.0 or newer. Please update PostgreSQL before updating Mastodon.' if version < 120_000
pg_version = ActiveRecord::Base.connection.database_version
abort 'This version of Mastodon requires PostgreSQL 13.0 or newer. Please update PostgreSQL before updating Mastodon.' if pg_version < 130_000
schema_version = ActiveRecord::Migrator.current_version
abort <<~MESSAGE if ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] && schema_version < 2023_09_07_150100
Zero-downtime migrations from Mastodon versions earlier than 4.2.0 are not supported.
Please update to Mastodon 4.2.x first or upgrade by stopping all services and running migrations without `SKIP_POST_DEPLOYMENT_MIGRATIONS`.
MESSAGE
end
Rake::Task['db:migrate'].enhance(['db:pre_migration_check'])

422
lib/tasks/dev.rake Normal file
View file

@ -0,0 +1,422 @@
# frozen_string_literal: true
namespace :dev do
desc 'Populate database with test data. Can be run multiple times. Should not be run in production environments'
task populate_sample_data: :environment do
# Create a valid account to showcase multiple post types
showcase_account = Account.create_with(username: 'showcase_account').find_or_create_by!(id: 10_000_000)
showcase_user = User.create_with(
account_id: showcase_account.id,
agreement: true,
password: SecureRandom.hex,
email: ENV.fetch('TEST_DATA_SHOWCASE_EMAIL', 'showcase_account@joinmastodon.org'),
confirmed_at: Time.now.utc,
approved: true,
bypass_registration_checks: true
).find_or_create_by!(id: 10_000_000)
showcase_user.mark_email_as_confirmed!
showcase_user.approve!
french_post = Status.create_with(
text: 'Ceci est un sondage public écrit en Français',
language: 'fr',
account: showcase_account,
visibility: :public,
poll_attributes: {
voters_count: 0,
account: showcase_account,
expires_at: 1.day.from_now,
options: ['ceci est un choix', 'ceci est un autre choix'],
multiple: false,
}
).find_or_create_by!(id: 10_000_000)
private_mentionless = Status.create_with(
text: 'This is a private message written in English',
language: 'en',
account: showcase_account,
visibility: :private
).find_or_create_by!(id: 10_000_001)
public_self_reply_with_cw = Status.create_with(
text: 'This is a public self-reply written in English; it has a CW and a multi-choice poll',
spoiler_text: 'poll (CW example)',
language: 'en',
account: showcase_account,
visibility: :public,
thread: french_post,
poll_attributes: {
voters_count: 0,
account: showcase_account,
expires_at: 1.day.from_now,
options: ['this is a choice', 'this is another choice', 'you can chose any number of them'],
multiple: true,
}
).find_or_create_by!(id: 10_000_002)
ProcessHashtagsService.new.call(public_self_reply_with_cw)
unlisted_self_reply_with_cw_tag_mention = Status.create_with(
text: 'This is an unlisted (Quiet Public) self-reply written in #English; it has a CW, mentions @showcase_account, and uses an emoji 🦣',
spoiler_text: 'CW example',
language: 'en',
account: showcase_account,
visibility: :unlisted,
thread: public_self_reply_with_cw
).find_or_create_by!(id: 10_000_003)
Mention.find_or_create_by!(status: unlisted_self_reply_with_cw_tag_mention, account: showcase_account)
ProcessHashtagsService.new.call(unlisted_self_reply_with_cw_tag_mention)
media_attachment = MediaAttachment.create_with(
account: showcase_account,
file: File.open('spec/fixtures/files/600x400.png'),
description: 'Mastodon logo'
).find_or_create_by!(id: 10_000_000)
status_with_media = Status.create_with(
text: "This is a public status with a picture and tags. The attached picture has an alt text\n\n#Mastodon #Logo #English #Test",
ordered_media_attachment_ids: [media_attachment.id],
account: showcase_account,
visibility: :public
).find_or_create_by!(id: 10_000_004)
media_attachment.update(status_id: status_with_media.id)
ProcessHashtagsService.new.call(status_with_media)
media_attachment = MediaAttachment.create_with(
account: showcase_account,
file: File.open('spec/fixtures/files/600x400.png'),
description: 'Mastodon logo'
).find_or_create_by!(id: 10_000_001)
status_with_sensitive_media = Status.create_with(
text: "This is the same public status with a picture and tags, but it is marked as sensitive. The attached picture has an alt text\n\n#Mastodon #Logo #English #Test",
ordered_media_attachment_ids: [media_attachment.id],
account: showcase_account,
visibility: :public,
sensitive: true,
thread: status_with_media
).find_or_create_by!(id: 10_000_005)
media_attachment.update(status_id: status_with_sensitive_media.id)
ProcessHashtagsService.new.call(status_with_sensitive_media)
media_attachment = MediaAttachment.create_with(
account: showcase_account,
file: File.open('spec/fixtures/files/600x400.png'),
description: 'Mastodon logo'
).find_or_create_by!(id: 10_000_002)
status_with_cw_media = Status.create_with(
text: "This is the same public status with a picture and tags, but it is behind a CW. The attached picture has an alt text\n\n#Mastodon #Logo #English #Test",
spoiler_text: 'Mastodon logo',
ordered_media_attachment_ids: [media_attachment.id],
account: showcase_account,
visibility: :public,
sensitive: true,
thread: status_with_sensitive_media
).find_or_create_by!(id: 10_000_006)
media_attachment.update(status_id: status_with_cw_media.id)
ProcessHashtagsService.new.call(status_with_cw_media)
media_attachment = MediaAttachment.create_with(
account: showcase_account,
file: File.open('spec/fixtures/files/boop.ogg'),
description: 'Mastodon boop'
).find_or_create_by!(id: 10_000_003)
status_with_audio = Status.create_with(
text: "This is the same public status with an audio file and tags. The attached picture has an alt text\n\n#Mastodon #English #Test",
ordered_media_attachment_ids: [media_attachment.id],
account: showcase_account,
visibility: :public,
thread: status_with_cw_media
).find_or_create_by!(id: 10_000_007)
media_attachment.update(status_id: status_with_audio.id)
ProcessHashtagsService.new.call(status_with_audio)
media_attachment = MediaAttachment.create_with(
account: showcase_account,
file: File.open('spec/fixtures/files/boop.ogg'),
description: 'Mastodon boop'
).find_or_create_by!(id: 10_000_004)
status_with_sensitive_audio = Status.create_with(
text: "This is the same public status with an audio file and tags, but it is marked as sensitive. The attached picture has an alt text\n\n#Mastodon #English #Test",
ordered_media_attachment_ids: [media_attachment.id],
account: showcase_account,
visibility: :public,
sensitive: true,
thread: status_with_audio
).find_or_create_by!(id: 10_000_008)
media_attachment.update(status_id: status_with_sensitive_audio.id)
ProcessHashtagsService.new.call(status_with_sensitive_audio)
media_attachment = MediaAttachment.create_with(
account: showcase_account,
file: File.open('spec/fixtures/files/boop.ogg'),
description: 'Mastodon boop'
).find_or_create_by!(id: 10_000_005)
status_with_cw_audio = Status.create_with(
text: "This is the same public status with an audio file and tags, but it is behind a CW. The attached picture has an alt text\n\n#Mastodon #English #Test",
spoiler_text: 'Mastodon boop',
ordered_media_attachment_ids: [media_attachment.id],
account: showcase_account,
visibility: :public,
sensitive: true,
thread: status_with_sensitive_audio
).find_or_create_by!(id: 10_000_009)
media_attachment.update(status_id: status_with_cw_audio.id)
ProcessHashtagsService.new.call(status_with_cw_audio)
media_attachments = [
MediaAttachment.create_with(
account: showcase_account,
file: File.open('spec/fixtures/files/600x400.png'),
description: 'Mastodon logo'
).find_or_create_by!(id: 10_000_006),
MediaAttachment.create_with(
account: showcase_account,
file: File.open('spec/fixtures/files/attachment.jpg')
).find_or_create_by!(id: 10_000_007),
MediaAttachment.create_with(
account: showcase_account,
file: File.open('spec/fixtures/files/avatar-high.gif'),
description: 'Walking cartoon cat'
).find_or_create_by!(id: 10_000_008),
MediaAttachment.create_with(
account: showcase_account,
file: File.open('spec/fixtures/files/text.png'),
description: 'Text saying “Hello Mastodon”'
).find_or_create_by!(id: 10_000_009),
]
status_with_multiple_attachments = Status.create_with(
text: "This is a post with multiple attachments, not all of which have a description\n\n#Mastodon #English #Test",
spoiler_text: 'multiple attachments',
ordered_media_attachment_ids: media_attachments.pluck(:id),
account: showcase_account,
visibility: :public,
sensitive: true,
thread: status_with_cw_audio
).find_or_create_by!(id: 10_000_010)
media_attachments.each { |attachment| attachment.update!(status_id: status_with_multiple_attachments.id) }
ProcessHashtagsService.new.call(status_with_multiple_attachments)
remote_account = Account.create_with(
username: 'fake.example',
domain: 'example.org',
uri: 'https://example.org/foo/bar',
url: 'https://example.org/foo/bar',
locked: true
).find_or_create_by!(id: 10_000_001)
remote_formatted_post = Status.create_with(
text: <<~HTML,
<p>This is a post with a variety of HTML in it</p>
<p>For instance, <strong>this text is bold</strong> and <b>this one as well</b>, while <del>this text is stricken through</del> and <s>this one as well</s>.</p>
<blockquote>
<p>This thing, here, is a block quote<br/>with some <strong>bold</strong> as well</p>
<ul>
<li>a list item</li>
<li>
and another with
<ul>
<li>nested</li>
<li>items!</li>
</ul>
</li>
</ul>
</blockquote>
<pre><code>// And this is some code
// with two lines of comments
</code></pre>
<p>And this is <code>inline</code> code</p>
<p>Finally, please observe this Ruby element: <ruby> 明日 <rp>(</rp><rt>Ashita</rt><rp>)</rp> </ruby></p>
HTML
account: remote_account,
uri: 'https://example.org/foo/bar/baz',
url: 'https://example.org/foo/bar/baz'
).find_or_create_by!(id: 10_000_011)
Status.create_with(account: showcase_account, reblog: remote_formatted_post).find_or_create_by!(id: 10_000_012)
unattached_quote_post = Status.create_with(
text: 'This is a quote of a post that does not exist',
account: showcase_account,
visibility: :public
).find_or_create_by!(id: 10_000_013)
Quote.create_with(
status: unattached_quote_post,
quoted_status: nil
).find_or_create_by!(id: 10_000_000)
self_quote = Status.create_with(
text: 'This is a quote of a public self-post',
account: showcase_account,
visibility: :public
).find_or_create_by!(id: 10_000_014)
Quote.create_with(
status: self_quote,
quoted_status: status_with_media,
state: :accepted
).find_or_create_by!(id: 10_000_001)
nested_self_quote = Status.create_with(
text: 'This is a quote of a public self-post which itself is a self-quote',
account: showcase_account,
visibility: :public
).find_or_create_by!(id: 10_000_015)
Quote.create_with(
status: nested_self_quote,
quoted_status: self_quote,
state: :accepted
).find_or_create_by!(id: 10_000_002)
recursive_self_quote = Status.create_with(
text: 'This is a recursive self-quote; no real reason for it to exist, but just to make sure we handle them gracefuly',
account: showcase_account,
visibility: :public
).find_or_create_by!(id: 10_000_016)
Quote.create_with(
status: recursive_self_quote,
quoted_status: recursive_self_quote,
state: :accepted
).find_or_create_by!(id: 10_000_003)
self_private_quote = Status.create_with(
text: 'This is a public post of a private self-post: the quoted post should not be visible to non-followers',
account: showcase_account,
visibility: :public
).find_or_create_by!(id: 10_000_017)
Quote.create_with(
status: self_private_quote,
quoted_status: private_mentionless,
state: :accepted
).find_or_create_by!(id: 10_000_004)
uncwed_quote_cwed = Status.create_with(
text: 'This is a quote without CW of a quoted post that has a CW',
account: showcase_account,
visibility: :public
).find_or_create_by!(id: 10_000_018)
Quote.create_with(
status: uncwed_quote_cwed,
quoted_status: public_self_reply_with_cw,
state: :accepted
).find_or_create_by!(id: 10_000_005)
cwed_quote_cwed = Status.create_with(
text: 'This is a quote with a CW of a quoted post that itself has a CW',
spoiler_text: 'Quote post with a CW',
account: showcase_account,
visibility: :public
).find_or_create_by!(id: 10_000_019)
Quote.create_with(
status: cwed_quote_cwed,
quoted_status: public_self_reply_with_cw,
state: :accepted
).find_or_create_by!(id: 10_000_006)
pending_quote_post = Status.create_with(
text: 'This quote post is pending',
account: showcase_account,
visibility: :public
).find_or_create_by!(id: 10_000_020)
Quote.create_with(
status: pending_quote_post,
quoted_status: remote_formatted_post,
activity_uri: 'https://foo/bar',
state: :pending
).find_or_create_by!(id: 10_000_007)
rejected_quote_post = Status.create_with(
text: 'This quote post is rejected',
account: showcase_account,
visibility: :public
).find_or_create_by!(id: 10_000_021)
Quote.create_with(
status: rejected_quote_post,
quoted_status: remote_formatted_post,
activity_uri: 'https://foo/foo',
state: :rejected
).find_or_create_by!(id: 10_000_008)
revoked_quote_post = Status.create_with(
text: 'This quote post is revoked',
account: showcase_account,
visibility: :public
).find_or_create_by!(id: 10_000_022)
Quote.create_with(
status: revoked_quote_post,
quoted_status: remote_formatted_post,
activity_uri: 'https://foo/baz',
state: :revoked
).find_or_create_by!(id: 10_000_009)
StatusPin.create_with(account: showcase_account, status: public_self_reply_with_cw).find_or_create_by!(id: 10_000_000)
StatusPin.create_with(account: showcase_account, status: private_mentionless).find_or_create_by!(id: 10_000_001)
showcase_account.update!(
display_name: 'Mastodon test/showcase account',
note: 'Test account to showcase many Mastodon features. Most of its posts are public, but some are private!'
)
remote_quote = Status.create_with(
text: <<~HTML,
<p>This is a self-quote of a remote formatted post</p>
<p class="quote-inline">RE: <a href="https://example.org/foo/bar/baz">https://example.org/foo/bar/baz</a></p>
HTML
account: remote_account,
uri: 'https://example.org/foo/bar/quote',
url: 'https://example.org/foo/bar/quote'
).find_or_create_by!(id: 10_000_023)
Quote.create_with(
status: remote_quote,
quoted_status: remote_formatted_post,
state: :accepted
).find_or_create_by!(id: 10_000_010)
Status.create_with(
account: showcase_account,
reblog: remote_quote
).find_or_create_by!(id: 10_000_024)
media_attachment = MediaAttachment.create_with(
account: showcase_account,
file: File.open('spec/fixtures/files/attachment.jpg')
).find_or_create_by!(id: 10_000_010)
quote_post_with_media = Status.create_with(
text: "This is a status with a picture and tags which also quotes a status with a picture.\n\n#Mastodon #Test",
ordered_media_attachment_ids: [media_attachment.id],
account: showcase_account,
visibility: :public
).find_or_create_by!(id: 10_000_025)
media_attachment.update(status_id: quote_post_with_media.id)
ProcessHashtagsService.new.call(quote_post_with_media)
Quote.create_with(
status: quote_post_with_media,
quoted_status: status_with_media,
state: :accepted
).find_or_create_by!(id: 10_000_011)
showcase_sidekick_account = Account.create_with(username: 'showcase_sidekick').find_or_create_by!(id: 10_000_002)
sidekick_user = User.create_with(
account_id: showcase_sidekick_account.id,
agreement: true,
password: SecureRandom.hex,
email: ENV.fetch('TEST_DATA_SHOWCASE_SIDEKICK_EMAIL', 'showcase_sidekick@joinmastodon.org'),
confirmed_at: Time.now.utc,
approved: true,
bypass_registration_checks: true
).find_or_create_by!(id: 10_000_001)
sidekick_user.mark_email_as_confirmed!
sidekick_user.approve!
sidekick_post = Status.create_with(
text: 'This post only exists to be quoted.',
account: showcase_sidekick_account,
visibility: :public
).find_or_create_by!(id: 10_000_026)
sidekick_quote_post = Status.create_with(
text: 'This is a quote of a different user.',
account: showcase_account,
visibility: :public
).find_or_create_by!(id: 10_000_027)
Quote.create_with(
status: sidekick_quote_post,
quoted_status: sidekick_post,
activity_uri: 'https://foo/cross-account-quote',
state: :accepted
).find_or_create_by!(id: 10_000_012)
end
end

125
lib/tasks/emojis.rake
View file

@ -42,10 +42,24 @@ def codepoints_to_unicode(codepoints)
end
end
def get_image(row, emoji_base, fallback, compressed)
path = emoji_base.join("#{row[compressed ? 'b' : 'unified'].downcase}.svg")
path = emoji_base.join("#{row[compressed ? 'c' : 'non_qualified'].downcase.sub(/^00/, '')}.svg") if !path.exist? && row[compressed ? 'c' : 'non_qualified']
if path.exist?
Vips::Image.new_from_file(path.to_s, dpi: 64)
else
fallback
end
end
def titleize(string)
string.humanize.gsub(/\b(?<!['’`()])(?!(and|the|or|with|a)\b)[a-z]/, &:capitalize)
end
namespace :emojis do
desc 'Generate a unicode to filename mapping'
task :generate do
source = 'http://www.unicode.org/Public/emoji/15.0/emoji-test.txt'
source = 'http://www.unicode.org/Public/emoji/15.1/emoji-test.txt'
codes = []
dest = Rails.root.join('app', 'javascript', 'mastodon', 'features', 'emoji', 'emoji_map.json')
@ -103,4 +117,113 @@ namespace :emojis do
gen_border map[emoji], 'white'
end
end
desc 'Generate the JSON emoji data'
task :generate_json do
data_source = 'https://raw.githubusercontent.com/iamcal/emoji-data/refs/tags/v15.1.2/emoji.json'
keyword_source = 'https://raw.githubusercontent.com/muan/emojilib/refs/tags/v3.0.12/dist/emoji-en-US.json'
data_dest = Rails.root.join('app', 'javascript', 'mastodon', 'features', 'emoji', 'emoji_data.json')
puts "Downloading emoji data from source... (#{data_source})"
res = HTTP.get(data_source).to_s
data = JSON.parse(res)
puts "Downloading keyword data from source... (#{keyword_source})"
res = HTTP.get(keyword_source).to_s
keywords = JSON.parse(res)
puts 'Generating JSON emoji data...'
emoji_data = {
compressed: true,
categories: [
{ id: 'smileys', name: 'Smileys & Emotion', emojis: [] },
{ id: 'people', name: 'People & Body', emojis: [] },
{ id: 'nature', name: 'Animals & Nature', emojis: [] },
{ id: 'foods', name: 'Food & Drink', emojis: [] },
{ id: 'activity', name: 'Activities', emojis: [] },
{ id: 'places', name: 'Travel & Places', emojis: [] },
{ id: 'objects', name: 'Objects', emojis: [] },
{ id: 'symbols', name: 'Symbols', emojis: [] },
{ id: 'flags', name: 'Flags', emojis: [] },
],
emojis: {},
aliases: {},
}
sorted = data.sort { |a, b| (a['sort_order'] || a['short_name']) - (b['sort_order'] || b['sort_name']) }
category_map = emoji_data[:categories].each_with_index.to_h { |c, i| [c[:name], i] }
sorted.each do |emoji|
emoji_keywords = keywords[codepoints_to_unicode(emoji['unified'].downcase)]
single_emoji = {
a: titleize(emoji['name']), # name
b: emoji['unified'], # unified
f: true, # has_img_twitter
k: [emoji['sheet_x'], emoji['sheet_y']], # sheet
}
single_emoji[:c] = emoji['non_qualified'] unless emoji['non_qualified'].nil? # non_qualified
single_emoji[:j] = emoji_keywords.filter { |k| k != emoji['short_name'] } if emoji_keywords.present? # keywords
single_emoji[:l] = emoji['texts'] if emoji['texts'].present? # emoticons
single_emoji[:m] = emoji['text'] if emoji['text'].present? # text
single_emoji[:skin_variations] = emoji['skin_variations'] if emoji['skin_variations'].present?
emoji_data[:emojis][emoji['short_name']] = single_emoji
emoji_data[:categories][category_map[emoji['category']]][:emojis].push(emoji['short_name']) if emoji['category'] != 'Component'
emoji['short_names'].each do |name|
emoji_data[:aliases][name] = emoji['short_name'] unless name == emoji['short_name']
end
end
smileys = emoji_data[:categories][0]
people = emoji_data[:categories][1]
smileys_and_people = { id: 'people', name: 'Smileys & People', emojis: [*smileys[:emojis][..128], *people[:emojis], *smileys[:emojis][129..]] }
emoji_data[:categories].unshift(smileys_and_people)
emoji_data[:categories] -= emoji_data[:categories][1, 2]
File.write(data_dest, JSON.generate(emoji_data))
end
desc 'Generate a spritesheet of emojis'
task :generate_emoji_sheet do
require 'vips'
src = Rails.root.join('app', 'javascript', 'mastodon', 'features', 'emoji', 'emoji_data.json')
sheet = Oj.load(File.read(src))
max = 0
sheet['emojis'].each_value do |row|
max = [max, row['k'][0], row['k'][1]].max
next if row['skin_variations'].blank?
row['skin_variations'].each_value do |variation|
max = [max, variation['sheet_x'], variation['sheet_y']].max
end
end
size = max + 1
puts 'Generating spritesheet...'
emoji_base = Rails.public_path.join('emoji')
fallback = Vips::Image.new_from_file(emoji_base.join('2753.svg').to_s, dpi: 64)
comp = Array.new(size) do
Array.new(size, 0)
end
sheet['emojis'].each_value do |row|
comp[row['k'][1]][row['k'][0]] = get_image(row, emoji_base, fallback, true)
next if row['skin_variations'].blank?
row['skin_variations'].each_value do |variation|
comp[variation['sheet_y']][variation['sheet_x']] = get_image(variation, emoji_base, fallback, false)
end
end
joined = Vips::Image.arrayjoin(comp.flatten, across: size, hspacing: 34, halign: :centre, vspacing: 34, valign: :centre)
joined.write_to_file(emoji_base.join('sheet_15_1.png').to_s, palette: true, dither: 0, Q: 100)
end
end

8
lib/tasks/mastodon.rake
View file

@ -32,7 +32,7 @@ namespace :mastodon do
prompt.say('Single user mode disables registrations and redirects the landing page to your public profile.')
env['SINGLE_USER_MODE'] = prompt.yes?('Do you want to enable single user mode?', default: false)
%w(SECRET_KEY_BASE OTP_SECRET).each do |key|
%w(SECRET_KEY_BASE).each do |key|
env[key] = SecureRandom.hex(64)
end
@ -552,7 +552,7 @@ namespace :mastodon do
password = SecureRandom.hex(16)
owner_role = UserRole.find_by(name: 'Owner')
user = User.new(email: email, password: password, confirmed_at: Time.now.utc, account_attributes: { username: username }, bypass_invite_request_check: true, role: owner_role)
user = User.new(email: email, password: password, confirmed_at: Time.now.utc, account_attributes: { username: username }, bypass_registration_checks: true, role: owner_role)
user.save(validate: false)
user.approve!
@ -604,11 +604,11 @@ namespace :mastodon do
end
def disable_log_stdout!
dev_null = Logger.new('/dev/null')
dev_null = Logger.new(File::NULL)
Rails.logger = dev_null
ActiveRecord::Base.logger = dev_null
HttpLog.configuration.logger = dev_null
HttpLog.configuration.logger = dev_null if defined?(HttpLog)
Paperclip.options[:log] = false
end

6
lib/tasks/repo.rake
View file

@ -18,7 +18,7 @@ namespace :repo do
url = "https://api.github.com/repos/#{REPOSITORY_NAME}/contributors?anon=1"
HttpLog.config.compact_log = true
HttpLog.config.compact_log = true if defined?(HttpLog)
while url.present?
response = HTTP.get(url)
@ -43,7 +43,7 @@ namespace :repo do
path = Rails.root.join('CHANGELOG.md')
tmp = Tempfile.new
HttpLog.config.compact_log = true
HttpLog.config.compact_log = true if defined?(HttpLog)
begin
File.open(path, 'r') do |file|
@ -57,7 +57,7 @@ namespace :repo do
response = nil
loop do
response = HTTP.headers('Authorization' => "token #{ENV['GITHUB_API_TOKEN']}").get("https://api.github.com/repos/#{REPOSITORY_NAME}/pulls/#{pull_request_number}")
response = HTTP.headers('Authorization' => "token #{ENV.fetch('GITHUB_API_TOKEN')}").get("https://api.github.com/repos/#{REPOSITORY_NAME}/pulls/#{pull_request_number}")
if response.code == 403
sleep_for = (response.headers['X-RateLimit-Reset'].to_i - Time.now.to_i).abs

21
lib/tasks/statistics.rake
View file

@ -1,21 +0,0 @@
# frozen_string_literal: true
task stats: 'mastodon:stats'
namespace :mastodon do
desc 'Report code statistics (KLOCs, etc)'
task :stats do
require 'rails/code_statistics'
[
['App Libraries', 'app/lib'],
%w(Presenters app/presenters),
%w(Policies app/policies),
%w(Serializers app/serializers),
%w(Services app/services),
%w(Validators app/validators),
%w(Workers app/workers),
].each do |name, dir|
STATS_DIRECTORIES << [name, dir]
end
end
end

8
lib/tasks/tests.rake
View file

@ -11,7 +11,7 @@ namespace :tests do
'3_3_0' => 2020_12_18_054746,
}.each do |release, version|
ActiveRecord::Tasks::DatabaseTasks
.migration_connection
.migration_connection_pool
.migration_context
.migrate(version)
Rake::Task["tests:migrations:populate_v#{release}"]
@ -112,17 +112,17 @@ namespace :tests do
exit(1)
end
unless Identity.where(provider: 'foo', uid: 0).count == 1
unless Identity.where(provider: 'foo', uid: 0).one?
puts 'Identities not deduplicated as expected'
exit(1)
end
unless WebauthnCredential.where(user_id: 1, nickname: 'foo').count == 1
unless WebauthnCredential.where(user_id: 1, nickname: 'foo').one?
puts 'Webauthn credentials not deduplicated as expected'
exit(1)
end
unless AccountAlias.where(account_id: 1, uri: 'https://example.com/users/foobar').count == 1
unless AccountAlias.where(account_id: 1, uri: 'https://example.com/users/foobar').one?
puts 'Account aliases not deduplicated as expected'
exit(1)
end

34
lib/tasks/webpacker.rake
View file

@ -1,34 +0,0 @@
# frozen_string_literal: true
# Disable this task as we use pnpm
require 'semantic_range'
Rake::Task['webpacker:check_yarn'].clear
namespace :webpacker do
desc 'Verifies if Yarn is installed'
task check_yarn: :environment do
begin
yarn_version = `yarn --version`.strip
raise Errno::ENOENT if yarn_version.blank?
yarn_range = '>=4 <5'
is_valid = begin
SemanticRange.satisfies?(yarn_version, yarn_range)
rescue
false
end
unless is_valid
warn "Mastodon and Webpacker requires Yarn \"#{yarn_range}\" and you are using #{yarn_version}"
warn 'Exiting!'
exit!
end
rescue Errno::ENOENT
warn 'Yarn not installed. Please see the Mastodon documentation to install the correct version.'
warn 'Exiting!'
exit!
end
end
end

152
lib/vite_ruby/sri_extensions.rb Normal file
View file

@ -0,0 +1,152 @@
# frozen_string_literal: true
module ViteRuby::ManifestIntegrityExtension
def path_and_integrity_for(name, **)
entry = lookup!(name, **)
{ path: entry.fetch('file'), integrity: entry.fetch('integrity', nil) }
end
def load_manifest
# Invalidate the name lookup cache when reloading manifest
@name_lookup_cache = nil unless dev_server_running?
super
end
def load_name_lookup_cache
Oj.load(config.build_output_dir.join('.vite/manifest-lookup.json').read)
end
# Upstream's `virtual` type is a hack, re-implement it with efficient exact name lookup
def resolve_virtual_entry(name)
return name if dev_server_running?
@name_lookup_cache ||= load_name_lookup_cache
@name_lookup_cache.fetch(name)
end
# Find a manifest entry by the *final* file name
def integrity_hash_for_file(file_name)
@integrity_cache ||= {}
@integrity_cache[file_name] ||= begin
entry = manifest.find { |_key, entry| entry['file'] == file_name }
entry[1].fetch('integrity', nil) if entry
end
end
def resolve_entries_with_integrity(*names, **options)
entries = names.map { |name| lookup!(name, **options) }
script_paths = entries.map do |entry|
{
file: entry.fetch('file'),
# TODO: Secure this so we require the integrity hash outside of dev
integrity: entry['integrity'],
}
end
imports = dev_server_running? ? [] : entries.flat_map { |entry| entry['imports'] }.compact
{
scripts: script_paths,
imports: imports.filter_map { |entry| { file: entry.fetch('file'), integrity: entry.fetch('integrity') } }.uniq,
stylesheets: dev_server_running? ? [] : (entries + imports).flat_map { |entry| entry['css'] }.compact.uniq,
}
end
# We need to override this method to not include the manifest, as in our case it is too large and will cause a JSON max nesting error rather than raising the expected exception
def missing_entry_error(name, **)
raise ViteRuby::MissingEntrypointError.new(
file_name: resolve_entry_name(name, **),
last_build: builder.last_build_metadata,
manifest: '',
config: config
)
end
end
ViteRuby::Manifest.prepend ViteRuby::ManifestIntegrityExtension
module ViteRails::TagHelpers::IntegrityExtension
def vite_javascript_tag(*names,
type: 'module',
asset_type: :javascript,
skip_preload_tags: false,
skip_style_tags: false,
crossorigin: 'anonymous',
media: 'screen',
**options)
entries = vite_manifest.resolve_entries_with_integrity(*names, type: asset_type)
''.html_safe.tap do |tags|
entries.fetch(:scripts).each do |script|
tags << javascript_include_tag(
script[:file],
integrity: script[:integrity],
crossorigin: crossorigin,
type: type,
extname: false,
**options
)
end
unless skip_preload_tags
entries.fetch(:imports).each do |import|
tags << vite_preload_tag(import[:file], integrity: import[:integrity], crossorigin: crossorigin, **options)
end
end
options[:extname] = false if Rails::VERSION::MAJOR >= 7
unless skip_style_tags
entries.fetch(:stylesheets).each do |stylesheet|
# This is for stylesheets imported from Javascript. The entry for the JS entrypoint only contains the final CSS file name, so we need to look it up in the manifest
tags << stylesheet_link_tag(
stylesheet,
integrity: vite_manifest.integrity_hash_for_file(stylesheet),
media: media,
**options
)
end
end
end
end
def vite_stylesheet_tag(*names, type: :stylesheet, **options)
''.html_safe.tap do |tags|
names.each do |name|
entry = vite_manifest.path_and_integrity_for(name, type:)
options[:extname] = false if Rails::VERSION::MAJOR >= 7
tags << stylesheet_link_tag(entry[:path], integrity: entry[:integrity], **options)
end
end
end
def vite_preload_file_tag(name,
asset_type: :javascript,
crossorigin: 'anonymous', **options)
''.html_safe.tap do |tags|
entries = vite_manifest.resolve_entries_with_integrity(name, type: asset_type)
entries.fetch(:scripts).each do |script|
tags << vite_preload_tag(script[:file], integrity: script[:integrity], crossorigin: crossorigin, **options)
end
end
rescue ViteRuby::MissingEntrypointError
# Ignore this error, it is not critical if the file is not preloaded
end
def vite_polyfills_tag(crossorigin: 'anonymous', **)
return if ViteRuby.instance.dev_server_running?
entry = vite_manifest.path_and_integrity_for('polyfills', type: :virtual)
javascript_include_tag(entry[:path], type: 'module', integrity: entry[:integrity], crossorigin: crossorigin, **)
end
end
ViteRails::TagHelpers.prepend ViteRails::TagHelpers::IntegrityExtension

27
lib/webpacker/helper_extensions.rb
View file

@ -1,27 +0,0 @@
# frozen_string_literal: true
module Webpacker::HelperExtensions
def javascript_pack_tag(name, **options)
src, integrity = current_webpacker_instance.manifest.lookup!(name, type: :javascript, with_integrity: true)
javascript_include_tag(src, options.merge(integrity: integrity))
end
def stylesheet_pack_tag(name, **options)
src, integrity = current_webpacker_instance.manifest.lookup!(name, type: :stylesheet, with_integrity: true)
stylesheet_link_tag(src, options.merge(integrity: integrity))
end
def preload_pack_asset(name, **options)
src, integrity = current_webpacker_instance.manifest.lookup!(name, with_integrity: true)
# This attribute will only work if the assets are on a different domain.
# And Webpack will (correctly) only add it in this case, so we need to conditionally set it here
# otherwise the preloaded request and the real request will have different crossorigin values
# and the preloaded file wont be loaded
crossorigin = 'anonymous' if Rails.configuration.action_controller.asset_host.present?
preload_link_tag(src, options.merge(integrity: integrity, crossorigin: crossorigin))
end
end
Webpacker::Helper.prepend(Webpacker::HelperExtensions)

17
lib/webpacker/manifest_extensions.rb
View file

@ -1,17 +0,0 @@
# frozen_string_literal: true
module Webpacker::ManifestExtensions
def lookup(name, pack_type = {})
asset = super
if pack_type[:with_integrity] && asset.respond_to?(:dig)
[asset['src'], asset['integrity']]
elsif asset.respond_to?(:dig)
asset['src']
else
asset
end
end
end
Webpacker::Manifest.prepend(Webpacker::ManifestExtensions)