Merge tag 'v4.0.5'

This commit is contained in:
Mike Barnes 2023-07-07 01:51:21 +10:00
commit b45ba2d672
52 changed files with 599 additions and 170 deletions

View file

@ -3,6 +3,45 @@ Changelog
All notable changes to this project will be documented in this file.
## [4.0.5] - 2023-07-06
### Changed
- Change OpenGraph-based embeds to allow fullscreen ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25058))
- Change profile updates to be sent to recently-mentioned servers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24852))
- Change `/api/v1/statuses/:id/history` to always return at least one item ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25510))
- Change auto-linking to allow carets in URL query params ([renchap](https://github.com/mastodon/mastodon/pull/25216))
### Removed
- Remove invalid `X-Frame-Options: ALLOWALL` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25070))
### Fixed
- Fix wrong view being displayed when a webhook fails validation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25464))
- Fix soft-deleted post cleanup scheduler overwhelming the streaming server ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25519))
- Fix incorrect pagination headers in `/api/v2/admin/accounts` ([danielmbrasil](https://github.com/mastodon/mastodon/pull/25477))
- Fix performance of streaming by parsing message JSON once ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25278), [ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25361))
- Fix CSP headers when `S3_ALIAS_HOST` includes a path component ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25273))
- Fix `tootctl accounts approve --number N` not aproving N earliest registrations ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24605))
- Fix being able to vote on your own polls ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25015))
- Fix race condition when reblogging a status ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25016))
- Fix “Authorized applications” inefficiently and incorrectly getting last use date ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25060))
- Fix multiple N+1s in ConversationsController ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25134), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25399), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25499))
- Fix user archive takeouts when using OpenStack Swift ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24431))
- Fix searching for remote content by URL not working under certain conditions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25637))
- Fix inefficiencies in indexing content for search ([VyrCossont](https://github.com/mastodon/mastodon/pull/24285), [VyrCossont](https://github.com/mastodon/mastodon/pull/24342))
### Security
- Add finer permission requirements for managing webhooks ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25463))
- Update dependencies
- Add hardening headers for user-uploaded files ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25756))
- Fix verified links possibly hiding important parts of the URL (CVE-2023-36462)
- Fix timeout handling of outbound HTTP requests (CVE-2023-36461)
- Fix arbitrary file creation through media processing (CVE-2023-36460)
- Fix possible XSS in preview cards (CVE-2023-36459)
## [4.0.4] - 2023-04-04
### Fixed

View file

@ -10,40 +10,40 @@ GIT
GEM
remote: https://rubygems.org/
specs:
actioncable (6.1.7)
actionpack (= 6.1.7)
activesupport (= 6.1.7)
actioncable (6.1.7.4)
actionpack (= 6.1.7.4)
activesupport (= 6.1.7.4)
nio4r (~> 2.0)
websocket-driver (>= 0.6.1)
actionmailbox (6.1.7)
actionpack (= 6.1.7)
activejob (= 6.1.7)
activerecord (= 6.1.7)
activestorage (= 6.1.7)
activesupport (= 6.1.7)
actionmailbox (6.1.7.4)
actionpack (= 6.1.7.4)
activejob (= 6.1.7.4)
activerecord (= 6.1.7.4)
activestorage (= 6.1.7.4)
activesupport (= 6.1.7.4)
mail (>= 2.7.1)
actionmailer (6.1.7)
actionpack (= 6.1.7)
actionview (= 6.1.7)
activejob (= 6.1.7)
activesupport (= 6.1.7)
actionmailer (6.1.7.4)
actionpack (= 6.1.7.4)
actionview (= 6.1.7.4)
activejob (= 6.1.7.4)
activesupport (= 6.1.7.4)
mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0)
actionpack (6.1.7)
actionview (= 6.1.7)
activesupport (= 6.1.7)
actionpack (6.1.7.4)
actionview (= 6.1.7.4)
activesupport (= 6.1.7.4)
rack (~> 2.0, >= 2.0.9)
rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.2.0)
actiontext (6.1.7)
actionpack (= 6.1.7)
activerecord (= 6.1.7)
activestorage (= 6.1.7)
activesupport (= 6.1.7)
actiontext (6.1.7.4)
actionpack (= 6.1.7.4)
activerecord (= 6.1.7.4)
activestorage (= 6.1.7.4)
activesupport (= 6.1.7.4)
nokogiri (>= 1.8.5)
actionview (6.1.7)
activesupport (= 6.1.7)
actionview (6.1.7.4)
activesupport (= 6.1.7.4)
builder (~> 3.1)
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
@ -54,22 +54,22 @@ GEM
case_transform (>= 0.2)
jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
active_record_query_trace (1.8)
activejob (6.1.7)
activesupport (= 6.1.7)
activejob (6.1.7.4)
activesupport (= 6.1.7.4)
globalid (>= 0.3.6)
activemodel (6.1.7)
activesupport (= 6.1.7)
activerecord (6.1.7)
activemodel (= 6.1.7)
activesupport (= 6.1.7)
activestorage (6.1.7)
actionpack (= 6.1.7)
activejob (= 6.1.7)
activerecord (= 6.1.7)
activesupport (= 6.1.7)
activemodel (6.1.7.4)
activesupport (= 6.1.7.4)
activerecord (6.1.7.4)
activemodel (= 6.1.7.4)
activesupport (= 6.1.7.4)
activestorage (6.1.7.4)
actionpack (= 6.1.7.4)
activejob (= 6.1.7.4)
activerecord (= 6.1.7.4)
activesupport (= 6.1.7.4)
marcel (~> 1.0)
mini_mime (>= 1.1.0)
activesupport (6.1.7)
activesupport (6.1.7.4)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 1.6, < 2)
minitest (>= 5.1)
@ -206,7 +206,7 @@ GEM
docile (1.3.4)
domain_name (0.5.20190701)
unf (>= 0.0.5, < 1.0.0)
doorkeeper (5.6.0)
doorkeeper (5.6.6)
railties (>= 5)
dotenv (2.8.1)
dotenv-rails (2.8.1)
@ -282,7 +282,7 @@ GEM
addressable (~> 2.7)
omniauth (>= 1.9, < 3)
openid_connect (~> 1.2)
globalid (1.0.0)
globalid (1.0.1)
activesupport (>= 5.0)
hamlit (2.13.0)
temple (>= 0.8.2)
@ -382,7 +382,7 @@ GEM
activesupport (>= 4)
railties (>= 4)
request_store (~> 1.0)
loofah (2.19.0)
loofah (2.19.1)
crass (~> 1.0.2)
nokogiri (>= 1.5.9)
mail (2.7.1)
@ -402,7 +402,7 @@ GEM
mime-types-data (~> 3.2015)
mime-types-data (3.2022.0105)
mini_mime (1.1.2)
mini_portile2 (2.8.0)
mini_portile2 (2.8.2)
minitest (5.16.3)
msgpack (1.5.4)
multi_json (1.15.0)
@ -411,9 +411,9 @@ GEM
net-scp (4.0.0.rc1)
net-ssh (>= 2.6.5, < 8.0.0)
net-ssh (7.0.1)
nio4r (2.5.8)
nokogiri (1.13.9)
mini_portile2 (~> 2.8.0)
nio4r (2.5.9)
nokogiri (1.15.2)
mini_portile2 (~> 2.8.2)
racc (~> 1.4)
nsa (0.2.8)
activesupport (>= 4.2, < 7)
@ -482,8 +482,8 @@ GEM
pundit (2.2.0)
activesupport (>= 3.0.0)
raabro (1.4.0)
racc (1.6.0)
rack (2.2.4)
racc (1.7.1)
rack (2.2.7)
rack-attack (6.6.1)
rack (>= 1.0, < 3)
rack-cors (1.1.1)
@ -498,20 +498,20 @@ GEM
rack
rack-test (2.0.2)
rack (>= 1.3)
rails (6.1.7)
actioncable (= 6.1.7)
actionmailbox (= 6.1.7)
actionmailer (= 6.1.7)
actionpack (= 6.1.7)
actiontext (= 6.1.7)
actionview (= 6.1.7)
activejob (= 6.1.7)
activemodel (= 6.1.7)
activerecord (= 6.1.7)
activestorage (= 6.1.7)
activesupport (= 6.1.7)
rails (6.1.7.4)
actioncable (= 6.1.7.4)
actionmailbox (= 6.1.7.4)
actionmailer (= 6.1.7.4)
actionpack (= 6.1.7.4)
actiontext (= 6.1.7.4)
actionview (= 6.1.7.4)
activejob (= 6.1.7.4)
activemodel (= 6.1.7.4)
activerecord (= 6.1.7.4)
activestorage (= 6.1.7.4)
activesupport (= 6.1.7.4)
bundler (>= 1.15.0)
railties (= 6.1.7)
railties (= 6.1.7.4)
sprockets-rails (>= 2.0.0)
rails-controller-testing (1.0.5)
actionpack (>= 5.0.1.rc1)
@ -520,16 +520,16 @@ GEM
rails-dom-testing (2.0.3)
activesupport (>= 4.2.0)
nokogiri (>= 1.6)
rails-html-sanitizer (1.4.3)
loofah (~> 2.3)
rails-html-sanitizer (1.4.4)
loofah (~> 2.19, >= 2.19.1)
rails-i18n (6.0.0)
i18n (>= 0.7, < 2)
railties (>= 6.0.0, < 7)
rails-settings-cached (0.6.6)
rails (>= 4.2.0)
railties (6.1.7)
actionpack (= 6.1.7)
activesupport (= 6.1.7)
railties (6.1.7.4)
actionpack (= 6.1.7.4)
activesupport (= 6.1.7.4)
method_source
rake (>= 12.2)
thor (~> 1.0)
@ -602,7 +602,7 @@ GEM
fugit (~> 1.1, >= 1.1.6)
safety_net_attestation (0.4.0)
jwt (~> 2.0)
sanitize (6.0.0)
sanitize (6.0.1)
crass (~> 1.0.2)
nokogiri (>= 1.12.0)
scenic (1.6.0)
@ -661,7 +661,7 @@ GEM
unicode-display_width (>= 1.1.1, < 3)
terrapin (0.6.0)
climate_control (>= 0.0.3, < 1.0)
thor (1.2.1)
thor (1.2.2)
tilt (2.0.11)
tpm-key_attestation (0.11.0)
bindata (~> 2.4)
@ -680,7 +680,7 @@ GEM
twitter-text (3.1.0)
idn-ruby
unf (~> 0.1.0)
tzinfo (2.0.5)
tzinfo (2.0.6)
concurrent-ruby (~> 1.0)
tzinfo-data (1.2022.4)
tzinfo (>= 1.0.0)
@ -725,7 +725,7 @@ GEM
xorcist (1.1.3)
xpath (3.2.0)
nokogiri (~> 1.8)
zeitwerk (2.6.0)
zeitwerk (2.6.8)
PLATFORMS
ruby

View file

@ -20,6 +20,7 @@ module Admin
authorize :webhook, :create?
@webhook = Webhook.new(resource_params)
@webhook.current_account = current_account
if @webhook.save
redirect_to admin_webhook_path(@webhook)
@ -39,10 +40,12 @@ module Admin
def update
authorize @webhook, :update?
@webhook.current_account = current_account
if @webhook.update(resource_params)
redirect_to admin_webhook_path(@webhook)
else
render :show
render :edit
end
end

View file

@ -11,7 +11,7 @@ class Api::V1::ConversationsController < Api::BaseController
def index
@conversations = paginated_conversations
render json: @conversations, each_serializer: REST::ConversationSerializer
render json: @conversations, each_serializer: REST::ConversationSerializer, relationships: StatusRelationshipsPresenter.new(@conversations.map(&:last_status), current_user&.account_id)
end
def read
@ -32,6 +32,19 @@ class Api::V1::ConversationsController < Api::BaseController
def paginated_conversations
AccountConversation.where(account: current_account)
.includes(
account: :account_stat,
last_status: [
:media_attachments,
:preview_cards,
:status_stat,
:tags,
{
active_mentions: [account: :account_stat],
account: :account_stat,
},
]
)
.to_a_paginated_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id))
end

View file

@ -7,11 +7,15 @@ class Api::V1::Statuses::HistoriesController < Api::BaseController
before_action :set_status
def show
render json: @status.edits.includes(:account, status: [:account]), each_serializer: REST::StatusEditSerializer
render json: status_edits, each_serializer: REST::StatusEditSerializer
end
private
def status_edits
@status.edits.includes(:account, status: [:account]).to_a.presence || [@status.build_snapshot(at_time: @status.edited_at || @status.created_at)]
end
def set_status
@status = Status.find(params[:status_id])
authorize @status, :show?

View file

@ -2,6 +2,8 @@
class Api::V1::Statuses::ReblogsController < Api::BaseController
include Authorization
include Redisable
include Lockable
before_action -> { doorkeeper_authorize! :write, :'write:statuses' }
before_action :require_user!
@ -10,7 +12,9 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController
override_rate_limit_headers :create, family: :statuses
def create
with_lock("reblog:#{current_account.id}:#{@reblog.id}") do
@status = ReblogService.new.call(current_account, @reblog, reblog_params)
end
render json: @status, serializer: REST::StatusSerializer
end

View file

@ -18,6 +18,14 @@ class Api::V2::Admin::AccountsController < Api::V1::Admin::AccountsController
private
def next_path
api_v2_admin_accounts_url(pagination_params(max_id: pagination_max_id)) if records_continue?
end
def prev_path
api_v2_admin_accounts_url(pagination_params(min_id: pagination_since_id)) unless @accounts.empty?
end
def filtered_accounts
AccountFilter.new(translated_filter_params).results
end

View file

@ -13,7 +13,7 @@ class BackupsController < ApplicationController
when :s3
redirect_to @backup.dump.expiring_url(10)
when :fog
if Paperclip::Attachment.default_options.dig(:storage, :fog_credentials, :openstack_temp_url_key).present?
if Paperclip::Attachment.default_options.dig(:fog_credentials, :openstack_temp_url_key).present?
redirect_to @backup.dump.expiring_url(Time.now.utc + 10)
else
redirect_to full_asset_url(@backup.dump.url)

View file

@ -46,6 +46,6 @@ class MediaController < ApplicationController
end
def allow_iframing
response.headers['X-Frame-Options'] = 'ALLOWALL'
response.headers.delete('X-Frame-Options')
end
end

View file

@ -8,6 +8,8 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
before_action :require_not_suspended!, only: :destroy
before_action :set_body_classes
before_action :set_last_used_at_by_app, only: :index, unless: -> { request.format == :json }
skip_before_action :require_functional!
include Localized
@ -30,4 +32,14 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
def require_not_suspended!
forbidden if current_account.suspended?
end
def set_last_used_at_by_app
@last_used_at_by_app = Doorkeeper::AccessToken
.select('DISTINCT ON (application_id) application_id, last_used_at')
.where(resource_owner_id: current_resource_owner.id)
.where.not(last_used_at: nil)
.order(application_id: :desc, last_used_at: :desc)
.pluck(:application_id, :last_used_at)
.to_h
end
end

View file

@ -43,7 +43,7 @@ class StatusesController < ApplicationController
return not_found if @status.hidden? || @status.reblog?
expires_in 180, public: true
response.headers['X-Frame-Options'] = 'ALLOWALL'
response.headers.delete('X-Frame-Options')
render layout: 'embedded'
end

View file

@ -49,6 +49,10 @@ module FormattingHelper
end
def account_field_value_format(field, with_rel_me: true)
if field.verified? && !field.account.local?
TextFormatter.shortened_link(field.value_for_verification)
else
html_aware_format(field.value, field.account.local?, with_rel_me: with_rel_me, with_domains: true, multiline: false)
end
end
end

View file

@ -6,7 +6,7 @@ class AccountReachFinder
end
def inboxes
(followers_inboxes + reporters_inboxes + relay_inboxes).uniq
(followers_inboxes + reporters_inboxes + recently_mentioned_inboxes + relay_inboxes).uniq
end
private
@ -19,6 +19,13 @@ class AccountReachFinder
Account.where(id: @account.targeted_reports.select(:account_id)).inboxes
end
def recently_mentioned_inboxes
cutoff_id = Mastodon::Snowflake.id_at(2.days.ago, with_random: false)
recent_statuses = @account.statuses.recent.where(id: cutoff_id...).limit(200)
Account.joins(:mentions).where(mentions: { status: recent_statuses }).inboxes.take(2000)
end
def relay_inboxes
Relay.enabled.pluck(:inbox_url)
end

View file

@ -9,10 +9,6 @@ module ApplicationExtension
validates :redirect_uri, length: { maximum: 2_000 }
end
def most_recently_used_access_token
@most_recently_used_access_token ||= access_tokens.where.not(last_used_at: nil).order(last_used_at: :desc).first
end
def confirmation_redirect_uri
redirect_uri.lines.first.strip
end

View file

@ -140,7 +140,7 @@ class LinkDetailsExtractor
end
def html
player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowfullscreen: 'true', allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
end
def width

View file

@ -7,11 +7,48 @@ require 'resolv'
# Monkey-patch the HTTP.rb timeout class to avoid using a timeout block
# around the Socket#open method, since we use our own timeout blocks inside
# that method
#
# Also changes how the read timeout behaves so that it is cumulative (closer
# to HTTP::Timeout::Global, but still having distinct timeouts for other
# operation types)
class HTTP::Timeout::PerOperation
def connect(socket_class, host, port, nodelay = false)
@socket = socket_class.open(host, port)
@socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
end
# Reset deadline when the connection is re-used for different requests
def reset_counter
@deadline = nil
end
# Read data from the socket
def readpartial(size, buffer = nil)
@deadline ||= Process.clock_gettime(Process::CLOCK_MONOTONIC) + @read_timeout
timeout = false
loop do
result = @socket.read_nonblock(size, buffer, exception: false)
return :eof if result.nil?
remaining_time = @deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
raise HTTP::TimeoutError, "Read timed out after #{@read_timeout} seconds" if timeout || remaining_time <= 0
return result if result != :wait_readable
# marking the socket for timeout. Why is this not being raised immediately?
# it seems there is some race-condition on the network level between calling
# #read_nonblock and #wait_readable, in which #read_nonblock signalizes waiting
# for reads, and when waiting for x seconds, it returns nil suddenly without completing
# the x seconds. In a normal case this would be a timeout on wait/read, but it can
# also mean that the socket has been closed by the server. Therefore we "mark" the
# socket for timeout and try to read more bytes. If it returns :eof, it's all good, no
# timeout. Else, the first timeout was a proper timeout.
# This hack has to be done because io/wait#wait_readable doesn't provide a value for when
# the socket is closed by the server, and HTTP::Parser doesn't provide the limit for the chunks.
timeout = true unless @socket.to_io.wait_readable(remaining_time)
end
end
end
class Request

View file

@ -48,6 +48,26 @@ class TextFormatter
html.html_safe # rubocop:disable Rails/OutputSafety
end
class << self
include ERB::Util
def shortened_link(url, rel_me: false)
url = Addressable::URI.parse(url).to_s
rel = rel_me ? (DEFAULT_REL + %w(me)) : DEFAULT_REL
prefix = url.match(URL_PREFIX_REGEX).to_s
display_url = url[prefix.length, 30]
suffix = url[prefix.length + 30..-1]
cutoff = url[prefix.length..-1].length > 30
<<~HTML.squish
<a href="#{h(url)}" target="_blank" rel="#{rel.join(' ')}"><span class="invisible">#{h(prefix)}</span><span class="#{cutoff ? 'ellipsis' : ''}">#{h(display_url)}</span><span class="invisible">#{h(suffix)}</span></a>
HTML
rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
h(url)
end
end
private
def rewrite
@ -70,19 +90,7 @@ class TextFormatter
end
def link_to_url(entity)
url = Addressable::URI.parse(entity[:url]).to_s
rel = with_rel_me? ? (DEFAULT_REL + %w(me)) : DEFAULT_REL
prefix = url.match(URL_PREFIX_REGEX).to_s
display_url = url[prefix.length, 30]
suffix = url[prefix.length + 30..-1]
cutoff = url[prefix.length..-1].length > 30
<<~HTML.squish
<a href="#{h(url)}" target="_blank" rel="#{rel.join(' ')}"><span class="invisible">#{h(prefix)}</span><span class="#{cutoff ? 'ellipsis' : ''}">#{h(display_url)}</span><span class="invisible">#{h(suffix)}</span></a>
HTML
rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
h(entity[:url])
TextFormatter.shortened_link(entity[:url], rel_me: with_rel_me?)
end
def link_to_hashtag(entity)

View file

@ -16,29 +16,28 @@
class AccountConversation < ApplicationRecord
include Redisable
attr_writer :participant_accounts
before_validation :set_last_status
after_commit :push_to_streaming_api
belongs_to :account
belongs_to :conversation
belongs_to :last_status, class_name: 'Status'
before_validation :set_last_status
def participant_account_ids=(arr)
self[:participant_account_ids] = arr.sort
@participant_accounts = nil
end
def participant_accounts
if participant_account_ids.empty?
[account]
else
participants = Account.where(id: participant_account_ids)
participants.empty? ? [account] : participants
end
@participant_accounts ||= Account.where(id: participant_account_ids).to_a
@participant_accounts.presence || [account]
end
class << self
def to_a_paginated_by_id(limit, options = {})
array = begin
if options[:min_id]
paginate_by_min_id(limit, options[:min_id], options[:max_id]).reverse
else
@ -46,6 +45,17 @@ class AccountConversation < ApplicationRecord
end
end
# Preload participants
participant_ids = array.flat_map(&:participant_account_ids)
accounts_by_id = Account.where(id: participant_ids).index_by(&:id)
array.each do |conversation|
conversation.participant_accounts = conversation.participant_account_ids.filter_map { |id| accounts_by_id[id] }
end
array
end
def paginate_by_min_id(limit, min_id = nil, max_id = nil)
query = order(arel_table[:last_status_id].asc).limit(limit)
query = query.where(arel_table[:last_status_id].gt(min_id)) if min_id.present?

View file

@ -22,15 +22,14 @@ module Attachmentable
included do
def self.has_attached_file(name, options = {}) # rubocop:disable Naming/PredicateName
options = { validate_media_type: false }.merge(options)
super(name, options)
send(:"before_#{name}_post_process") do
send(:"before_#{name}_validate") do
attachment = send(name)
check_image_dimension(attachment)
set_file_content_type(attachment)
obfuscate_file_name(attachment)
set_file_extension(attachment)
Paperclip::Validators::MediaTypeSpoofDetectionValidator.new(attributes: [name]).validate(self)
end
end
end

View file

@ -12,7 +12,7 @@
#
class Identity < ApplicationRecord
belongs_to :user, dependent: :destroy
belongs_to :user
validates :uid, presence: true, uniqueness: { scope: :provider }
validates :provider, presence: true

View file

@ -19,6 +19,8 @@ class Webhook < ApplicationRecord
report.created
).freeze
attr_writer :current_account
scope :enabled, -> { where(enabled: true) }
validates :url, presence: true, url: true
@ -26,6 +28,7 @@ class Webhook < ApplicationRecord
validates :events, presence: true
validate :validate_events
validate :validate_permissions
before_validation :strip_events
before_validation :generate_secret
@ -42,12 +45,29 @@ class Webhook < ApplicationRecord
update!(enabled: false)
end
def required_permissions
events.map { |event| Webhook.permission_for_event(event) }
end
def self.permission_for_event(event)
case event
when 'account.approved', 'account.created', 'account.updated'
:manage_users
when 'report.created'
:manage_reports
end
end
private
def validate_events
errors.add(:events, :invalid) if events.any? { |e| !EVENTS.include?(e) }
end
def validate_permissions
errors.add(:events, :invalid_permissions) if defined?(@current_account) && required_permissions.any? { |permission| !@current_account.user_role.can?(permission) }
end
def strip_events
self.events = events.map { |str| str.strip.presence }.compact if events.present?
end

View file

@ -14,7 +14,7 @@ class WebhookPolicy < ApplicationPolicy
end
def update?
role.can?(:manage_webhooks)
role.can?(:manage_webhooks) && record.required_permissions.all? { |permission| role.can?(permission) }
end
def enable?
@ -30,6 +30,6 @@ class WebhookPolicy < ApplicationPolicy
end
def destroy?
role.can?(:manage_webhooks)
role.can?(:manage_webhooks) && record.required_permissions.all? { |permission| role.can?(permission) }
end
end

View file

@ -11,4 +11,8 @@ class REST::PreviewCardSerializer < ActiveModel::Serializer
def image
object.image? ? full_asset_url(object.image.url(:original)) : nil
end
def html
Sanitize.fragment(object.html, Sanitize::Config::MASTODON_OEMBED)
end
end

View file

@ -12,6 +12,7 @@ class RemoveStatusService < BaseService
# @option [Boolean] :immediate
# @option [Boolean] :preserve
# @option [Boolean] :original_removed
# @option [Boolean] :skip_streaming
def call(status, **options)
@payload = Oj.dump(event: :delete, payload: status.id.to_s)
@status = status
@ -52,6 +53,9 @@ class RemoveStatusService < BaseService
private
# The following FeedManager calls all do not result in redis publishes for
# streaming, as the `:update` option is false
def remove_from_self
FeedManager.instance.unpush_from_home(@account, @status)
end
@ -75,6 +79,8 @@ class RemoveStatusService < BaseService
# followers. Here we send a delete to actively mentioned accounts
# that may not follow the account
return if skip_streaming?
@status.active_mentions.find_each do |mention|
redis.publish("timeline:#{mention.account_id}", @payload)
end
@ -103,7 +109,7 @@ class RemoveStatusService < BaseService
# without us being able to do all the fancy stuff
@status.reblogs.rewhere(deleted_at: [nil, @status.deleted_at]).includes(:account).reorder(nil).find_each do |reblog|
RemoveStatusService.new.call(reblog, original_removed: true)
RemoveStatusService.new.call(reblog, original_removed: true, skip_streaming: skip_streaming?)
end
end
@ -114,6 +120,8 @@ class RemoveStatusService < BaseService
return unless @status.public_visibility?
return if skip_streaming?
@status.tags.map(&:name).each do |hashtag|
redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}", @payload)
redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}:local", @payload) if @status.local?
@ -123,6 +131,8 @@ class RemoveStatusService < BaseService
def remove_from_public
return unless @status.public_visibility?
return if skip_streaming?
redis.publish('timeline:public', @payload)
redis.publish(@status.local? ? 'timeline:public:local' : 'timeline:public:remote', @payload)
end
@ -130,6 +140,8 @@ class RemoveStatusService < BaseService
def remove_from_media
return unless @status.public_visibility?
return if skip_streaming?
redis.publish('timeline:public:media', @payload)
redis.publish(@status.local? ? 'timeline:public:local:media' : 'timeline:public:remote:media', @payload)
end
@ -143,4 +155,8 @@ class RemoveStatusService < BaseService
def permanently?
@options[:immediate] || !(@options[:preserve] || @status.reported?)
end
def skip_streaming?
!!@options[:skip_streaming]
end
end

View file

@ -89,13 +89,28 @@ class ResolveURLService < BaseService
def process_local_url
recognized_params = Rails.application.routes.recognize_path(@url)
case recognized_params[:controller]
when 'statuses'
return unless recognized_params[:action] == 'show'
if recognized_params[:controller] == 'statuses'
status = Status.find_by(id: recognized_params[:id])
check_local_status(status)
elsif recognized_params[:controller] == 'accounts'
when 'accounts'
return unless recognized_params[:action] == 'show'
Account.find_local(recognized_params[:username])
when 'home'
return unless recognized_params[:action] == 'index' && recognized_params[:username_with_domain].present?
if recognized_params[:any]&.match?(/\A[0-9]+\Z/)
status = Status.find_by(id: recognized_params[:any])
check_local_status(status)
elsif recognized_params[:any].blank?
username, domain = recognized_params[:username_with_domain].gsub(/\A@/, '').split('@')
return unless username.present? && domain.present?
Account.find_remote(username, domain)
end
end
end

View file

@ -3,8 +3,8 @@
class VoteValidator < ActiveModel::Validator
def validate(vote)
vote.errors.add(:base, I18n.t('polls.errors.expired')) if vote.poll.expired?
vote.errors.add(:base, I18n.t('polls.errors.invalid_choice')) if invalid_choice?(vote)
vote.errors.add(:base, I18n.t('polls.errors.self_vote')) if self_vote?(vote)
if vote.poll.multiple? && vote.poll.votes.where(account: vote.account, choice: vote.choice).exists?
vote.errors.add(:base, I18n.t('polls.errors.already_voted'))
@ -18,4 +18,8 @@ class VoteValidator < ActiveModel::Validator
def invalid_choice?(vote)
vote.choice.negative? || vote.choice >= vote.poll.options.size
end
def self_vote?(vote)
vote.account_id == vote.poll.account_id
end
end

View file

@ -5,7 +5,7 @@
= f.input :url, wrapper: :with_block_label, input_html: { placeholder: 'https://' }
.fields-group
= f.input :events, collection: Webhook::EVENTS, wrapper: :with_block_label, include_blank: false, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li'
= f.input :events, collection: Webhook::EVENTS, wrapper: :with_block_label, include_blank: false, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li', disabled: Webhook::EVENTS.filter { |event| !current_user.role.can?(Webhook.permission_for_event(event)) }
.actions
= f.button :button, @webhook.new_record? ? t('admin.webhooks.add_new') : t('generic.save_changes'), type: :submit

View file

@ -18,8 +18,8 @@
.announcements-list__item__action-bar
.announcements-list__item__meta
- if application.most_recently_used_access_token
= t('doorkeeper.authorized_applications.index.last_used_at', date: l(application.most_recently_used_access_token.last_used_at.to_date))
- if @last_used_at_by_app[application.id]
= t('doorkeeper.authorized_applications.index.last_used_at', date: l(@last_used_at_by_app[application.id].to_date))
- else
= t('doorkeeper.authorized_applications.index.never_used')

View file

@ -6,17 +6,19 @@ class Scheduler::IndexingScheduler
sidekiq_options retry: 0
IMPORT_BATCH_SIZE = 1000
SCAN_BATCH_SIZE = 10 * IMPORT_BATCH_SIZE
def perform
return unless Chewy.enabled?
indexes.each do |type|
with_redis do |redis|
ids = redis.smembers("chewy:queue:#{type.name}")
redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids|
type.import!(ids)
redis.pipelined do |pipeline|
ids.each { |id| pipeline.srem("chewy:queue:#{type.name}", id) }
pipeline.srem("chewy:queue:#{type.name}", ids)
end
end
end
end

View file

@ -24,7 +24,7 @@ class Scheduler::UserCleanupScheduler
def clean_discarded_statuses!
Status.unscoped.discarded.where('deleted_at <= ?', 30.days.ago).find_in_batches do |statuses|
RemovalWorker.push_bulk(statuses) do |status|
[status.id, { 'immediate' => true }]
[status.id, { 'immediate' => true, 'skip_streaming' => true }]
end
end
end

View file

@ -29,6 +29,7 @@ require_relative '../lib/paperclip/url_generator_extensions'
require_relative '../lib/paperclip/attachment_extensions'
require_relative '../lib/paperclip/lazy_thumbnail'
require_relative '../lib/paperclip/gif_transcoder'
require_relative '../lib/paperclip/media_type_spoof_detector_extensions'
require_relative '../lib/paperclip/transcoder'
require_relative '../lib/paperclip/type_corrector'
require_relative '../lib/paperclip/response_with_limit_adapter'
@ -160,6 +161,10 @@ module Mastodon
end
end
config.public_file_server.headers = {
'X-Content-Type-Options' => 'nosniff',
}
# config.paths.add File.join('app', 'api'), glob: File.join('**', '*.rb')
# config.autoload_paths += Dir[Rails.root.join('app', 'api', '*')]

View file

@ -0,0 +1,27 @@
<policymap>
<!-- Set some basic system resource limits -->
<policy domain="resource" name="time" value="60" />
<policy domain="module" rights="none" pattern="URL" />
<policy domain="filter" rights="none" pattern="*" />
<!--
Ideally, we would restrict ImageMagick to only accessing its own
disk-backed pixel cache as well as Mastodon-created Tempfiles.
However, those paths depend on the operating system and environment
variables, so they can only be known at runtime.
Furthermore, those paths are not necessarily shared across Mastodon
processes, so even creating a policy.xml at runtime is impractical.
For the time being, only disable indirect reads.
-->
<policy domain="path" rights="none" pattern="@*" />
<!-- Disallow any coder by default, and only enable ones required by Mastodon -->
<policy domain="coder" rights="none" pattern="*" />
<policy domain="coder" rights="read | write" pattern="{PNG,JPEG,GIF,HEIC,WEBP}" />
<policy domain="coder" rights="write" pattern="{HISTOGRAM,RGB,INFO}" />
</policymap>

View file

@ -3,7 +3,7 @@
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy
def host_to_url(str)
"http#{Rails.configuration.x.use_https ? 's' : ''}://#{str}" unless str.blank?
"http#{Rails.configuration.x.use_https ? 's' : ''}://#{str}".split('/').first if str.present?
end
base_host = Rails.configuration.x.web_domain

View file

@ -155,3 +155,10 @@ unless defined?(Seahorse)
end
end
end
# Set our ImageMagick security policy, but allow admins to override it
ENV['MAGICK_CONFIGURE_PATH'] = begin
imagemagick_config_paths = ENV.fetch('MAGICK_CONFIGURE_PATH', '').split(File::PATH_SEPARATOR)
imagemagick_config_paths << Rails.root.join('config', 'imagemagick').expand_path.to_s
imagemagick_config_paths.join(File::PATH_SEPARATOR)
end

View file

@ -25,7 +25,7 @@ module Twitter::TwitterText
\)
/iox
UCHARS = '\u{A0}-\u{D7FF}\u{F900}-\u{FDCF}\u{FDF0}-\u{FFEF}\u{10000}-\u{1FFFD}\u{20000}-\u{2FFFD}\u{30000}-\u{3FFFD}\u{40000}-\u{4FFFD}\u{50000}-\u{5FFFD}\u{60000}-\u{6FFFD}\u{70000}-\u{7FFFD}\u{80000}-\u{8FFFD}\u{90000}-\u{9FFFD}\u{A0000}-\u{AFFFD}\u{B0000}-\u{BFFFD}\u{C0000}-\u{CFFFD}\u{D0000}-\u{DFFFD}\u{E1000}-\u{EFFFD}\u{E000}-\u{F8FF}\u{F0000}-\u{FFFFD}\u{100000}-\u{10FFFD}'
REGEXEN[:valid_url_query_chars] = /[a-z0-9!?\*'\(\);:&=\+\$\/%#\[\]\-_\.,~|@#{UCHARS}]/iou
REGEXEN[:valid_url_query_chars] = /[a-z0-9!?\*'\(\);:&=\+\$\/%#\[\]\-_\.,~|@\^#{UCHARS}]/iou
REGEXEN[:valid_url_query_ending_chars] = /[a-z0-9_&=#\/\-#{UCHARS}]/iou
REGEXEN[:valid_url_path] = /(?:
(?:

View file

@ -53,3 +53,7 @@ en:
position:
elevated: cannot be higher than your current role
own_role: cannot be changed with your current role
webhook:
attributes:
events:
invalid_permissions: cannot include events you don't have the rights to

View file

@ -1332,6 +1332,7 @@ en:
expired: The poll has already ended
invalid_choice: The chosen vote option does not exist
over_character_limit: cannot be longer than %{max} characters each
self_vote: You cannot vote in your own polls
too_few_options: must have more than one item
too_many_options: can't contain more than %{max} items
preferences:

2
dist/nginx.conf vendored
View file

@ -109,6 +109,8 @@ server {
location ~ ^/system/ {
add_header Cache-Control "public, max-age=2419200, immutable";
add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
add_header X-Content-Type-Options nosniff;
add_header Content-Security-Policy "default-src 'none'; form-action 'none'";
try_files $uri =404;
}

View file

@ -513,7 +513,7 @@ module Mastodon
User.pending.find_each(&:approve!)
say('OK', :green)
elsif options[:number]
User.pending.limit(options[:number]).each(&:approve!)
User.pending.order(created_at: :asc).limit(options[:number]).each(&:approve!)
say('OK', :green)
elsif username.present?
account = Account.find_local(username)

View file

@ -13,7 +13,7 @@ module Mastodon
end
def patch
4
5
end
def flags

View file

@ -0,0 +1,22 @@
# frozen_string_literal: true
module Paperclip
module MediaTypeSpoofDetectorExtensions
def calculated_content_type
return @calculated_content_type if defined?(@calculated_content_type)
@calculated_content_type = type_from_file_command.chomp
# The `file` command fails to recognize some MP3 files as such
@calculated_content_type = type_from_marcel if @calculated_content_type == 'application/octet-stream' && type_from_marcel == 'audio/mpeg'
@calculated_content_type
end
def type_from_marcel
@type_from_marcel ||= Marcel::MimeType.for Pathname.new(@file.path),
name: @file.path
end
end
end
Paperclip::MediaTypeSpoofDetector.prepend(Paperclip::MediaTypeSpoofDetectorExtensions)

View file

@ -19,10 +19,7 @@ module Paperclip
def make
metadata = VideoMetadataExtractor.new(@file.path)
unless metadata.valid?
Paperclip.log("Unsupported file #{@file.path}")
return File.open(@file.path)
end
raise Paperclip::Error, "Error while transcoding #{@file.path}: unsupported file" unless metadata.valid?
update_attachment_type(metadata)
update_options_from_metadata(metadata)

View file

@ -94,26 +94,26 @@ class Sanitize
]
)
MASTODON_OEMBED ||= freeze_config merge(
RELAXED,
elements: RELAXED[:elements] + %w(audio embed iframe source video),
MASTODON_OEMBED ||= freeze_config(
elements: %w(audio embed iframe source video),
attributes: merge(
RELAXED[:attributes],
attributes: {
'audio' => %w(controls),
'embed' => %w(height src type width),
'iframe' => %w(allowfullscreen frameborder height scrolling src width),
'source' => %w(src type),
'video' => %w(controls height loop width),
'div' => [:data]
),
},
protocols: merge(
RELAXED[:protocols],
protocols: {
'embed' => { 'src' => HTTP_PROTOCOLS },
'iframe' => { 'src' => HTTP_PROTOCOLS },
'source' => { 'src' => HTTP_PROTOCOLS }
)
'source' => { 'src' => HTTP_PROTOCOLS },
},
add_attributes: {
'iframe' => { 'sandbox' => 'allow-scripts allow-same-origin allow-popups allow-popups-to-escape-sandbox allow-forms' },
}
)
end
end

View file

@ -16,6 +16,7 @@ RSpec.describe Api::V1::ConversationsController, type: :controller do
before do
PostStatusService.new.call(other.account, text: 'Hey @alice', visibility: 'direct')
PostStatusService.new.call(user.account, text: 'Hey, nobody here', visibility: 'direct')
end
it 'returns http success' do
@ -31,7 +32,26 @@ RSpec.describe Api::V1::ConversationsController, type: :controller do
it 'returns conversations' do
get :index
json = body_as_json
expect(json.size).to eq 1
expect(json.size).to eq 2
expect(json[0][:accounts].size).to eq 1
end
context 'with since_id' do
context 'when requesting old posts' do
it 'returns conversations' do
get :index, params: { since_id: Mastodon::Snowflake.id_at(1.hour.ago, with_random: false) }
json = body_as_json
expect(json.size).to eq 2
end
end
context 'when requesting posts in the future' do
it 'returns no conversation' do
get :index, params: { since_id: Mastodon::Snowflake.id_at(1.hour.from_now, with_random: false) }
json = body_as_json
expect(json.size).to eq 0
end
end
end
end
end

View file

@ -23,6 +23,7 @@ describe Api::V1::Statuses::HistoriesController do
it 'returns http success' do
expect(response).to have_http_status(200)
expect(body_as_json.size).to_not be 0
end
end
end

View file

@ -69,5 +69,13 @@ RSpec.describe Api::V2::Admin::AccountsController, type: :controller do
end
end
end
context 'with limit param' do
let(:params) { { limit: 1 } }
it 'sets the correct pagination headers' do
expect(response.headers['Link'].find_link(%w(rel next)).href).to eq api_v2_admin_accounts_url(limit: 1, max_id: admin_account.id)
end
end
end
end

BIN
spec/fixtures/files/boop.mp3 vendored Normal file

Binary file not shown.

View file

@ -0,0 +1,53 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe AccountReachFinder do
let(:account) { Fabricate(:account) }
let(:follower1) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-1') }
let(:follower2) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-2') }
let(:follower3) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://foo.bar/users/a/inbox', shared_inbox_url: 'https://foo.bar/inbox') }
let(:mentioned1) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://foo.bar/users/b/inbox', shared_inbox_url: 'https://foo.bar/inbox') }
let(:mentioned2) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-3') }
let(:mentioned3) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-4') }
let(:unrelated_account) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/unrelated-inbox') }
before do
follower1.follow!(account)
follower2.follow!(account)
follower3.follow!(account)
Fabricate(:status, account: account).tap do |status|
status.mentions << Mention.new(account: follower1)
status.mentions << Mention.new(account: mentioned1)
end
Fabricate(:status, account: account)
Fabricate(:status, account: account).tap do |status|
status.mentions << Mention.new(account: mentioned2)
status.mentions << Mention.new(account: mentioned3)
end
Fabricate(:status).tap do |status|
status.mentions << Mention.new(account: unrelated_account)
end
end
describe '#inboxes' do
it 'includes the preferred inbox URL of followers' do
expect(described_class.new(account).inboxes).to include(*[follower1, follower2, follower3].map(&:preferred_inbox_url))
end
it 'includes the preferred inbox URL of recently-mentioned accounts' do
expect(described_class.new(account).inboxes).to include(*[mentioned1, mentioned2, mentioned3].map(&:preferred_inbox_url))
end
it 'does not include the inbox of unrelated users' do
expect(described_class.new(account).inboxes).to_not include(unrelated_account.preferred_inbox_url)
end
end
end

View file

@ -150,6 +150,26 @@ RSpec.describe MediaAttachment, type: :model do
end
end
describe 'mp3 with large cover art' do
let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture('boop.mp3')) }
it 'detects it as an audio file' do
expect(media.type).to eq 'audio'
end
it 'sets meta for the duration' do
expect(media.file.meta['original']['duration']).to be_within(0.05).of(0.235102)
end
it 'extracts thumbnail' do
expect(media.thumbnail.present?).to be true
end
it 'gives the file a random name' do
expect(media.file_file_name).to_not eq 'boop.mp3'
end
end
describe 'jpeg' do
let(:media) { MediaAttachment.create(account: Fabricate(:account), file: attachment_fixture('attachment.jpg')) }

View file

@ -10,6 +10,7 @@ RSpec.describe FetchLinkCardService, type: :service do
stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt'))
stub_request(:get, 'http://example.com/日本語').to_return(request_fixture('sjis.txt'))
stub_request(:get, 'https://github.com/qbi/WannaCry').to_return(status: 404)
stub_request(:get, 'http://example.com/test?data=file.gpx%5E1').to_return(status: 200)
stub_request(:get, 'http://example.com/test-').to_return(request_fixture('idn.txt'))
stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt'))
@ -85,6 +86,15 @@ RSpec.describe FetchLinkCardService, type: :service do
expect(a_request(:get, 'http://example.com/sjis')).to_not have_been_made
end
end
context do
let(:status) { Fabricate(:status, text: 'test http://example.com/test?data=file.gpx^1') }
it 'does fetch URLs with a caret in search params' do
expect(a_request(:get, 'http://example.com/test?data=file.gpx')).to_not have_been_made
expect(a_request(:get, 'http://example.com/test?data=file.gpx%5E1')).to have_been_made.once
end
end
end
context 'in a remote status' do

View file

@ -145,5 +145,35 @@ describe ResolveURLService, type: :service do
expect(subject.call(url, on_behalf_of: account)).to eq(status)
end
end
context 'when searching for a local link of a remote private status' do
let(:account) { Fabricate(:account) }
let(:poster) { Fabricate(:account, username: 'foo', domain: 'example.com') }
let(:url) { 'https://example.com/@foo/42' }
let(:uri) { 'https://example.com/users/foo/statuses/42' }
let!(:status) { Fabricate(:status, url: url, uri: uri, account: poster, visibility: :private) }
let(:search_url) { "https://#{Rails.configuration.x.local_domain}/@foo@example.com/#{status.id}" }
before do
stub_request(:get, url).to_return(status: 404) if url.present?
stub_request(:get, uri).to_return(status: 404)
end
context 'when the account follows the poster' do
before do
account.follow!(poster)
end
it 'returns the status' do
expect(subject.call(search_url, on_behalf_of: account)).to eq(status)
end
end
context 'when the account does not follow the poster' do
it 'does not return the status' do
expect(subject.call(search_url, on_behalf_of: account)).to be_nil
end
end
end
end
end

View file

@ -92,19 +92,32 @@ const redisUrlToClient = async (defaultConfig, redisUrl) => {
const numWorkers = +process.env.STREAMING_CLUSTER_NUM || (env === 'development' ? 1 : Math.max(os.cpus().length - 1, 1));
/**
* Attempts to safely parse a string as JSON, used when both receiving a message
* from redis and when receiving a message from a client over a websocket
* connection, this is why it accepts a `req` argument.
* @param {string} json
* @param {any} req
* @return {Object.<string, any>|null}
* @param {any?} req
* @returns {Object.<string, any>|null}
*/
const parseJSON = (json, req) => {
try {
return JSON.parse(json);
} catch (err) {
/* FIXME: This logging isn't great, and should probably be done at the
* call-site of parseJSON, not in the method, but this would require changing
* the signature of parseJSON to return something akin to a Result type:
* [Error|null, null|Object<string,any}], and then handling the error
* scenarios.
*/
if (req) {
if (req.accountId) {
log.warn(req.requestId, `Error parsing message from user ${req.accountId}: ${err}`);
} else {
log.silly(req.requestId, `Error parsing message from ${req.remoteAddress}: ${err}`);
}
} else {
log.warn(`Error parsing message from redis: ${err}`);
}
return null;
}
};
@ -169,7 +182,7 @@ const startWorker = async (workerId) => {
const redisPrefix = redisNamespace ? `${redisNamespace}:` : '';
/**
* @type {Object.<string, Array.<function(string): void>>}
* @type {Object.<string, Array.<function(Object<string, any>): void>>}
*/
const subs = {};
@ -209,7 +222,10 @@ const startWorker = async (workerId) => {
return;
}
callbacks.forEach(callback => callback(message));
const json = parseJSON(message, null);
if (!json) return;
callbacks.forEach(callback => callback(json));
};
/**
@ -231,6 +247,7 @@ const startWorker = async (workerId) => {
/**
* @param {string} channel
* @param {function(Object<string, any>): void} callback
*/
const unsubscribe = (channel, callback) => {
log.silly(`Removing listener for ${channel}`);
@ -380,7 +397,7 @@ const startWorker = async (workerId) => {
/**
* @param {any} req
* @return {string}
* @returns {string|undefined}
*/
const channelNameFromPath = req => {
const { path, query } = req;
@ -489,15 +506,11 @@ const startWorker = async (workerId) => {
/**
* @param {any} req
* @param {SystemMessageHandlers} eventHandlers
* @return {function(string): void}
* @returns {function(object): void}
*/
const createSystemMessageListener = (req, eventHandlers) => {
return message => {
const json = parseJSON(message, req);
if (!json) return;
const { event } = json;
const { event } = message;
log.silly(req.requestId, `System message for ${req.accountId}: ${event}`);
@ -614,19 +627,16 @@ const startWorker = async (workerId) => {
* @param {function(string, string): void} output
* @param {function(string[], function(string): void): void} attachCloseHandler
* @param {boolean=} needsFiltering
* @return {function(string): void}
* @returns {function(object): void}
*/
const streamFrom = (ids, req, output, attachCloseHandler, needsFiltering = false) => {
const accountId = req.accountId || req.remoteAddress;
log.verbose(req.requestId, `Starting stream from ${ids.join(', ')} for ${accountId}`);
// Currently message is of type string, soon it'll be Record<string, any>
const listener = message => {
const json = parseJSON(message, req);
if (!json) return;
const { event, payload, queued_at } = json;
const { event, payload, queued_at } = message;
const transmit = () => {
const now = new Date().getTime();
@ -1188,8 +1198,15 @@ const startWorker = async (workerId) => {
ws.on('close', onEnd);
ws.on('error', onEnd);
ws.on('message', data => {
const json = parseJSON(data, session.request);
ws.on('message', (data, isBinary) => {
if (isBinary) {
log.warn('socket', 'Received binary data, closing connection');
ws.close(1003, 'The mastodon streaming server does not support binary messages');
return;
}
const message = data.toString('utf8');
const json = parseJSON(message, session.request);
if (!json) return;