Merge tag 'v4.0.5'

This commit is contained in:
Mike Barnes 2023-07-07 01:51:21 +10:00
commit b45ba2d672
52 changed files with 599 additions and 170 deletions

View file

@ -3,6 +3,45 @@ Changelog
All notable changes to this project will be documented in this file. All notable changes to this project will be documented in this file.
## [4.0.5] - 2023-07-06
### Changed
- Change OpenGraph-based embeds to allow fullscreen ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25058))
- Change profile updates to be sent to recently-mentioned servers ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24852))
- Change `/api/v1/statuses/:id/history` to always return at least one item ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25510))
- Change auto-linking to allow carets in URL query params ([renchap](https://github.com/mastodon/mastodon/pull/25216))
### Removed
- Remove invalid `X-Frame-Options: ALLOWALL` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25070))
### Fixed
- Fix wrong view being displayed when a webhook fails validation ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25464))
- Fix soft-deleted post cleanup scheduler overwhelming the streaming server ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25519))
- Fix incorrect pagination headers in `/api/v2/admin/accounts` ([danielmbrasil](https://github.com/mastodon/mastodon/pull/25477))
- Fix performance of streaming by parsing message JSON once ([ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25278), [ThisIsMissEm](https://github.com/mastodon/mastodon/pull/25361))
- Fix CSP headers when `S3_ALIAS_HOST` includes a path component ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25273))
- Fix `tootctl accounts approve --number N` not aproving N earliest registrations ([danielmbrasil](https://github.com/mastodon/mastodon/pull/24605))
- Fix being able to vote on your own polls ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25015))
- Fix race condition when reblogging a status ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25016))
- Fix “Authorized applications” inefficiently and incorrectly getting last use date ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25060))
- Fix multiple N+1s in ConversationsController ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25134), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25399), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/25499))
- Fix user archive takeouts when using OpenStack Swift ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/24431))
- Fix searching for remote content by URL not working under certain conditions ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25637))
- Fix inefficiencies in indexing content for search ([VyrCossont](https://github.com/mastodon/mastodon/pull/24285), [VyrCossont](https://github.com/mastodon/mastodon/pull/24342))
### Security
- Add finer permission requirements for managing webhooks ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25463))
- Update dependencies
- Add hardening headers for user-uploaded files ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/25756))
- Fix verified links possibly hiding important parts of the URL (CVE-2023-36462)
- Fix timeout handling of outbound HTTP requests (CVE-2023-36461)
- Fix arbitrary file creation through media processing (CVE-2023-36460)
- Fix possible XSS in preview cards (CVE-2023-36459)
## [4.0.4] - 2023-04-04 ## [4.0.4] - 2023-04-04
### Fixed ### Fixed

View file

@ -10,40 +10,40 @@ GIT
GEM GEM
remote: https://rubygems.org/ remote: https://rubygems.org/
specs: specs:
actioncable (6.1.7) actioncable (6.1.7.4)
actionpack (= 6.1.7) actionpack (= 6.1.7.4)
activesupport (= 6.1.7) activesupport (= 6.1.7.4)
nio4r (~> 2.0) nio4r (~> 2.0)
websocket-driver (>= 0.6.1) websocket-driver (>= 0.6.1)
actionmailbox (6.1.7) actionmailbox (6.1.7.4)
actionpack (= 6.1.7) actionpack (= 6.1.7.4)
activejob (= 6.1.7) activejob (= 6.1.7.4)
activerecord (= 6.1.7) activerecord (= 6.1.7.4)
activestorage (= 6.1.7) activestorage (= 6.1.7.4)
activesupport (= 6.1.7) activesupport (= 6.1.7.4)
mail (>= 2.7.1) mail (>= 2.7.1)
actionmailer (6.1.7) actionmailer (6.1.7.4)
actionpack (= 6.1.7) actionpack (= 6.1.7.4)
actionview (= 6.1.7) actionview (= 6.1.7.4)
activejob (= 6.1.7) activejob (= 6.1.7.4)
activesupport (= 6.1.7) activesupport (= 6.1.7.4)
mail (~> 2.5, >= 2.5.4) mail (~> 2.5, >= 2.5.4)
rails-dom-testing (~> 2.0) rails-dom-testing (~> 2.0)
actionpack (6.1.7) actionpack (6.1.7.4)
actionview (= 6.1.7) actionview (= 6.1.7.4)
activesupport (= 6.1.7) activesupport (= 6.1.7.4)
rack (~> 2.0, >= 2.0.9) rack (~> 2.0, >= 2.0.9)
rack-test (>= 0.6.3) rack-test (>= 0.6.3)
rails-dom-testing (~> 2.0) rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.0, >= 1.2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0)
actiontext (6.1.7) actiontext (6.1.7.4)
actionpack (= 6.1.7) actionpack (= 6.1.7.4)
activerecord (= 6.1.7) activerecord (= 6.1.7.4)
activestorage (= 6.1.7) activestorage (= 6.1.7.4)
activesupport (= 6.1.7) activesupport (= 6.1.7.4)
nokogiri (>= 1.8.5) nokogiri (>= 1.8.5)
actionview (6.1.7) actionview (6.1.7.4)
activesupport (= 6.1.7) activesupport (= 6.1.7.4)
builder (~> 3.1) builder (~> 3.1)
erubi (~> 1.4) erubi (~> 1.4)
rails-dom-testing (~> 2.0) rails-dom-testing (~> 2.0)
@ -54,22 +54,22 @@ GEM
case_transform (>= 0.2) case_transform (>= 0.2)
jsonapi-renderer (>= 0.1.1.beta1, < 0.3) jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
active_record_query_trace (1.8) active_record_query_trace (1.8)
activejob (6.1.7) activejob (6.1.7.4)
activesupport (= 6.1.7) activesupport (= 6.1.7.4)
globalid (>= 0.3.6) globalid (>= 0.3.6)
activemodel (6.1.7) activemodel (6.1.7.4)
activesupport (= 6.1.7) activesupport (= 6.1.7.4)
activerecord (6.1.7) activerecord (6.1.7.4)
activemodel (= 6.1.7) activemodel (= 6.1.7.4)
activesupport (= 6.1.7) activesupport (= 6.1.7.4)
activestorage (6.1.7) activestorage (6.1.7.4)
actionpack (= 6.1.7) actionpack (= 6.1.7.4)
activejob (= 6.1.7) activejob (= 6.1.7.4)
activerecord (= 6.1.7) activerecord (= 6.1.7.4)
activesupport (= 6.1.7) activesupport (= 6.1.7.4)
marcel (~> 1.0) marcel (~> 1.0)
mini_mime (>= 1.1.0) mini_mime (>= 1.1.0)
activesupport (6.1.7) activesupport (6.1.7.4)
concurrent-ruby (~> 1.0, >= 1.0.2) concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 1.6, < 2) i18n (>= 1.6, < 2)
minitest (>= 5.1) minitest (>= 5.1)
@ -206,7 +206,7 @@ GEM
docile (1.3.4) docile (1.3.4)
domain_name (0.5.20190701) domain_name (0.5.20190701)
unf (>= 0.0.5, < 1.0.0) unf (>= 0.0.5, < 1.0.0)
doorkeeper (5.6.0) doorkeeper (5.6.6)
railties (>= 5) railties (>= 5)
dotenv (2.8.1) dotenv (2.8.1)
dotenv-rails (2.8.1) dotenv-rails (2.8.1)
@ -282,7 +282,7 @@ GEM
addressable (~> 2.7) addressable (~> 2.7)
omniauth (>= 1.9, < 3) omniauth (>= 1.9, < 3)
openid_connect (~> 1.2) openid_connect (~> 1.2)
globalid (1.0.0) globalid (1.0.1)
activesupport (>= 5.0) activesupport (>= 5.0)
hamlit (2.13.0) hamlit (2.13.0)
temple (>= 0.8.2) temple (>= 0.8.2)
@ -382,7 +382,7 @@ GEM
activesupport (>= 4) activesupport (>= 4)
railties (>= 4) railties (>= 4)
request_store (~> 1.0) request_store (~> 1.0)
loofah (2.19.0) loofah (2.19.1)
crass (~> 1.0.2) crass (~> 1.0.2)
nokogiri (>= 1.5.9) nokogiri (>= 1.5.9)
mail (2.7.1) mail (2.7.1)
@ -402,7 +402,7 @@ GEM
mime-types-data (~> 3.2015) mime-types-data (~> 3.2015)
mime-types-data (3.2022.0105) mime-types-data (3.2022.0105)
mini_mime (1.1.2) mini_mime (1.1.2)
mini_portile2 (2.8.0) mini_portile2 (2.8.2)
minitest (5.16.3) minitest (5.16.3)
msgpack (1.5.4) msgpack (1.5.4)
multi_json (1.15.0) multi_json (1.15.0)
@ -411,9 +411,9 @@ GEM
net-scp (4.0.0.rc1) net-scp (4.0.0.rc1)
net-ssh (>= 2.6.5, < 8.0.0) net-ssh (>= 2.6.5, < 8.0.0)
net-ssh (7.0.1) net-ssh (7.0.1)
nio4r (2.5.8) nio4r (2.5.9)
nokogiri (1.13.9) nokogiri (1.15.2)
mini_portile2 (~> 2.8.0) mini_portile2 (~> 2.8.2)
racc (~> 1.4) racc (~> 1.4)
nsa (0.2.8) nsa (0.2.8)
activesupport (>= 4.2, < 7) activesupport (>= 4.2, < 7)
@ -482,8 +482,8 @@ GEM
pundit (2.2.0) pundit (2.2.0)
activesupport (>= 3.0.0) activesupport (>= 3.0.0)
raabro (1.4.0) raabro (1.4.0)
racc (1.6.0) racc (1.7.1)
rack (2.2.4) rack (2.2.7)
rack-attack (6.6.1) rack-attack (6.6.1)
rack (>= 1.0, < 3) rack (>= 1.0, < 3)
rack-cors (1.1.1) rack-cors (1.1.1)
@ -498,20 +498,20 @@ GEM
rack rack
rack-test (2.0.2) rack-test (2.0.2)
rack (>= 1.3) rack (>= 1.3)
rails (6.1.7) rails (6.1.7.4)
actioncable (= 6.1.7) actioncable (= 6.1.7.4)
actionmailbox (= 6.1.7) actionmailbox (= 6.1.7.4)
actionmailer (= 6.1.7) actionmailer (= 6.1.7.4)
actionpack (= 6.1.7) actionpack (= 6.1.7.4)
actiontext (= 6.1.7) actiontext (= 6.1.7.4)
actionview (= 6.1.7) actionview (= 6.1.7.4)
activejob (= 6.1.7) activejob (= 6.1.7.4)
activemodel (= 6.1.7) activemodel (= 6.1.7.4)
activerecord (= 6.1.7) activerecord (= 6.1.7.4)
activestorage (= 6.1.7) activestorage (= 6.1.7.4)
activesupport (= 6.1.7) activesupport (= 6.1.7.4)
bundler (>= 1.15.0) bundler (>= 1.15.0)
railties (= 6.1.7) railties (= 6.1.7.4)
sprockets-rails (>= 2.0.0) sprockets-rails (>= 2.0.0)
rails-controller-testing (1.0.5) rails-controller-testing (1.0.5)
actionpack (>= 5.0.1.rc1) actionpack (>= 5.0.1.rc1)
@ -520,16 +520,16 @@ GEM
rails-dom-testing (2.0.3) rails-dom-testing (2.0.3)
activesupport (>= 4.2.0) activesupport (>= 4.2.0)
nokogiri (>= 1.6) nokogiri (>= 1.6)
rails-html-sanitizer (1.4.3) rails-html-sanitizer (1.4.4)
loofah (~> 2.3) loofah (~> 2.19, >= 2.19.1)
rails-i18n (6.0.0) rails-i18n (6.0.0)
i18n (>= 0.7, < 2) i18n (>= 0.7, < 2)
railties (>= 6.0.0, < 7) railties (>= 6.0.0, < 7)
rails-settings-cached (0.6.6) rails-settings-cached (0.6.6)
rails (>= 4.2.0) rails (>= 4.2.0)
railties (6.1.7) railties (6.1.7.4)
actionpack (= 6.1.7) actionpack (= 6.1.7.4)
activesupport (= 6.1.7) activesupport (= 6.1.7.4)
method_source method_source
rake (>= 12.2) rake (>= 12.2)
thor (~> 1.0) thor (~> 1.0)
@ -602,7 +602,7 @@ GEM
fugit (~> 1.1, >= 1.1.6) fugit (~> 1.1, >= 1.1.6)
safety_net_attestation (0.4.0) safety_net_attestation (0.4.0)
jwt (~> 2.0) jwt (~> 2.0)
sanitize (6.0.0) sanitize (6.0.1)
crass (~> 1.0.2) crass (~> 1.0.2)
nokogiri (>= 1.12.0) nokogiri (>= 1.12.0)
scenic (1.6.0) scenic (1.6.0)
@ -661,7 +661,7 @@ GEM
unicode-display_width (>= 1.1.1, < 3) unicode-display_width (>= 1.1.1, < 3)
terrapin (0.6.0) terrapin (0.6.0)
climate_control (>= 0.0.3, < 1.0) climate_control (>= 0.0.3, < 1.0)
thor (1.2.1) thor (1.2.2)
tilt (2.0.11) tilt (2.0.11)
tpm-key_attestation (0.11.0) tpm-key_attestation (0.11.0)
bindata (~> 2.4) bindata (~> 2.4)
@ -680,7 +680,7 @@ GEM
twitter-text (3.1.0) twitter-text (3.1.0)
idn-ruby idn-ruby
unf (~> 0.1.0) unf (~> 0.1.0)
tzinfo (2.0.5) tzinfo (2.0.6)
concurrent-ruby (~> 1.0) concurrent-ruby (~> 1.0)
tzinfo-data (1.2022.4) tzinfo-data (1.2022.4)
tzinfo (>= 1.0.0) tzinfo (>= 1.0.0)
@ -725,7 +725,7 @@ GEM
xorcist (1.1.3) xorcist (1.1.3)
xpath (3.2.0) xpath (3.2.0)
nokogiri (~> 1.8) nokogiri (~> 1.8)
zeitwerk (2.6.0) zeitwerk (2.6.8)
PLATFORMS PLATFORMS
ruby ruby

View file

@ -20,6 +20,7 @@ module Admin
authorize :webhook, :create? authorize :webhook, :create?
@webhook = Webhook.new(resource_params) @webhook = Webhook.new(resource_params)
@webhook.current_account = current_account
if @webhook.save if @webhook.save
redirect_to admin_webhook_path(@webhook) redirect_to admin_webhook_path(@webhook)
@ -39,10 +40,12 @@ module Admin
def update def update
authorize @webhook, :update? authorize @webhook, :update?
@webhook.current_account = current_account
if @webhook.update(resource_params) if @webhook.update(resource_params)
redirect_to admin_webhook_path(@webhook) redirect_to admin_webhook_path(@webhook)
else else
render :show render :edit
end end
end end

View file

@ -11,7 +11,7 @@ class Api::V1::ConversationsController < Api::BaseController
def index def index
@conversations = paginated_conversations @conversations = paginated_conversations
render json: @conversations, each_serializer: REST::ConversationSerializer render json: @conversations, each_serializer: REST::ConversationSerializer, relationships: StatusRelationshipsPresenter.new(@conversations.map(&:last_status), current_user&.account_id)
end end
def read def read
@ -32,6 +32,19 @@ class Api::V1::ConversationsController < Api::BaseController
def paginated_conversations def paginated_conversations
AccountConversation.where(account: current_account) AccountConversation.where(account: current_account)
.includes(
account: :account_stat,
last_status: [
:media_attachments,
:preview_cards,
:status_stat,
:tags,
{
active_mentions: [account: :account_stat],
account: :account_stat,
},
]
)
.to_a_paginated_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id)) .to_a_paginated_by_id(limit_param(LIMIT), params_slice(:max_id, :since_id, :min_id))
end end

View file

@ -7,11 +7,15 @@ class Api::V1::Statuses::HistoriesController < Api::BaseController
before_action :set_status before_action :set_status
def show def show
render json: @status.edits.includes(:account, status: [:account]), each_serializer: REST::StatusEditSerializer render json: status_edits, each_serializer: REST::StatusEditSerializer
end end
private private
def status_edits
@status.edits.includes(:account, status: [:account]).to_a.presence || [@status.build_snapshot(at_time: @status.edited_at || @status.created_at)]
end
def set_status def set_status
@status = Status.find(params[:status_id]) @status = Status.find(params[:status_id])
authorize @status, :show? authorize @status, :show?

View file

@ -2,6 +2,8 @@
class Api::V1::Statuses::ReblogsController < Api::BaseController class Api::V1::Statuses::ReblogsController < Api::BaseController
include Authorization include Authorization
include Redisable
include Lockable
before_action -> { doorkeeper_authorize! :write, :'write:statuses' } before_action -> { doorkeeper_authorize! :write, :'write:statuses' }
before_action :require_user! before_action :require_user!
@ -10,7 +12,9 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController
override_rate_limit_headers :create, family: :statuses override_rate_limit_headers :create, family: :statuses
def create def create
@status = ReblogService.new.call(current_account, @reblog, reblog_params) with_lock("reblog:#{current_account.id}:#{@reblog.id}") do
@status = ReblogService.new.call(current_account, @reblog, reblog_params)
end
render json: @status, serializer: REST::StatusSerializer render json: @status, serializer: REST::StatusSerializer
end end

View file

@ -18,6 +18,14 @@ class Api::V2::Admin::AccountsController < Api::V1::Admin::AccountsController
private private
def next_path
api_v2_admin_accounts_url(pagination_params(max_id: pagination_max_id)) if records_continue?
end
def prev_path
api_v2_admin_accounts_url(pagination_params(min_id: pagination_since_id)) unless @accounts.empty?
end
def filtered_accounts def filtered_accounts
AccountFilter.new(translated_filter_params).results AccountFilter.new(translated_filter_params).results
end end

View file

@ -13,7 +13,7 @@ class BackupsController < ApplicationController
when :s3 when :s3
redirect_to @backup.dump.expiring_url(10) redirect_to @backup.dump.expiring_url(10)
when :fog when :fog
if Paperclip::Attachment.default_options.dig(:storage, :fog_credentials, :openstack_temp_url_key).present? if Paperclip::Attachment.default_options.dig(:fog_credentials, :openstack_temp_url_key).present?
redirect_to @backup.dump.expiring_url(Time.now.utc + 10) redirect_to @backup.dump.expiring_url(Time.now.utc + 10)
else else
redirect_to full_asset_url(@backup.dump.url) redirect_to full_asset_url(@backup.dump.url)

View file

@ -46,6 +46,6 @@ class MediaController < ApplicationController
end end
def allow_iframing def allow_iframing
response.headers['X-Frame-Options'] = 'ALLOWALL' response.headers.delete('X-Frame-Options')
end end
end end

View file

@ -8,6 +8,8 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
before_action :require_not_suspended!, only: :destroy before_action :require_not_suspended!, only: :destroy
before_action :set_body_classes before_action :set_body_classes
before_action :set_last_used_at_by_app, only: :index, unless: -> { request.format == :json }
skip_before_action :require_functional! skip_before_action :require_functional!
include Localized include Localized
@ -30,4 +32,14 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
def require_not_suspended! def require_not_suspended!
forbidden if current_account.suspended? forbidden if current_account.suspended?
end end
def set_last_used_at_by_app
@last_used_at_by_app = Doorkeeper::AccessToken
.select('DISTINCT ON (application_id) application_id, last_used_at')
.where(resource_owner_id: current_resource_owner.id)
.where.not(last_used_at: nil)
.order(application_id: :desc, last_used_at: :desc)
.pluck(:application_id, :last_used_at)
.to_h
end
end end

View file

@ -43,7 +43,7 @@ class StatusesController < ApplicationController
return not_found if @status.hidden? || @status.reblog? return not_found if @status.hidden? || @status.reblog?
expires_in 180, public: true expires_in 180, public: true
response.headers['X-Frame-Options'] = 'ALLOWALL' response.headers.delete('X-Frame-Options')
render layout: 'embedded' render layout: 'embedded'
end end

View file

@ -49,6 +49,10 @@ module FormattingHelper
end end
def account_field_value_format(field, with_rel_me: true) def account_field_value_format(field, with_rel_me: true)
html_aware_format(field.value, field.account.local?, with_rel_me: with_rel_me, with_domains: true, multiline: false) if field.verified? && !field.account.local?
TextFormatter.shortened_link(field.value_for_verification)
else
html_aware_format(field.value, field.account.local?, with_rel_me: with_rel_me, with_domains: true, multiline: false)
end
end end
end end

View file

@ -6,7 +6,7 @@ class AccountReachFinder
end end
def inboxes def inboxes
(followers_inboxes + reporters_inboxes + relay_inboxes).uniq (followers_inboxes + reporters_inboxes + recently_mentioned_inboxes + relay_inboxes).uniq
end end
private private
@ -19,6 +19,13 @@ class AccountReachFinder
Account.where(id: @account.targeted_reports.select(:account_id)).inboxes Account.where(id: @account.targeted_reports.select(:account_id)).inboxes
end end
def recently_mentioned_inboxes
cutoff_id = Mastodon::Snowflake.id_at(2.days.ago, with_random: false)
recent_statuses = @account.statuses.recent.where(id: cutoff_id...).limit(200)
Account.joins(:mentions).where(mentions: { status: recent_statuses }).inboxes.take(2000)
end
def relay_inboxes def relay_inboxes
Relay.enabled.pluck(:inbox_url) Relay.enabled.pluck(:inbox_url)
end end

View file

@ -9,10 +9,6 @@ module ApplicationExtension
validates :redirect_uri, length: { maximum: 2_000 } validates :redirect_uri, length: { maximum: 2_000 }
end end
def most_recently_used_access_token
@most_recently_used_access_token ||= access_tokens.where.not(last_used_at: nil).order(last_used_at: :desc).first
end
def confirmation_redirect_uri def confirmation_redirect_uri
redirect_uri.lines.first.strip redirect_uri.lines.first.strip
end end

View file

@ -140,7 +140,7 @@ class LinkDetailsExtractor
end end
def html def html
player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil player_url.present? ? content_tag(:iframe, nil, src: player_url, width: width, height: height, allowfullscreen: 'true', allowtransparency: 'true', scrolling: 'no', frameborder: '0') : nil
end end
def width def width

View file

@ -7,11 +7,48 @@ require 'resolv'
# Monkey-patch the HTTP.rb timeout class to avoid using a timeout block # Monkey-patch the HTTP.rb timeout class to avoid using a timeout block
# around the Socket#open method, since we use our own timeout blocks inside # around the Socket#open method, since we use our own timeout blocks inside
# that method # that method
#
# Also changes how the read timeout behaves so that it is cumulative (closer
# to HTTP::Timeout::Global, but still having distinct timeouts for other
# operation types)
class HTTP::Timeout::PerOperation class HTTP::Timeout::PerOperation
def connect(socket_class, host, port, nodelay = false) def connect(socket_class, host, port, nodelay = false)
@socket = socket_class.open(host, port) @socket = socket_class.open(host, port)
@socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
end end
# Reset deadline when the connection is re-used for different requests
def reset_counter
@deadline = nil
end
# Read data from the socket
def readpartial(size, buffer = nil)
@deadline ||= Process.clock_gettime(Process::CLOCK_MONOTONIC) + @read_timeout
timeout = false
loop do
result = @socket.read_nonblock(size, buffer, exception: false)
return :eof if result.nil?
remaining_time = @deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
raise HTTP::TimeoutError, "Read timed out after #{@read_timeout} seconds" if timeout || remaining_time <= 0
return result if result != :wait_readable
# marking the socket for timeout. Why is this not being raised immediately?
# it seems there is some race-condition on the network level between calling
# #read_nonblock and #wait_readable, in which #read_nonblock signalizes waiting
# for reads, and when waiting for x seconds, it returns nil suddenly without completing
# the x seconds. In a normal case this would be a timeout on wait/read, but it can
# also mean that the socket has been closed by the server. Therefore we "mark" the
# socket for timeout and try to read more bytes. If it returns :eof, it's all good, no
# timeout. Else, the first timeout was a proper timeout.
# This hack has to be done because io/wait#wait_readable doesn't provide a value for when
# the socket is closed by the server, and HTTP::Parser doesn't provide the limit for the chunks.
timeout = true unless @socket.to_io.wait_readable(remaining_time)
end
end
end end
class Request class Request

View file

@ -48,6 +48,26 @@ class TextFormatter
html.html_safe # rubocop:disable Rails/OutputSafety html.html_safe # rubocop:disable Rails/OutputSafety
end end
class << self
include ERB::Util
def shortened_link(url, rel_me: false)
url = Addressable::URI.parse(url).to_s
rel = rel_me ? (DEFAULT_REL + %w(me)) : DEFAULT_REL
prefix = url.match(URL_PREFIX_REGEX).to_s
display_url = url[prefix.length, 30]
suffix = url[prefix.length + 30..-1]
cutoff = url[prefix.length..-1].length > 30
<<~HTML.squish
<a href="#{h(url)}" target="_blank" rel="#{rel.join(' ')}"><span class="invisible">#{h(prefix)}</span><span class="#{cutoff ? 'ellipsis' : ''}">#{h(display_url)}</span><span class="invisible">#{h(suffix)}</span></a>
HTML
rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
h(url)
end
end
private private
def rewrite def rewrite
@ -70,19 +90,7 @@ class TextFormatter
end end
def link_to_url(entity) def link_to_url(entity)
url = Addressable::URI.parse(entity[:url]).to_s TextFormatter.shortened_link(entity[:url], rel_me: with_rel_me?)
rel = with_rel_me? ? (DEFAULT_REL + %w(me)) : DEFAULT_REL
prefix = url.match(URL_PREFIX_REGEX).to_s
display_url = url[prefix.length, 30]
suffix = url[prefix.length + 30..-1]
cutoff = url[prefix.length..-1].length > 30
<<~HTML.squish
<a href="#{h(url)}" target="_blank" rel="#{rel.join(' ')}"><span class="invisible">#{h(prefix)}</span><span class="#{cutoff ? 'ellipsis' : ''}">#{h(display_url)}</span><span class="invisible">#{h(suffix)}</span></a>
HTML
rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
h(entity[:url])
end end
def link_to_hashtag(entity) def link_to_hashtag(entity)

View file

@ -16,34 +16,44 @@
class AccountConversation < ApplicationRecord class AccountConversation < ApplicationRecord
include Redisable include Redisable
attr_writer :participant_accounts
before_validation :set_last_status
after_commit :push_to_streaming_api after_commit :push_to_streaming_api
belongs_to :account belongs_to :account
belongs_to :conversation belongs_to :conversation
belongs_to :last_status, class_name: 'Status' belongs_to :last_status, class_name: 'Status'
before_validation :set_last_status
def participant_account_ids=(arr) def participant_account_ids=(arr)
self[:participant_account_ids] = arr.sort self[:participant_account_ids] = arr.sort
@participant_accounts = nil
end end
def participant_accounts def participant_accounts
if participant_account_ids.empty? @participant_accounts ||= Account.where(id: participant_account_ids).to_a
[account] @participant_accounts.presence || [account]
else
participants = Account.where(id: participant_account_ids)
participants.empty? ? [account] : participants
end
end end
class << self class << self
def to_a_paginated_by_id(limit, options = {}) def to_a_paginated_by_id(limit, options = {})
if options[:min_id] array = begin
paginate_by_min_id(limit, options[:min_id], options[:max_id]).reverse if options[:min_id]
else paginate_by_min_id(limit, options[:min_id], options[:max_id]).reverse
paginate_by_max_id(limit, options[:max_id], options[:since_id]).to_a else
paginate_by_max_id(limit, options[:max_id], options[:since_id]).to_a
end
end end
# Preload participants
participant_ids = array.flat_map(&:participant_account_ids)
accounts_by_id = Account.where(id: participant_ids).index_by(&:id)
array.each do |conversation|
conversation.participant_accounts = conversation.participant_account_ids.filter_map { |id| accounts_by_id[id] }
end
array
end end
def paginate_by_min_id(limit, min_id = nil, max_id = nil) def paginate_by_min_id(limit, min_id = nil, max_id = nil)

View file

@ -22,15 +22,14 @@ module Attachmentable
included do included do
def self.has_attached_file(name, options = {}) # rubocop:disable Naming/PredicateName def self.has_attached_file(name, options = {}) # rubocop:disable Naming/PredicateName
options = { validate_media_type: false }.merge(options)
super(name, options) super(name, options)
send(:"before_#{name}_post_process") do
send(:"before_#{name}_validate") do
attachment = send(name) attachment = send(name)
check_image_dimension(attachment) check_image_dimension(attachment)
set_file_content_type(attachment) set_file_content_type(attachment)
obfuscate_file_name(attachment) obfuscate_file_name(attachment)
set_file_extension(attachment) set_file_extension(attachment)
Paperclip::Validators::MediaTypeSpoofDetectionValidator.new(attributes: [name]).validate(self)
end end
end end
end end

View file

@ -12,7 +12,7 @@
# #
class Identity < ApplicationRecord class Identity < ApplicationRecord
belongs_to :user, dependent: :destroy belongs_to :user
validates :uid, presence: true, uniqueness: { scope: :provider } validates :uid, presence: true, uniqueness: { scope: :provider }
validates :provider, presence: true validates :provider, presence: true

View file

@ -19,6 +19,8 @@ class Webhook < ApplicationRecord
report.created report.created
).freeze ).freeze
attr_writer :current_account
scope :enabled, -> { where(enabled: true) } scope :enabled, -> { where(enabled: true) }
validates :url, presence: true, url: true validates :url, presence: true, url: true
@ -26,6 +28,7 @@ class Webhook < ApplicationRecord
validates :events, presence: true validates :events, presence: true
validate :validate_events validate :validate_events
validate :validate_permissions
before_validation :strip_events before_validation :strip_events
before_validation :generate_secret before_validation :generate_secret
@ -42,12 +45,29 @@ class Webhook < ApplicationRecord
update!(enabled: false) update!(enabled: false)
end end
def required_permissions
events.map { |event| Webhook.permission_for_event(event) }
end
def self.permission_for_event(event)
case event
when 'account.approved', 'account.created', 'account.updated'
:manage_users
when 'report.created'
:manage_reports
end
end
private private
def validate_events def validate_events
errors.add(:events, :invalid) if events.any? { |e| !EVENTS.include?(e) } errors.add(:events, :invalid) if events.any? { |e| !EVENTS.include?(e) }
end end
def validate_permissions
errors.add(:events, :invalid_permissions) if defined?(@current_account) && required_permissions.any? { |permission| !@current_account.user_role.can?(permission) }
end
def strip_events def strip_events
self.events = events.map { |str| str.strip.presence }.compact if events.present? self.events = events.map { |str| str.strip.presence }.compact if events.present?
end end

View file

@ -14,7 +14,7 @@ class WebhookPolicy < ApplicationPolicy
end end
def update? def update?
role.can?(:manage_webhooks) role.can?(:manage_webhooks) && record.required_permissions.all? { |permission| role.can?(permission) }
end end
def enable? def enable?
@ -30,6 +30,6 @@ class WebhookPolicy < ApplicationPolicy
end end
def destroy? def destroy?
role.can?(:manage_webhooks) role.can?(:manage_webhooks) && record.required_permissions.all? { |permission| role.can?(permission) }
end end
end end

View file

@ -11,4 +11,8 @@ class REST::PreviewCardSerializer < ActiveModel::Serializer
def image def image
object.image? ? full_asset_url(object.image.url(:original)) : nil object.image? ? full_asset_url(object.image.url(:original)) : nil
end end
def html
Sanitize.fragment(object.html, Sanitize::Config::MASTODON_OEMBED)
end
end end

View file

@ -12,6 +12,7 @@ class RemoveStatusService < BaseService
# @option [Boolean] :immediate # @option [Boolean] :immediate
# @option [Boolean] :preserve # @option [Boolean] :preserve
# @option [Boolean] :original_removed # @option [Boolean] :original_removed
# @option [Boolean] :skip_streaming
def call(status, **options) def call(status, **options)
@payload = Oj.dump(event: :delete, payload: status.id.to_s) @payload = Oj.dump(event: :delete, payload: status.id.to_s)
@status = status @status = status
@ -52,6 +53,9 @@ class RemoveStatusService < BaseService
private private
# The following FeedManager calls all do not result in redis publishes for
# streaming, as the `:update` option is false
def remove_from_self def remove_from_self
FeedManager.instance.unpush_from_home(@account, @status) FeedManager.instance.unpush_from_home(@account, @status)
end end
@ -75,6 +79,8 @@ class RemoveStatusService < BaseService
# followers. Here we send a delete to actively mentioned accounts # followers. Here we send a delete to actively mentioned accounts
# that may not follow the account # that may not follow the account
return if skip_streaming?
@status.active_mentions.find_each do |mention| @status.active_mentions.find_each do |mention|
redis.publish("timeline:#{mention.account_id}", @payload) redis.publish("timeline:#{mention.account_id}", @payload)
end end
@ -103,7 +109,7 @@ class RemoveStatusService < BaseService
# without us being able to do all the fancy stuff # without us being able to do all the fancy stuff
@status.reblogs.rewhere(deleted_at: [nil, @status.deleted_at]).includes(:account).reorder(nil).find_each do |reblog| @status.reblogs.rewhere(deleted_at: [nil, @status.deleted_at]).includes(:account).reorder(nil).find_each do |reblog|
RemoveStatusService.new.call(reblog, original_removed: true) RemoveStatusService.new.call(reblog, original_removed: true, skip_streaming: skip_streaming?)
end end
end end
@ -114,6 +120,8 @@ class RemoveStatusService < BaseService
return unless @status.public_visibility? return unless @status.public_visibility?
return if skip_streaming?
@status.tags.map(&:name).each do |hashtag| @status.tags.map(&:name).each do |hashtag|
redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}", @payload) redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}", @payload)
redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}:local", @payload) if @status.local? redis.publish("timeline:hashtag:#{hashtag.mb_chars.downcase}:local", @payload) if @status.local?
@ -123,6 +131,8 @@ class RemoveStatusService < BaseService
def remove_from_public def remove_from_public
return unless @status.public_visibility? return unless @status.public_visibility?
return if skip_streaming?
redis.publish('timeline:public', @payload) redis.publish('timeline:public', @payload)
redis.publish(@status.local? ? 'timeline:public:local' : 'timeline:public:remote', @payload) redis.publish(@status.local? ? 'timeline:public:local' : 'timeline:public:remote', @payload)
end end
@ -130,6 +140,8 @@ class RemoveStatusService < BaseService
def remove_from_media def remove_from_media
return unless @status.public_visibility? return unless @status.public_visibility?
return if skip_streaming?
redis.publish('timeline:public:media', @payload) redis.publish('timeline:public:media', @payload)
redis.publish(@status.local? ? 'timeline:public:local:media' : 'timeline:public:remote:media', @payload) redis.publish(@status.local? ? 'timeline:public:local:media' : 'timeline:public:remote:media', @payload)
end end
@ -143,4 +155,8 @@ class RemoveStatusService < BaseService
def permanently? def permanently?
@options[:immediate] || !(@options[:preserve] || @status.reported?) @options[:immediate] || !(@options[:preserve] || @status.reported?)
end end
def skip_streaming?
!!@options[:skip_streaming]
end
end end

View file

@ -89,13 +89,28 @@ class ResolveURLService < BaseService
def process_local_url def process_local_url
recognized_params = Rails.application.routes.recognize_path(@url) recognized_params = Rails.application.routes.recognize_path(@url)
return unless recognized_params[:action] == 'show' case recognized_params[:controller]
when 'statuses'
return unless recognized_params[:action] == 'show'
if recognized_params[:controller] == 'statuses'
status = Status.find_by(id: recognized_params[:id]) status = Status.find_by(id: recognized_params[:id])
check_local_status(status) check_local_status(status)
elsif recognized_params[:controller] == 'accounts' when 'accounts'
return unless recognized_params[:action] == 'show'
Account.find_local(recognized_params[:username]) Account.find_local(recognized_params[:username])
when 'home'
return unless recognized_params[:action] == 'index' && recognized_params[:username_with_domain].present?
if recognized_params[:any]&.match?(/\A[0-9]+\Z/)
status = Status.find_by(id: recognized_params[:any])
check_local_status(status)
elsif recognized_params[:any].blank?
username, domain = recognized_params[:username_with_domain].gsub(/\A@/, '').split('@')
return unless username.present? && domain.present?
Account.find_remote(username, domain)
end
end end
end end

View file

@ -3,8 +3,8 @@
class VoteValidator < ActiveModel::Validator class VoteValidator < ActiveModel::Validator
def validate(vote) def validate(vote)
vote.errors.add(:base, I18n.t('polls.errors.expired')) if vote.poll.expired? vote.errors.add(:base, I18n.t('polls.errors.expired')) if vote.poll.expired?
vote.errors.add(:base, I18n.t('polls.errors.invalid_choice')) if invalid_choice?(vote) vote.errors.add(:base, I18n.t('polls.errors.invalid_choice')) if invalid_choice?(vote)
vote.errors.add(:base, I18n.t('polls.errors.self_vote')) if self_vote?(vote)
if vote.poll.multiple? && vote.poll.votes.where(account: vote.account, choice: vote.choice).exists? if vote.poll.multiple? && vote.poll.votes.where(account: vote.account, choice: vote.choice).exists?
vote.errors.add(:base, I18n.t('polls.errors.already_voted')) vote.errors.add(:base, I18n.t('polls.errors.already_voted'))
@ -18,4 +18,8 @@ class VoteValidator < ActiveModel::Validator
def invalid_choice?(vote) def invalid_choice?(vote)
vote.choice.negative? || vote.choice >= vote.poll.options.size vote.choice.negative? || vote.choice >= vote.poll.options.size
end end
def self_vote?(vote)
vote.account_id == vote.poll.account_id
end
end end

View file

@ -5,7 +5,7 @@
= f.input :url, wrapper: :with_block_label, input_html: { placeholder: 'https://' } = f.input :url, wrapper: :with_block_label, input_html: { placeholder: 'https://' }
.fields-group .fields-group
= f.input :events, collection: Webhook::EVENTS, wrapper: :with_block_label, include_blank: false, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li' = f.input :events, collection: Webhook::EVENTS, wrapper: :with_block_label, include_blank: false, as: :check_boxes, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li', disabled: Webhook::EVENTS.filter { |event| !current_user.role.can?(Webhook.permission_for_event(event)) }
.actions .actions
= f.button :button, @webhook.new_record? ? t('admin.webhooks.add_new') : t('generic.save_changes'), type: :submit = f.button :button, @webhook.new_record? ? t('admin.webhooks.add_new') : t('generic.save_changes'), type: :submit

View file

@ -18,8 +18,8 @@
.announcements-list__item__action-bar .announcements-list__item__action-bar
.announcements-list__item__meta .announcements-list__item__meta
- if application.most_recently_used_access_token - if @last_used_at_by_app[application.id]
= t('doorkeeper.authorized_applications.index.last_used_at', date: l(application.most_recently_used_access_token.last_used_at.to_date)) = t('doorkeeper.authorized_applications.index.last_used_at', date: l(@last_used_at_by_app[application.id].to_date))
- else - else
= t('doorkeeper.authorized_applications.index.never_used') = t('doorkeeper.authorized_applications.index.never_used')

View file

@ -6,17 +6,19 @@ class Scheduler::IndexingScheduler
sidekiq_options retry: 0 sidekiq_options retry: 0
IMPORT_BATCH_SIZE = 1000
SCAN_BATCH_SIZE = 10 * IMPORT_BATCH_SIZE
def perform def perform
return unless Chewy.enabled? return unless Chewy.enabled?
indexes.each do |type| indexes.each do |type|
with_redis do |redis| with_redis do |redis|
ids = redis.smembers("chewy:queue:#{type.name}") redis.sscan_each("chewy:queue:#{type.name}", count: SCAN_BATCH_SIZE).each_slice(IMPORT_BATCH_SIZE) do |ids|
type.import!(ids)
type.import!(ids) redis.pipelined do |pipeline|
pipeline.srem("chewy:queue:#{type.name}", ids)
redis.pipelined do |pipeline| end
ids.each { |id| pipeline.srem("chewy:queue:#{type.name}", id) }
end end
end end
end end

View file

@ -24,7 +24,7 @@ class Scheduler::UserCleanupScheduler
def clean_discarded_statuses! def clean_discarded_statuses!
Status.unscoped.discarded.where('deleted_at <= ?', 30.days.ago).find_in_batches do |statuses| Status.unscoped.discarded.where('deleted_at <= ?', 30.days.ago).find_in_batches do |statuses|
RemovalWorker.push_bulk(statuses) do |status| RemovalWorker.push_bulk(statuses) do |status|
[status.id, { 'immediate' => true }] [status.id, { 'immediate' => true, 'skip_streaming' => true }]
end end
end end
end end

View file

@ -29,6 +29,7 @@ require_relative '../lib/paperclip/url_generator_extensions'
require_relative '../lib/paperclip/attachment_extensions' require_relative '../lib/paperclip/attachment_extensions'
require_relative '../lib/paperclip/lazy_thumbnail' require_relative '../lib/paperclip/lazy_thumbnail'
require_relative '../lib/paperclip/gif_transcoder' require_relative '../lib/paperclip/gif_transcoder'
require_relative '../lib/paperclip/media_type_spoof_detector_extensions'
require_relative '../lib/paperclip/transcoder' require_relative '../lib/paperclip/transcoder'
require_relative '../lib/paperclip/type_corrector' require_relative '../lib/paperclip/type_corrector'
require_relative '../lib/paperclip/response_with_limit_adapter' require_relative '../lib/paperclip/response_with_limit_adapter'
@ -160,6 +161,10 @@ module Mastodon
end end
end end
config.public_file_server.headers = {
'X-Content-Type-Options' => 'nosniff',
}
# config.paths.add File.join('app', 'api'), glob: File.join('**', '*.rb') # config.paths.add File.join('app', 'api'), glob: File.join('**', '*.rb')
# config.autoload_paths += Dir[Rails.root.join('app', 'api', '*')] # config.autoload_paths += Dir[Rails.root.join('app', 'api', '*')]

View file

@ -0,0 +1,27 @@
<policymap>
<!-- Set some basic system resource limits -->
<policy domain="resource" name="time" value="60" />
<policy domain="module" rights="none" pattern="URL" />
<policy domain="filter" rights="none" pattern="*" />
<!--
Ideally, we would restrict ImageMagick to only accessing its own
disk-backed pixel cache as well as Mastodon-created Tempfiles.
However, those paths depend on the operating system and environment
variables, so they can only be known at runtime.
Furthermore, those paths are not necessarily shared across Mastodon
processes, so even creating a policy.xml at runtime is impractical.
For the time being, only disable indirect reads.
-->
<policy domain="path" rights="none" pattern="@*" />
<!-- Disallow any coder by default, and only enable ones required by Mastodon -->
<policy domain="coder" rights="none" pattern="*" />
<policy domain="coder" rights="read | write" pattern="{PNG,JPEG,GIF,HEIC,WEBP}" />
<policy domain="coder" rights="write" pattern="{HISTOGRAM,RGB,INFO}" />
</policymap>

View file

@ -3,7 +3,7 @@
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy
def host_to_url(str) def host_to_url(str)
"http#{Rails.configuration.x.use_https ? 's' : ''}://#{str}" unless str.blank? "http#{Rails.configuration.x.use_https ? 's' : ''}://#{str}".split('/').first if str.present?
end end
base_host = Rails.configuration.x.web_domain base_host = Rails.configuration.x.web_domain

View file

@ -155,3 +155,10 @@ unless defined?(Seahorse)
end end
end end
end end
# Set our ImageMagick security policy, but allow admins to override it
ENV['MAGICK_CONFIGURE_PATH'] = begin
imagemagick_config_paths = ENV.fetch('MAGICK_CONFIGURE_PATH', '').split(File::PATH_SEPARATOR)
imagemagick_config_paths << Rails.root.join('config', 'imagemagick').expand_path.to_s
imagemagick_config_paths.join(File::PATH_SEPARATOR)
end

View file

@ -25,7 +25,7 @@ module Twitter::TwitterText
\) \)
/iox /iox
UCHARS = '\u{A0}-\u{D7FF}\u{F900}-\u{FDCF}\u{FDF0}-\u{FFEF}\u{10000}-\u{1FFFD}\u{20000}-\u{2FFFD}\u{30000}-\u{3FFFD}\u{40000}-\u{4FFFD}\u{50000}-\u{5FFFD}\u{60000}-\u{6FFFD}\u{70000}-\u{7FFFD}\u{80000}-\u{8FFFD}\u{90000}-\u{9FFFD}\u{A0000}-\u{AFFFD}\u{B0000}-\u{BFFFD}\u{C0000}-\u{CFFFD}\u{D0000}-\u{DFFFD}\u{E1000}-\u{EFFFD}\u{E000}-\u{F8FF}\u{F0000}-\u{FFFFD}\u{100000}-\u{10FFFD}' UCHARS = '\u{A0}-\u{D7FF}\u{F900}-\u{FDCF}\u{FDF0}-\u{FFEF}\u{10000}-\u{1FFFD}\u{20000}-\u{2FFFD}\u{30000}-\u{3FFFD}\u{40000}-\u{4FFFD}\u{50000}-\u{5FFFD}\u{60000}-\u{6FFFD}\u{70000}-\u{7FFFD}\u{80000}-\u{8FFFD}\u{90000}-\u{9FFFD}\u{A0000}-\u{AFFFD}\u{B0000}-\u{BFFFD}\u{C0000}-\u{CFFFD}\u{D0000}-\u{DFFFD}\u{E1000}-\u{EFFFD}\u{E000}-\u{F8FF}\u{F0000}-\u{FFFFD}\u{100000}-\u{10FFFD}'
REGEXEN[:valid_url_query_chars] = /[a-z0-9!?\*'\(\);:&=\+\$\/%#\[\]\-_\.,~|@#{UCHARS}]/iou REGEXEN[:valid_url_query_chars] = /[a-z0-9!?\*'\(\);:&=\+\$\/%#\[\]\-_\.,~|@\^#{UCHARS}]/iou
REGEXEN[:valid_url_query_ending_chars] = /[a-z0-9_&=#\/\-#{UCHARS}]/iou REGEXEN[:valid_url_query_ending_chars] = /[a-z0-9_&=#\/\-#{UCHARS}]/iou
REGEXEN[:valid_url_path] = /(?: REGEXEN[:valid_url_path] = /(?:
(?: (?:

View file

@ -53,3 +53,7 @@ en:
position: position:
elevated: cannot be higher than your current role elevated: cannot be higher than your current role
own_role: cannot be changed with your current role own_role: cannot be changed with your current role
webhook:
attributes:
events:
invalid_permissions: cannot include events you don't have the rights to

View file

@ -1332,6 +1332,7 @@ en:
expired: The poll has already ended expired: The poll has already ended
invalid_choice: The chosen vote option does not exist invalid_choice: The chosen vote option does not exist
over_character_limit: cannot be longer than %{max} characters each over_character_limit: cannot be longer than %{max} characters each
self_vote: You cannot vote in your own polls
too_few_options: must have more than one item too_few_options: must have more than one item
too_many_options: can't contain more than %{max} items too_many_options: can't contain more than %{max} items
preferences: preferences:

2
dist/nginx.conf vendored
View file

@ -109,6 +109,8 @@ server {
location ~ ^/system/ { location ~ ^/system/ {
add_header Cache-Control "public, max-age=2419200, immutable"; add_header Cache-Control "public, max-age=2419200, immutable";
add_header Strict-Transport-Security "max-age=63072000; includeSubDomains"; add_header Strict-Transport-Security "max-age=63072000; includeSubDomains";
add_header X-Content-Type-Options nosniff;
add_header Content-Security-Policy "default-src 'none'; form-action 'none'";
try_files $uri =404; try_files $uri =404;
} }

View file

@ -513,7 +513,7 @@ module Mastodon
User.pending.find_each(&:approve!) User.pending.find_each(&:approve!)
say('OK', :green) say('OK', :green)
elsif options[:number] elsif options[:number]
User.pending.limit(options[:number]).each(&:approve!) User.pending.order(created_at: :asc).limit(options[:number]).each(&:approve!)
say('OK', :green) say('OK', :green)
elsif username.present? elsif username.present?
account = Account.find_local(username) account = Account.find_local(username)

View file

@ -13,7 +13,7 @@ module Mastodon
end end
def patch def patch
4 5
end end
def flags def flags

View file

@ -0,0 +1,22 @@
# frozen_string_literal: true
module Paperclip
module MediaTypeSpoofDetectorExtensions
def calculated_content_type
return @calculated_content_type if defined?(@calculated_content_type)
@calculated_content_type = type_from_file_command.chomp
# The `file` command fails to recognize some MP3 files as such
@calculated_content_type = type_from_marcel if @calculated_content_type == 'application/octet-stream' && type_from_marcel == 'audio/mpeg'
@calculated_content_type
end
def type_from_marcel
@type_from_marcel ||= Marcel::MimeType.for Pathname.new(@file.path),
name: @file.path
end
end
end
Paperclip::MediaTypeSpoofDetector.prepend(Paperclip::MediaTypeSpoofDetectorExtensions)

View file

@ -19,10 +19,7 @@ module Paperclip
def make def make
metadata = VideoMetadataExtractor.new(@file.path) metadata = VideoMetadataExtractor.new(@file.path)
unless metadata.valid? raise Paperclip::Error, "Error while transcoding #{@file.path}: unsupported file" unless metadata.valid?
Paperclip.log("Unsupported file #{@file.path}")
return File.open(@file.path)
end
update_attachment_type(metadata) update_attachment_type(metadata)
update_options_from_metadata(metadata) update_options_from_metadata(metadata)

View file

@ -94,26 +94,26 @@ class Sanitize
] ]
) )
MASTODON_OEMBED ||= freeze_config merge( MASTODON_OEMBED ||= freeze_config(
RELAXED, elements: %w(audio embed iframe source video),
elements: RELAXED[:elements] + %w(audio embed iframe source video),
attributes: merge( attributes: {
RELAXED[:attributes],
'audio' => %w(controls), 'audio' => %w(controls),
'embed' => %w(height src type width), 'embed' => %w(height src type width),
'iframe' => %w(allowfullscreen frameborder height scrolling src width), 'iframe' => %w(allowfullscreen frameborder height scrolling src width),
'source' => %w(src type), 'source' => %w(src type),
'video' => %w(controls height loop width), 'video' => %w(controls height loop width),
'div' => [:data] },
),
protocols: merge( protocols: {
RELAXED[:protocols],
'embed' => { 'src' => HTTP_PROTOCOLS }, 'embed' => { 'src' => HTTP_PROTOCOLS },
'iframe' => { 'src' => HTTP_PROTOCOLS }, 'iframe' => { 'src' => HTTP_PROTOCOLS },
'source' => { 'src' => HTTP_PROTOCOLS } 'source' => { 'src' => HTTP_PROTOCOLS },
) },
add_attributes: {
'iframe' => { 'sandbox' => 'allow-scripts allow-same-origin allow-popups allow-popups-to-escape-sandbox allow-forms' },
}
) )
end end
end end

View file

@ -16,6 +16,7 @@ RSpec.describe Api::V1::ConversationsController, type: :controller do
before do before do
PostStatusService.new.call(other.account, text: 'Hey @alice', visibility: 'direct') PostStatusService.new.call(other.account, text: 'Hey @alice', visibility: 'direct')
PostStatusService.new.call(user.account, text: 'Hey, nobody here', visibility: 'direct')
end end
it 'returns http success' do it 'returns http success' do
@ -31,7 +32,26 @@ RSpec.describe Api::V1::ConversationsController, type: :controller do
it 'returns conversations' do it 'returns conversations' do
get :index get :index
json = body_as_json json = body_as_json
expect(json.size).to eq 1 expect(json.size).to eq 2
expect(json[0][:accounts].size).to eq 1
end
context 'with since_id' do
context 'when requesting old posts' do
it 'returns conversations' do
get :index, params: { since_id: Mastodon::Snowflake.id_at(1.hour.ago, with_random: false) }
json = body_as_json
expect(json.size).to eq 2
end
end
context 'when requesting posts in the future' do
it 'returns no conversation' do
get :index, params: { since_id: Mastodon::Snowflake.id_at(1.hour.from_now, with_random: false) }
json = body_as_json
expect(json.size).to eq 0
end
end
end end
end end
end end

View file

@ -23,6 +23,7 @@ describe Api::V1::Statuses::HistoriesController do
it 'returns http success' do it 'returns http success' do
expect(response).to have_http_status(200) expect(response).to have_http_status(200)
expect(body_as_json.size).to_not be 0
end end
end end
end end

View file

@ -69,5 +69,13 @@ RSpec.describe Api::V2::Admin::AccountsController, type: :controller do
end end
end end
end end
context 'with limit param' do
let(:params) { { limit: 1 } }
it 'sets the correct pagination headers' do
expect(response.headers['Link'].find_link(%w(rel next)).href).to eq api_v2_admin_accounts_url(limit: 1, max_id: admin_account.id)
end
end
end end
end end

BIN
spec/fixtures/files/boop.mp3 vendored Normal file

Binary file not shown.

View file

@ -0,0 +1,53 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe AccountReachFinder do
let(:account) { Fabricate(:account) }
let(:follower1) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-1') }
let(:follower2) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-2') }
let(:follower3) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://foo.bar/users/a/inbox', shared_inbox_url: 'https://foo.bar/inbox') }
let(:mentioned1) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://foo.bar/users/b/inbox', shared_inbox_url: 'https://foo.bar/inbox') }
let(:mentioned2) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-3') }
let(:mentioned3) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/inbox-4') }
let(:unrelated_account) { Fabricate(:account, protocol: :activitypub, inbox_url: 'https://example.com/unrelated-inbox') }
before do
follower1.follow!(account)
follower2.follow!(account)
follower3.follow!(account)
Fabricate(:status, account: account).tap do |status|
status.mentions << Mention.new(account: follower1)
status.mentions << Mention.new(account: mentioned1)
end
Fabricate(:status, account: account)
Fabricate(:status, account: account).tap do |status|
status.mentions << Mention.new(account: mentioned2)
status.mentions << Mention.new(account: mentioned3)
end
Fabricate(:status).tap do |status|
status.mentions << Mention.new(account: unrelated_account)
end
end
describe '#inboxes' do
it 'includes the preferred inbox URL of followers' do
expect(described_class.new(account).inboxes).to include(*[follower1, follower2, follower3].map(&:preferred_inbox_url))
end
it 'includes the preferred inbox URL of recently-mentioned accounts' do
expect(described_class.new(account).inboxes).to include(*[mentioned1, mentioned2, mentioned3].map(&:preferred_inbox_url))
end
it 'does not include the inbox of unrelated users' do
expect(described_class.new(account).inboxes).to_not include(unrelated_account.preferred_inbox_url)
end
end
end

View file

@ -150,6 +150,26 @@ RSpec.describe MediaAttachment, type: :model do
end end
end end
describe 'mp3 with large cover art' do
let(:media) { described_class.create(account: Fabricate(:account), file: attachment_fixture('boop.mp3')) }
it 'detects it as an audio file' do
expect(media.type).to eq 'audio'
end
it 'sets meta for the duration' do
expect(media.file.meta['original']['duration']).to be_within(0.05).of(0.235102)
end
it 'extracts thumbnail' do
expect(media.thumbnail.present?).to be true
end
it 'gives the file a random name' do
expect(media.file_file_name).to_not eq 'boop.mp3'
end
end
describe 'jpeg' do describe 'jpeg' do
let(:media) { MediaAttachment.create(account: Fabricate(:account), file: attachment_fixture('attachment.jpg')) } let(:media) { MediaAttachment.create(account: Fabricate(:account), file: attachment_fixture('attachment.jpg')) }

View file

@ -10,6 +10,7 @@ RSpec.describe FetchLinkCardService, type: :service do
stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt')) stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt'))
stub_request(:get, 'http://example.com/日本語').to_return(request_fixture('sjis.txt')) stub_request(:get, 'http://example.com/日本語').to_return(request_fixture('sjis.txt'))
stub_request(:get, 'https://github.com/qbi/WannaCry').to_return(status: 404) stub_request(:get, 'https://github.com/qbi/WannaCry').to_return(status: 404)
stub_request(:get, 'http://example.com/test?data=file.gpx%5E1').to_return(status: 200)
stub_request(:get, 'http://example.com/test-').to_return(request_fixture('idn.txt')) stub_request(:get, 'http://example.com/test-').to_return(request_fixture('idn.txt'))
stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt')) stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt'))
@ -85,6 +86,15 @@ RSpec.describe FetchLinkCardService, type: :service do
expect(a_request(:get, 'http://example.com/sjis')).to_not have_been_made expect(a_request(:get, 'http://example.com/sjis')).to_not have_been_made
end end
end end
context do
let(:status) { Fabricate(:status, text: 'test http://example.com/test?data=file.gpx^1') }
it 'does fetch URLs with a caret in search params' do
expect(a_request(:get, 'http://example.com/test?data=file.gpx')).to_not have_been_made
expect(a_request(:get, 'http://example.com/test?data=file.gpx%5E1')).to have_been_made.once
end
end
end end
context 'in a remote status' do context 'in a remote status' do

View file

@ -145,5 +145,35 @@ describe ResolveURLService, type: :service do
expect(subject.call(url, on_behalf_of: account)).to eq(status) expect(subject.call(url, on_behalf_of: account)).to eq(status)
end end
end end
context 'when searching for a local link of a remote private status' do
let(:account) { Fabricate(:account) }
let(:poster) { Fabricate(:account, username: 'foo', domain: 'example.com') }
let(:url) { 'https://example.com/@foo/42' }
let(:uri) { 'https://example.com/users/foo/statuses/42' }
let!(:status) { Fabricate(:status, url: url, uri: uri, account: poster, visibility: :private) }
let(:search_url) { "https://#{Rails.configuration.x.local_domain}/@foo@example.com/#{status.id}" }
before do
stub_request(:get, url).to_return(status: 404) if url.present?
stub_request(:get, uri).to_return(status: 404)
end
context 'when the account follows the poster' do
before do
account.follow!(poster)
end
it 'returns the status' do
expect(subject.call(search_url, on_behalf_of: account)).to eq(status)
end
end
context 'when the account does not follow the poster' do
it 'does not return the status' do
expect(subject.call(search_url, on_behalf_of: account)).to be_nil
end
end
end
end end
end end

View file

@ -92,18 +92,31 @@ const redisUrlToClient = async (defaultConfig, redisUrl) => {
const numWorkers = +process.env.STREAMING_CLUSTER_NUM || (env === 'development' ? 1 : Math.max(os.cpus().length - 1, 1)); const numWorkers = +process.env.STREAMING_CLUSTER_NUM || (env === 'development' ? 1 : Math.max(os.cpus().length - 1, 1));
/** /**
* Attempts to safely parse a string as JSON, used when both receiving a message
* from redis and when receiving a message from a client over a websocket
* connection, this is why it accepts a `req` argument.
* @param {string} json * @param {string} json
* @param {any} req * @param {any?} req
* @return {Object.<string, any>|null} * @returns {Object.<string, any>|null}
*/ */
const parseJSON = (json, req) => { const parseJSON = (json, req) => {
try { try {
return JSON.parse(json); return JSON.parse(json);
} catch (err) { } catch (err) {
if (req.accountId) { /* FIXME: This logging isn't great, and should probably be done at the
log.warn(req.requestId, `Error parsing message from user ${req.accountId}: ${err}`); * call-site of parseJSON, not in the method, but this would require changing
* the signature of parseJSON to return something akin to a Result type:
* [Error|null, null|Object<string,any}], and then handling the error
* scenarios.
*/
if (req) {
if (req.accountId) {
log.warn(req.requestId, `Error parsing message from user ${req.accountId}: ${err}`);
} else {
log.silly(req.requestId, `Error parsing message from ${req.remoteAddress}: ${err}`);
}
} else { } else {
log.silly(req.requestId, `Error parsing message from ${req.remoteAddress}: ${err}`); log.warn(`Error parsing message from redis: ${err}`);
} }
return null; return null;
} }
@ -169,7 +182,7 @@ const startWorker = async (workerId) => {
const redisPrefix = redisNamespace ? `${redisNamespace}:` : ''; const redisPrefix = redisNamespace ? `${redisNamespace}:` : '';
/** /**
* @type {Object.<string, Array.<function(string): void>>} * @type {Object.<string, Array.<function(Object<string, any>): void>>}
*/ */
const subs = {}; const subs = {};
@ -209,7 +222,10 @@ const startWorker = async (workerId) => {
return; return;
} }
callbacks.forEach(callback => callback(message)); const json = parseJSON(message, null);
if (!json) return;
callbacks.forEach(callback => callback(json));
}; };
/** /**
@ -231,6 +247,7 @@ const startWorker = async (workerId) => {
/** /**
* @param {string} channel * @param {string} channel
* @param {function(Object<string, any>): void} callback
*/ */
const unsubscribe = (channel, callback) => { const unsubscribe = (channel, callback) => {
log.silly(`Removing listener for ${channel}`); log.silly(`Removing listener for ${channel}`);
@ -380,7 +397,7 @@ const startWorker = async (workerId) => {
/** /**
* @param {any} req * @param {any} req
* @return {string} * @returns {string|undefined}
*/ */
const channelNameFromPath = req => { const channelNameFromPath = req => {
const { path, query } = req; const { path, query } = req;
@ -489,15 +506,11 @@ const startWorker = async (workerId) => {
/** /**
* @param {any} req * @param {any} req
* @param {SystemMessageHandlers} eventHandlers * @param {SystemMessageHandlers} eventHandlers
* @return {function(string): void} * @returns {function(object): void}
*/ */
const createSystemMessageListener = (req, eventHandlers) => { const createSystemMessageListener = (req, eventHandlers) => {
return message => { return message => {
const json = parseJSON(message, req); const { event } = message;
if (!json) return;
const { event } = json;
log.silly(req.requestId, `System message for ${req.accountId}: ${event}`); log.silly(req.requestId, `System message for ${req.accountId}: ${event}`);
@ -614,19 +627,16 @@ const startWorker = async (workerId) => {
* @param {function(string, string): void} output * @param {function(string, string): void} output
* @param {function(string[], function(string): void): void} attachCloseHandler * @param {function(string[], function(string): void): void} attachCloseHandler
* @param {boolean=} needsFiltering * @param {boolean=} needsFiltering
* @return {function(string): void} * @returns {function(object): void}
*/ */
const streamFrom = (ids, req, output, attachCloseHandler, needsFiltering = false) => { const streamFrom = (ids, req, output, attachCloseHandler, needsFiltering = false) => {
const accountId = req.accountId || req.remoteAddress; const accountId = req.accountId || req.remoteAddress;
log.verbose(req.requestId, `Starting stream from ${ids.join(', ')} for ${accountId}`); log.verbose(req.requestId, `Starting stream from ${ids.join(', ')} for ${accountId}`);
// Currently message is of type string, soon it'll be Record<string, any>
const listener = message => { const listener = message => {
const json = parseJSON(message, req); const { event, payload, queued_at } = message;
if (!json) return;
const { event, payload, queued_at } = json;
const transmit = () => { const transmit = () => {
const now = new Date().getTime(); const now = new Date().getTime();
@ -1188,8 +1198,15 @@ const startWorker = async (workerId) => {
ws.on('close', onEnd); ws.on('close', onEnd);
ws.on('error', onEnd); ws.on('error', onEnd);
ws.on('message', data => { ws.on('message', (data, isBinary) => {
const json = parseJSON(data, session.request); if (isBinary) {
log.warn('socket', 'Received binary data, closing connection');
ws.close(1003, 'The mastodon streaming server does not support binary messages');
return;
}
const message = data.toString('utf8');
const json = parseJSON(message, session.request);
if (!json) return; if (!json) return;