Fix remote files not using Content-Type header, streaming (#14184)

This commit is contained in:
Eugen Rochko 2020-06-30 23:58:02 +02:00 committed by GitHub
parent 65506bac3f
commit 7aaf2b44ec
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 139 additions and 68 deletions

View file

@ -0,0 +1,10 @@
# frozen_string_literal: true
class ResponseWithLimit
def initialize(response, limit)
@response = response
@limit = limit
end
attr_reader :response, :limit
end

View file

@ -8,6 +8,17 @@ module Attachmentable
MAX_MATRIX_LIMIT = 16_777_216 # 4096x4096px or approx. 16MB MAX_MATRIX_LIMIT = 16_777_216 # 4096x4096px or approx. 16MB
GIF_MATRIX_LIMIT = 921_600 # 1280x720px GIF_MATRIX_LIMIT = 921_600 # 1280x720px
# For some file extensions, there exist different content
# type variants, and browsers often send the wrong one,
# for example, sending an audio .ogg file as video/ogg,
# likewise, MimeMagic also misreports them as such. For
# those files, it is necessary to use the output of the
# `file` utility instead
INCORRECT_CONTENT_TYPES = %w(
video/ogg
video/webm
).freeze
included do included do
before_post_process :obfuscate_file_name before_post_process :obfuscate_file_name
before_post_process :set_file_extensions before_post_process :set_file_extensions
@ -21,7 +32,7 @@ module Attachmentable
self.class.attachment_definitions.each_key do |attachment_name| self.class.attachment_definitions.each_key do |attachment_name|
attachment = send(attachment_name) attachment = send(attachment_name)
next if attachment.blank? || attachment.queued_for_write[:original].blank? next if attachment.blank? || attachment.queued_for_write[:original].blank? || !INCORRECT_CONTENT_TYPES.include?(attachment.instance_read(:content_type))
attachment.instance_write :content_type, calculated_content_type(attachment) attachment.instance_write :content_type, calculated_content_type(attachment)
end end
@ -63,9 +74,7 @@ module Attachmentable
end end
def calculated_content_type(attachment) def calculated_content_type(attachment)
content_type = Paperclip.run('file', '-b --mime :file', file: attachment.queued_for_write[:original].path).split(/[:;\s]+/).first.chomp Paperclip.run('file', '-b --mime :file', file: attachment.queued_for_write[:original].path).split(/[:;\s]+/).first.chomp
content_type = 'video/mp4' if content_type == 'video/x-m4v'
content_type
rescue Terrapin::CommandLineError rescue Terrapin::CommandLineError
'' ''
end end

View file

@ -24,28 +24,16 @@ module Remotable
Request.new(:get, url).perform do |response| Request.new(:get, url).perform do |response|
raise Mastodon::UnexpectedResponseError, response unless (200...300).cover?(response.code) raise Mastodon::UnexpectedResponseError, response unless (200...300).cover?(response.code)
content_type = parse_content_type(response.headers.get('content-type').last) public_send("#{attachment_name}=", ResponseWithLimit.new(response, limit))
extname = detect_extname_from_content_type(content_type)
if extname.nil?
disposition = response.headers.get('content-disposition').last
matches = disposition&.match(/filename="([^"]*)"/)
filename = matches.nil? ? parsed_url.path.split('/').last : matches[1]
extname = filename.nil? ? '' : File.extname(filename)
end
basename = SecureRandom.hex(8)
public_send("#{attachment_name}_file_name=", basename + extname)
public_send("#{attachment_name}=", StringIO.new(response.body_with_limit(limit)))
end end
rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError => e rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError => e
Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}" Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}"
raise e unless suppress_errors raise e unless suppress_errors
rescue Paperclip::Errors::NotIdentifiedByImageMagickError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError, Paperclip::Error, Mastodon::DimensionsValidationError => e rescue Paperclip::Errors::NotIdentifiedByImageMagickError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError, Paperclip::Error, Mastodon::DimensionsValidationError => e
Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}" Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}"
nil
end end
nil
end end
define_method("#{attribute_name}=") do |url| define_method("#{attribute_name}=") do |url|
@ -59,26 +47,4 @@ module Remotable
alias_method("reset_#{attachment_name}!", "download_#{attachment_name}!") alias_method("reset_#{attachment_name}!", "download_#{attachment_name}!")
end end
end end
private
def detect_extname_from_content_type(content_type)
return if content_type.nil?
type = MIME::Types[content_type].first
return if type.nil?
extname = type.extensions.first
return if extname.nil?
".#{extname}"
end
def parse_content_type(content_type)
return if content_type.nil?
content_type.split(/\s*;\s*/).first
end
end end

View file

@ -9,10 +9,12 @@ Bundler.require(*Rails.groups)
require_relative '../app/lib/exceptions' require_relative '../app/lib/exceptions'
require_relative '../lib/paperclip/url_generator_extensions' require_relative '../lib/paperclip/url_generator_extensions'
require_relative '../lib/paperclip/attachment_extensions' require_relative '../lib/paperclip/attachment_extensions'
require_relative '../lib/paperclip/media_type_spoof_detector_extensions'
require_relative '../lib/paperclip/lazy_thumbnail' require_relative '../lib/paperclip/lazy_thumbnail'
require_relative '../lib/paperclip/gif_transcoder' require_relative '../lib/paperclip/gif_transcoder'
require_relative '../lib/paperclip/video_transcoder' require_relative '../lib/paperclip/video_transcoder'
require_relative '../lib/paperclip/type_corrector' require_relative '../lib/paperclip/type_corrector'
require_relative '../lib/paperclip/response_with_limit_adapter'
require_relative '../lib/mastodon/snowflake' require_relative '../lib/mastodon/snowflake'
require_relative '../lib/mastodon/version' require_relative '../lib/mastodon/version'
require_relative '../lib/devise/two_factor_ldap_authenticatable' require_relative '../lib/devise/two_factor_ldap_authenticatable'

View file

@ -1,6 +1,7 @@
# frozen_string_literal: true # frozen_string_literal: true
Paperclip::DataUriAdapter.register Paperclip::DataUriAdapter.register
Paperclip::ResponseWithLimitAdapter.register
Paperclip.interpolates :filename do |attachment, style| Paperclip.interpolates :filename do |attachment, style|
if style == :original if style == :original

View file

@ -4,28 +4,10 @@ require 'mime/types/columnar'
module Paperclip module Paperclip
class ImageExtractor < Paperclip::Processor class ImageExtractor < Paperclip::Processor
IMAGE_EXTRACTION_OPTIONS = {
convert_options: {
output: {
'loglevel' => 'fatal',
vf: 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease',
}.freeze,
}.freeze,
format: 'png',
time: -1,
file_geometry_parser: FastGeometryParser,
}.freeze
def make def make
return @file unless options[:style] == :original return @file unless options[:style] == :original
image = begin image = extract_image_from_file!
begin
Paperclip::Transcoder.make(file, IMAGE_EXTRACTION_OPTIONS.dup, attachment)
rescue Paperclip::Error, ::Av::CommandError
nil
end
end
unless image.nil? unless image.nil?
begin begin
@ -36,7 +18,7 @@ module Paperclip
# to make sure it's cleaned up # to make sure it's cleaned up
begin begin
FileUtils.rm(image) image.close(true)
rescue Errno::ENOENT rescue Errno::ENOENT
nil nil
end end
@ -45,5 +27,28 @@ module Paperclip
@file @file
end end
private
def extract_image_from_file!
::Av.logger = Paperclip.logger
cli = ::Av.cli
dst = Tempfile.new([File.basename(@file.path, '.*'), '.png'])
dst.binmode
cli.add_source(@file.path)
cli.add_destination(dst.path)
cli.add_output_param loglevel: 'fatal'
begin
cli.run
rescue Cocaine::ExitStatusError
dst.close(true)
return nil
end
dst
end
end end
end end

View file

@ -0,0 +1,27 @@
# frozen_string_literal: true
module Paperclip
module MediaTypeSpoofDetectorExtensions
def calculated_content_type
@calculated_content_type ||= type_from_mime_magic || type_from_file_command
end
def type_from_mime_magic
@type_from_mime_magic ||= begin
begin
File.open(@file.path) do |file|
MimeMagic.by_magic(file)&.type
end
rescue Errno::ENOENT
''
end
end
end
def type_from_file_command
@type_from_file_command ||= FileCommandContentTypeDetector.new(@file.path).detect
end
end
end
Paperclip::MediaTypeSpoofDetector.prepend(Paperclip::MediaTypeSpoofDetectorExtensions)

View file

@ -0,0 +1,55 @@
# frozen_string_literal: true
module Paperclip
class ResponseWithLimitAdapter < AbstractAdapter
def self.register
Paperclip.io_adapters.register self do |target|
target.is_a?(ResponseWithLimit)
end
end
def initialize(target, options = {})
super
cache_current_values
end
private
def cache_current_values
@original_filename = filename_from_content_disposition || filename_from_path || 'data'
@size = @target.response.content_length
@tempfile = copy_to_tempfile(@target)
@content_type = @target.response.mime_type || ContentTypeDetector.new(@tempfile.path).detect
end
def copy_to_tempfile(source)
bytes_read = 0
source.response.body.each do |chunk|
bytes_read += chunk.bytesize
destination.write(chunk)
chunk.clear
raise Mastodon::LengthValidationError if bytes_read > source.limit
end
destination.rewind
destination
rescue Mastodon::LengthValidationError
destination.close(true)
raise
ensure
source.response.connection.close
end
def filename_from_content_disposition
disposition = @target.response.headers['content-disposition']
disposition&.match(/filename="([^"]*)"/)&.captures&.first
end
def filename_from_path
@target.response.uri.path.split('/').last
end
end
end

View file

@ -162,19 +162,15 @@ RSpec.describe Remotable do
let(:headers) { { 'content-disposition' => file } } let(:headers) { { 'content-disposition' => file } }
it 'assigns file' do it 'assigns file' do
string_io = StringIO.new('') response_with_limit = ResponseWithLimit.new(nil, 0)
extname = '.txt'
basename = '0123456789abcdef'
allow(SecureRandom).to receive(:hex).and_return(basename) allow(ResponseWithLimit).to receive(:new).with(anything, anything).and_return(response_with_limit)
allow(StringIO).to receive(:new).with(anything).and_return(string_io)
expect(foo).to receive(:public_send).with("download_#{hoge}!", url) expect(foo).to receive(:public_send).with("download_#{hoge}!", url)
foo.hoge_remote_url = url foo.hoge_remote_url = url
expect(foo).to receive(:public_send).with("#{hoge}=", string_io) expect(foo).to receive(:public_send).with("#{hoge}=", response_with_limit)
expect(foo).to receive(:public_send).with("#{hoge}_file_name=", basename + extname)
foo.download_hoge!(url) foo.download_hoge!(url)
end end