Add import/export feature for bookmarks (#14956)

* Add ability to export bookmarks

* Add support for importing bookmarks

* Add bookmark import tests

* Add bookmarks export test
This commit is contained in:
ThibG 2020-11-19 17:48:13 +01:00 committed by GitHub
parent 022d2353a7
commit 96c1e71329
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 147 additions and 1 deletions

View file

@ -0,0 +1,19 @@
# frozen_string_literal: true
module Settings
module Exports
class BookmarksController < BaseController
include ExportControllerConcern
def index
send_export_file
end
private
def export_data
@export.to_bookmarks_csv
end
end
end
end

View file

@ -9,6 +9,14 @@ class Export
@account = account @account = account
end end
def to_bookmarks_csv
CSV.generate do |csv|
account.bookmarks.includes(:status).reorder(id: :desc).each do |bookmark|
csv << [ActivityPub::TagManager.instance.uri_for(bookmark.status)]
end
end
end
def to_blocked_accounts_csv def to_blocked_accounts_csv
to_csv account.blocking.select(:username, :domain) to_csv account.blocking.select(:username, :domain)
end end
@ -55,6 +63,10 @@ class Export
account.statuses_count account.statuses_count
end end
def total_bookmarks
account.bookmarks.count
end
def total_follows def total_follows
account.following_count account.following_count
end end

View file

@ -24,7 +24,7 @@ class Import < ApplicationRecord
belongs_to :account belongs_to :account
enum type: [:following, :blocking, :muting, :domain_blocking] enum type: [:following, :blocking, :muting, :domain_blocking, :bookmarks]
validates :type, presence: true validates :type, presence: true

View file

@ -18,6 +18,8 @@ class ImportService < BaseService
import_mutes! import_mutes!
when 'domain_blocking' when 'domain_blocking'
import_domain_blocks! import_domain_blocks!
when 'bookmarks'
import_bookmarks!
end end
end end
@ -88,6 +90,39 @@ class ImportService < BaseService
end end
end end
def import_bookmarks!
parse_import_data!(['#uri'])
items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row['#uri'].strip }
if @import.overwrite?
presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true }
@account.bookmarks.find_each do |bookmark|
if presence_hash[bookmark.status.uri]
items.delete(bookmark.status.uri)
else
bookmark.destroy!
end
end
end
statuses = items.map do |uri|
status = ActivityPub::TagManager.instance.uri_to_resource(uri, Status)
next if status.nil? && ActivityPub::TagManager.instance.local_uri?(uri)
status || ActivityPub::FetchRemoteStatusService.new.call(uri)
end.compact
account_ids = statuses.map(&:account_id)
preloaded_relations = relations_map_for_account(@account, account_ids)
statuses.keep_if { |status| StatusPolicy.new(@account, status, preloaded_relations).show? }
statuses.each do |status|
@account.bookmarks.find_or_create_by!(account: @account, status: status)
end
end
def parse_import_data!(default_headers) def parse_import_data!(default_headers)
data = CSV.parse(import_data, headers: true) data = CSV.parse(import_data, headers: true)
data = CSV.parse(import_data, headers: default_headers) unless data.headers&.first&.strip&.include?(' ') data = CSV.parse(import_data, headers: default_headers) unless data.headers&.first&.strip&.include?(' ')
@ -101,4 +136,14 @@ class ImportService < BaseService
def follow_limit def follow_limit
FollowLimitValidator.limit_for_account(@account) FollowLimitValidator.limit_for_account(@account)
end end
def relations_map_for_account(account, account_ids)
{
blocking: {},
blocked_by: Account.blocked_by_map(account_ids, account.id),
muting: {},
following: Account.following_map(account_ids, account.id),
domain_blocking_by_domain: {},
}
end
end end

View file

@ -36,6 +36,10 @@
%th= t('exports.domain_blocks') %th= t('exports.domain_blocks')
%td= number_with_delimiter @export.total_domain_blocks %td= number_with_delimiter @export.total_domain_blocks
%td= table_link_to 'download', t('exports.csv'), settings_exports_domain_blocks_path(format: :csv) %td= table_link_to 'download', t('exports.csv'), settings_exports_domain_blocks_path(format: :csv)
%tr
%th= t('exports.bookmarks')
%td= number_with_delimiter @export.total_bookmarks
%td= table_link_to 'download', t('bookmarks.csv'), settings_exports_bookmarks_path(format: :csv)
%hr.spacer/ %hr.spacer/

View file

@ -842,6 +842,7 @@ en:
request: Request your archive request: Request your archive
size: Size size: Size
blocks: You block blocks: You block
bookmarks: Bookmarks
csv: CSV csv: CSV
domain_blocks: Domain blocks domain_blocks: Domain blocks
lists: Lists lists: Lists
@ -918,6 +919,7 @@ en:
success: Your data was successfully uploaded and will now be processed in due time success: Your data was successfully uploaded and will now be processed in due time
types: types:
blocking: Blocking list blocking: Blocking list
bookmarks: Bookmarks
domain_blocking: Domain blocking list domain_blocking: Domain blocking list
following: Following list following: Following list
muting: Muting list muting: Muting list

View file

@ -125,6 +125,7 @@ Rails.application.routes.draw do
resources :mutes, only: :index, controller: :muted_accounts resources :mutes, only: :index, controller: :muted_accounts
resources :lists, only: :index, controller: :lists resources :lists, only: :index, controller: :lists
resources :domain_blocks, only: :index, controller: :blocked_domains resources :domain_blocks, only: :index, controller: :blocked_domains
resources :bookmarks, only: :index, controller: :bookmarks
end end
resources :two_factor_authentication_methods, only: [:index] do resources :two_factor_authentication_methods, only: [:index] do

View file

@ -0,0 +1,17 @@
require 'rails_helper'
describe Settings::Exports::BookmarksController do
render_views
describe 'GET #index' do
it 'returns a csv of the bookmarked toots' do
user = Fabricate(:user)
user.account.bookmarks.create!(status: Fabricate(:status, uri: 'https://foo.bar/statuses/1312'))
sign_in user, scope: :user
get :index, format: :csv
expect(response.body).to eq "https://foo.bar/statuses/1312\n"
end
end
end

View file

@ -0,0 +1,4 @@
https://example.com/statuses/1312
https://local.com/users/foo/statuses/42
https://unknown-remote.com/users/bar/statuses/1
https://example.com/statuses/direct

View file

@ -1,6 +1,8 @@
require 'rails_helper' require 'rails_helper'
RSpec.describe ImportService, type: :service do RSpec.describe ImportService, type: :service do
include RoutingHelper
let!(:account) { Fabricate(:account, locked: false) } let!(:account) { Fabricate(:account, locked: false) }
let!(:bob) { Fabricate(:account, username: 'bob', locked: false) } let!(:bob) { Fabricate(:account, username: 'bob', locked: false) }
let!(:eve) { Fabricate(:account, username: 'eve', domain: 'example.com', locked: false, protocol: :activitypub, inbox_url: 'https://example.com/inbox') } let!(:eve) { Fabricate(:account, username: 'eve', domain: 'example.com', locked: false, protocol: :activitypub, inbox_url: 'https://example.com/inbox') }
@ -169,4 +171,44 @@ RSpec.describe ImportService, type: :service do
end end
end end
end end
context 'import bookmarks' do
subject { ImportService.new }
let(:csv) { attachment_fixture('bookmark-imports.txt') }
around(:each) do |example|
local_before = Rails.configuration.x.local_domain
web_before = Rails.configuration.x.web_domain
Rails.configuration.x.local_domain = 'local.com'
Rails.configuration.x.web_domain = 'local.com'
example.run
Rails.configuration.x.web_domain = web_before
Rails.configuration.x.local_domain = local_before
end
let(:local_account) { Fabricate(:account, username: 'foo', domain: '') }
let!(:remote_status) { Fabricate(:status, uri: 'https://example.com/statuses/1312') }
let!(:direct_status) { Fabricate(:status, uri: 'https://example.com/statuses/direct', visibility: :direct) }
before do
service = double
allow(ActivityPub::FetchRemoteStatusService).to receive(:new).and_return(service)
allow(service).to receive(:call).with('https://unknown-remote.com/users/bar/statuses/1') do
Fabricate(:status, uri: 'https://unknown-remote.com/users/bar/statuses/1')
end
end
describe 'when no bookmarks are set' do
let(:import) { Import.create(account: account, type: 'bookmarks', data: csv) }
it 'adds the toots the user has access to to bookmarks' do
local_status = Fabricate(:status, account: local_account, uri: 'https://local.com/users/foo/statuses/42', id: 42, local: true)
subject.call(import)
expect(account.bookmarks.map(&:status).map(&:id)).to include(local_status.id)
expect(account.bookmarks.map(&:status).map(&:id)).to include(remote_status.id)
expect(account.bookmarks.map(&:status).map(&:id)).not_to include(direct_status.id)
expect(account.bookmarks.count).to eq 3
end
end
end
end end