* Add option to overwrite imported data Fix #7465 * Add import for domain blocksmaster
@@ -12,6 +12,7 @@ | |||||
class AccountDomainBlock < ApplicationRecord | class AccountDomainBlock < ApplicationRecord | ||||
include Paginable | include Paginable | ||||
include DomainNormalizable | |||||
belongs_to :account | belongs_to :account | ||||
validates :domain, presence: true, uniqueness: { scope: :account_id } | validates :domain, presence: true, uniqueness: { scope: :account_id } | ||||
@@ -10,6 +10,6 @@ module DomainNormalizable | |||||
private | private | ||||
def normalize_domain | def normalize_domain | ||||
self.domain = TagManager.instance.normalize_domain(domain) | |||||
self.domain = TagManager.instance.normalize_domain(domain&.strip) | |||||
end | end | ||||
end | end |
@@ -1,4 +1,5 @@ | |||||
# frozen_string_literal: true | # frozen_string_literal: true | ||||
require 'csv' | require 'csv' | ||||
class Export | class Export | ||||
@@ -13,20 +13,30 @@ | |||||
# data_file_size :integer | # data_file_size :integer | ||||
# data_updated_at :datetime | # data_updated_at :datetime | ||||
# account_id :bigint(8) not null | # account_id :bigint(8) not null | ||||
# overwrite :boolean default(FALSE), not null | |||||
# | # | ||||
class Import < ApplicationRecord | class Import < ApplicationRecord | ||||
FILE_TYPES = ['text/plain', 'text/csv'].freeze | |||||
FILE_TYPES = %w(text/plain text/csv).freeze | |||||
MODES = %i(merge overwrite).freeze | |||||
self.inheritance_column = false | self.inheritance_column = false | ||||
belongs_to :account | belongs_to :account | ||||
enum type: [:following, :blocking, :muting] | |||||
enum type: [:following, :blocking, :muting, :domain_blocking] | |||||
validates :type, presence: true | validates :type, presence: true | ||||
has_attached_file :data | has_attached_file :data | ||||
validates_attachment_content_type :data, content_type: FILE_TYPES | validates_attachment_content_type :data, content_type: FILE_TYPES | ||||
validates_attachment_presence :data | validates_attachment_presence :data | ||||
def mode | |||||
overwrite? ? :overwrite : :merge | |||||
end | |||||
def mode=(str) | |||||
self.overwrite = str.to_sym == :overwrite | |||||
end | |||||
end | end |
@@ -0,0 +1,90 @@ | |||||
# frozen_string_literal: true | |||||
require 'csv' | |||||
class ImportService < BaseService | |||||
ROWS_PROCESSING_LIMIT = 20_000 | |||||
def call(import) | |||||
@import = import | |||||
@account = @import.account | |||||
@data = CSV.new(import_data).reject(&:blank?) | |||||
case @import.type | |||||
when 'following' | |||||
import_follows! | |||||
when 'blocking' | |||||
import_blocks! | |||||
when 'muting' | |||||
import_mutes! | |||||
when 'domain_blocking' | |||||
import_domain_blocks! | |||||
end | |||||
end | |||||
private | |||||
def import_follows! | |||||
import_relationships!('follow', 'unfollow', @account.following, follow_limit) | |||||
end | |||||
def import_blocks! | |||||
import_relationships!('block', 'unblock', @account.blocking, ROWS_PROCESSING_LIMIT) | |||||
end | |||||
def import_mutes! | |||||
import_relationships!('mute', 'unmute', @account.muting, ROWS_PROCESSING_LIMIT) | |||||
end | |||||
def import_domain_blocks! | |||||
items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row.first.strip } | |||||
if @import.overwrite? | |||||
presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true } | |||||
@account.domain_blocks.find_each do |domain_block| | |||||
if presence_hash[domain_block.domain] | |||||
items.delete(domain_block.domain) | |||||
else | |||||
@account.unblock_domain!(domain_block.domain) | |||||
end | |||||
end | |||||
end | |||||
items.each do |domain| | |||||
@account.block_domain!(domain) | |||||
end | |||||
AfterAccountDomainBlockWorker.push_bulk(items) do |domain| | |||||
[@account.id, domain] | |||||
end | |||||
end | |||||
def import_relationships!(action, undo_action, overwrite_scope, limit) | |||||
items = @data.take(limit).map { |row| row.first.strip } | |||||
if @import.overwrite? | |||||
presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true } | |||||
overwrite_scope.find_each do |target_account| | |||||
if presence_hash[target_account.acct] | |||||
items.delete(target_account.acct) | |||||
else | |||||
Import::RelationshipWorker.perform_async(@account.id, target_account.acct, undo_action) | |||||
end | |||||
end | |||||
end | |||||
Import::RelationshipWorker.push_bulk(items) do |acct| | |||||
[@account.id, acct, action] | |||||
end | |||||
end | |||||
def import_data | |||||
Paperclip.io_adapters.for(@import.data).read | |||||
end | |||||
def follow_limit | |||||
FollowLimitValidator.limit_for_account(@account) | |||||
end | |||||
end |
@@ -5,8 +5,11 @@ | |||||
.field-group | .field-group | ||||
= f.input :type, collection: Import.types.keys, wrapper: :with_block_label, include_blank: false, label_method: lambda { |type| I18n.t("imports.types.#{type}") }, hint: t('imports.preface') | = f.input :type, collection: Import.types.keys, wrapper: :with_block_label, include_blank: false, label_method: lambda { |type| I18n.t("imports.types.#{type}") }, hint: t('imports.preface') | ||||
.field-group | |||||
= f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data') | |||||
.fields-row | |||||
.fields-group.fields-row__column.fields-row__column-6 | |||||
= f.input :data, wrapper: :with_block_label, hint: t('simple_form.hints.imports.data') | |||||
.fields-group.fields-row__column.fields-row__column-6 | |||||
= f.input :mode, as: :radio_buttons, collection: Import::MODES, label_method: lambda { |mode| safe_join([I18n.t("imports.modes.#{mode}"), content_tag(:span, I18n.t("imports.modes.#{mode}_long"), class: 'hint')]) }, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li' | |||||
.actions | .actions | ||||
= f.button :button, t('imports.upload'), type: :submit | = f.button :button, t('imports.upload'), type: :submit |
@@ -13,11 +13,17 @@ class Import::RelationshipWorker | |||||
case relationship | case relationship | ||||
when 'follow' | when 'follow' | ||||
FollowService.new.call(from_account, target_account.acct) | |||||
FollowService.new.call(from_account, target_account) | |||||
when 'unfollow' | |||||
UnfollowService.new.call(from_account, target_account) | |||||
when 'block' | when 'block' | ||||
BlockService.new.call(from_account, target_account) | BlockService.new.call(from_account, target_account) | ||||
when 'unblock' | |||||
UnblockService.new.call(from_account, target_account) | |||||
when 'mute' | when 'mute' | ||||
MuteService.new.call(from_account, target_account) | MuteService.new.call(from_account, target_account) | ||||
when 'unmute' | |||||
UnmuteService.new.call(from_account, target_account) | |||||
end | end | ||||
rescue ActiveRecord::RecordNotFound | rescue ActiveRecord::RecordNotFound | ||||
true | true | ||||
@@ -1,44 +1,14 @@ | |||||
# frozen_string_literal: true | # frozen_string_literal: true | ||||
require 'csv' | |||||
class ImportWorker | class ImportWorker | ||||
include Sidekiq::Worker | include Sidekiq::Worker | ||||
sidekiq_options queue: 'pull', retry: false | sidekiq_options queue: 'pull', retry: false | ||||
attr_reader :import | |||||
def perform(import_id) | def perform(import_id) | ||||
@import = Import.find(import_id) | |||||
Import::RelationshipWorker.push_bulk(import_rows) do |row| | |||||
[@import.account_id, row.first, relationship_type] | |||||
end | |||||
@import.destroy | |||||
end | |||||
private | |||||
def import_contents | |||||
Paperclip.io_adapters.for(@import.data).read | |||||
end | |||||
def relationship_type | |||||
case @import.type | |||||
when 'following' | |||||
'follow' | |||||
when 'blocking' | |||||
'block' | |||||
when 'muting' | |||||
'mute' | |||||
end | |||||
end | |||||
def import_rows | |||||
rows = CSV.new(import_contents).reject(&:blank?) | |||||
rows = rows.take(FollowLimitValidator.limit_for_account(@import.account)) if @import.type == 'following' | |||||
rows | |||||
import = Import.find(import_id) | |||||
ImportService.new.call(import) | |||||
ensure | |||||
import&.destroy | |||||
end | end | ||||
end | end |
@@ -628,10 +628,16 @@ en: | |||||
one: Something isn't quite right yet! Please review the error below | one: Something isn't quite right yet! Please review the error below | ||||
other: Something isn't quite right yet! Please review %{count} errors below | other: Something isn't quite right yet! Please review %{count} errors below | ||||
imports: | imports: | ||||
modes: | |||||
merge: Merge | |||||
merge_long: Keep existing records and add new ones | |||||
overwrite: Overwrite | |||||
overwrite_long: Replace current records with the new ones | |||||
preface: You can import data that you have exported from another instance, such as a list of the people you are following or blocking. | preface: You can import data that you have exported from another instance, such as a list of the people you are following or blocking. | ||||
success: Your data was successfully uploaded and will now be processed in due time | success: Your data was successfully uploaded and will now be processed in due time | ||||
types: | types: | ||||
blocking: Blocking list | blocking: Blocking list | ||||
domain_blocking: Domain blocking list | |||||
following: Following list | following: Following list | ||||
muting: Muting list | muting: Muting list | ||||
upload: Upload | upload: Upload | ||||
@@ -0,0 +1,17 @@ | |||||
require Rails.root.join('lib', 'mastodon', 'migration_helpers') | |||||
class AddOverwriteToImports < ActiveRecord::Migration[5.2] | |||||
include Mastodon::MigrationHelpers | |||||
disable_ddl_transaction! | |||||
def up | |||||
safety_assured do | |||||
add_column_with_default :imports, :overwrite, :boolean, default: false, allow_null: false | |||||
end | |||||
end | |||||
def down | |||||
remove_column :imports, :overwrite, :boolean | |||||
end | |||||
end |
@@ -10,7 +10,7 @@ | |||||
# | # | ||||
# It's strongly recommended that you check this file into your version control system. | # It's strongly recommended that you check this file into your version control system. | ||||
ActiveRecord::Schema.define(version: 2019_01_17_114553) do | |||||
ActiveRecord::Schema.define(version: 2019_02_01_012802) do | |||||
# These are extensions that must be enabled in order to support this database | # These are extensions that must be enabled in order to support this database | ||||
enable_extension "plpgsql" | enable_extension "plpgsql" | ||||
@@ -290,6 +290,7 @@ ActiveRecord::Schema.define(version: 2019_01_17_114553) do | |||||
t.integer "data_file_size" | t.integer "data_file_size" | ||||
t.datetime "data_updated_at" | t.datetime "data_updated_at" | ||||
t.bigint "account_id", null: false | t.bigint "account_id", null: false | ||||
t.boolean "overwrite", default: false, null: false | |||||
end | end | ||||
create_table "invites", force: :cascade do |t| | create_table "invites", force: :cascade do |t| | ||||
@@ -237,9 +237,9 @@ describe AccountInteractions do | |||||
end | end | ||||
describe '#block_domain!' do | describe '#block_domain!' do | ||||
let(:domain_block) { Fabricate(:domain_block) } | |||||
let(:domain) { 'example.com' } | |||||
subject { account.block_domain!(domain_block) } | |||||
subject { account.block_domain!(domain) } | |||||
it 'creates and returns AccountDomainBlock' do | it 'creates and returns AccountDomainBlock' do | ||||
expect do | expect do | ||||