From f722bd2387df9163760014e9555928ec487ae95f Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 4 Apr 2017 00:53:20 +0200 Subject: [PATCH] Separate background jobs into different queues. ATTENTION: new queue "pull" must be added to the Sidekiq invokation in your systemd file The pull queue will handle link crawling, thread resolving, and OStatus processing. Such tasks are more likely to hang for a longer time (due to network requests) so it is more sensible to not make the "in-house" tasks wait for them. --- app/workers/after_remote_follow_request_worker.rb | 2 +- app/workers/after_remote_follow_worker.rb | 2 +- app/workers/import_worker.rb | 2 +- app/workers/link_crawl_worker.rb | 2 +- app/workers/merge_worker.rb | 2 ++ app/workers/notification_worker.rb | 2 +- app/workers/processing_worker.rb | 2 +- app/workers/regeneration_worker.rb | 2 ++ app/workers/salmon_worker.rb | 2 +- app/workers/thread_resolve_worker.rb | 2 +- app/workers/unmerge_worker.rb | 2 ++ docker-compose.yml | 2 +- docs/Running-Mastodon/Production-guide.md | 2 +- 13 files changed, 16 insertions(+), 10 deletions(-) diff --git a/app/workers/after_remote_follow_request_worker.rb b/app/workers/after_remote_follow_request_worker.rb index f1d6869..1f2db30 100644 --- a/app/workers/after_remote_follow_request_worker.rb +++ b/app/workers/after_remote_follow_request_worker.rb @@ -3,7 +3,7 @@ class AfterRemoteFollowRequestWorker include Sidekiq::Worker - sidekiq_options retry: 5 + sidekiq_options queue: 'pull', retry: 5 def perform(follow_request_id) follow_request = FollowRequest.find(follow_request_id) diff --git a/app/workers/after_remote_follow_worker.rb b/app/workers/after_remote_follow_worker.rb index 0d04456..bdd2c2a 100644 --- a/app/workers/after_remote_follow_worker.rb +++ b/app/workers/after_remote_follow_worker.rb @@ -3,7 +3,7 @@ class AfterRemoteFollowWorker include Sidekiq::Worker - sidekiq_options retry: 5 + sidekiq_options queue: 'pull', retry: 5 def perform(follow_id) follow = Follow.find(follow_id) diff --git a/app/workers/import_worker.rb b/app/workers/import_worker.rb index a3ae2a8..7cf29fb 100644 --- a/app/workers/import_worker.rb +++ b/app/workers/import_worker.rb @@ -5,7 +5,7 @@ require 'csv' class ImportWorker include Sidekiq::Worker - sidekiq_options retry: false + sidekiq_options queue: 'pull', retry: false def perform(import_id) import = Import.find(import_id) diff --git a/app/workers/link_crawl_worker.rb b/app/workers/link_crawl_worker.rb index af3394b..834b008 100644 --- a/app/workers/link_crawl_worker.rb +++ b/app/workers/link_crawl_worker.rb @@ -3,7 +3,7 @@ class LinkCrawlWorker include Sidekiq::Worker - sidekiq_options retry: false + sidekiq_options queue: 'pull', retry: false def perform(status_id) FetchLinkCardService.new.call(Status.find(status_id)) diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb index 0f288f4..d745cb9 100644 --- a/app/workers/merge_worker.rb +++ b/app/workers/merge_worker.rb @@ -3,6 +3,8 @@ class MergeWorker include Sidekiq::Worker + sidekiq_options queue: 'pull' + def perform(from_account_id, into_account_id) FeedManager.instance.merge_into_timeline(Account.find(from_account_id), Account.find(into_account_id)) end diff --git a/app/workers/notification_worker.rb b/app/workers/notification_worker.rb index 1a2faef..da1d6ab 100644 --- a/app/workers/notification_worker.rb +++ b/app/workers/notification_worker.rb @@ -3,7 +3,7 @@ class NotificationWorker include Sidekiq::Worker - sidekiq_options retry: 5 + sidekiq_options queue: 'push', retry: 5 def perform(xml, source_account_id, target_account_id) SendInteractionService.new.call(xml, Account.find(source_account_id), Account.find(target_account_id)) diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb index 5df404b..4a467d9 100644 --- a/app/workers/processing_worker.rb +++ b/app/workers/processing_worker.rb @@ -3,7 +3,7 @@ class ProcessingWorker include Sidekiq::Worker - sidekiq_options backtrace: true + sidekiq_options queue: 'pull', backtrace: true def perform(account_id, body) ProcessFeedService.new.call(body, Account.find(account_id)) diff --git a/app/workers/regeneration_worker.rb b/app/workers/regeneration_worker.rb index 3aece0b..289b63d 100644 --- a/app/workers/regeneration_worker.rb +++ b/app/workers/regeneration_worker.rb @@ -3,6 +3,8 @@ class RegenerationWorker include Sidekiq::Worker + sidekiq_options queue: 'pull', backtrace: true + def perform(account_id, timeline_type) PrecomputeFeedService.new.call(timeline_type, Account.find(account_id)) end diff --git a/app/workers/salmon_worker.rb b/app/workers/salmon_worker.rb index fc95ce4..2888b57 100644 --- a/app/workers/salmon_worker.rb +++ b/app/workers/salmon_worker.rb @@ -3,7 +3,7 @@ class SalmonWorker include Sidekiq::Worker - sidekiq_options backtrace: true + sidekiq_options queue: 'pull', backtrace: true def perform(account_id, body) ProcessInteractionService.new.call(body, Account.find(account_id)) diff --git a/app/workers/thread_resolve_worker.rb b/app/workers/thread_resolve_worker.rb index 593edd0..38287e8 100644 --- a/app/workers/thread_resolve_worker.rb +++ b/app/workers/thread_resolve_worker.rb @@ -3,7 +3,7 @@ class ThreadResolveWorker include Sidekiq::Worker - sidekiq_options retry: false + sidekiq_options queue: 'pull', retry: false def perform(child_status_id, parent_url) child_status = Status.find(child_status_id) diff --git a/app/workers/unmerge_worker.rb b/app/workers/unmerge_worker.rb index dbf7243..ea6aace 100644 --- a/app/workers/unmerge_worker.rb +++ b/app/workers/unmerge_worker.rb @@ -3,6 +3,8 @@ class UnmergeWorker include Sidekiq::Worker + sidekiq_options queue: 'pull' + def perform(from_account_id, into_account_id) FeedManager.instance.unmerge_from_timeline(Account.find(from_account_id), Account.find(into_account_id)) end diff --git a/docker-compose.yml b/docker-compose.yml index 68c8ef9..d6ba66d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,7 +33,7 @@ services: restart: always build: . env_file: .env.production - command: bundle exec sidekiq -q default -q mailers -q push + command: bundle exec sidekiq -q default -q mailers -q pull -q push depends_on: - db - redis diff --git a/docs/Running-Mastodon/Production-guide.md b/docs/Running-Mastodon/Production-guide.md index f0dd7bd..469fefa 100644 --- a/docs/Running-Mastodon/Production-guide.md +++ b/docs/Running-Mastodon/Production-guide.md @@ -180,7 +180,7 @@ User=mastodon WorkingDirectory=/home/mastodon/live Environment="RAILS_ENV=production" Environment="DB_POOL=5" -ExecStart=/home/mastodon/.rbenv/shims/bundle exec sidekiq -c 5 -q default -q mailers -q push +ExecStart=/home/mastodon/.rbenv/shims/bundle exec sidekiq -c 5 -q default -q mailers -q pull -q push TimeoutSec=15 Restart=always