* Add structure for lists * Add list timeline streaming API * Add list APIs, bind list-account relation to follow relation * Add API for adding/removing accounts from lists * Add pagination to lists API * Add pagination to list accounts API * Adjust scopes for new APIs - Creating and modifying lists merely requires "write" scope - Fetching information about lists merely requires "read" scope * Add test for wrong user context on list timeline * Clean up testsmaster
@@ -0,0 +1,81 @@ | |||
# frozen_string_literal: true | |||
class Api::V1::Lists::AccountsController < Api::BaseController | |||
before_action -> { doorkeeper_authorize! :read }, only: [:show] | |||
before_action -> { doorkeeper_authorize! :write }, except: [:show] | |||
before_action :require_user! | |||
before_action :set_list | |||
after_action :insert_pagination_headers, only: :show | |||
def show | |||
@accounts = @list.accounts.paginate_by_max_id(limit_param(DEFAULT_ACCOUNTS_LIMIT), params[:max_id], params[:since_id]) | |||
render json: @accounts, each_serializer: REST::AccountSerializer | |||
end | |||
def create | |||
ApplicationRecord.transaction do | |||
list_accounts.each do |account| | |||
@list.accounts << account | |||
end | |||
end | |||
render_empty | |||
end | |||
def destroy | |||
ListAccount.where(list: @list, account_id: account_ids).destroy_all | |||
render_empty | |||
end | |||
private | |||
def set_list | |||
@list = List.where(account: current_account).find(params[:list_id]) | |||
end | |||
def list_accounts | |||
Account.find(account_ids) | |||
end | |||
def account_ids | |||
Array(resource_params[:account_ids]) | |||
end | |||
def resource_params | |||
params.permit(account_ids: []) | |||
end | |||
def insert_pagination_headers | |||
set_pagination_headers(next_path, prev_path) | |||
end | |||
def next_path | |||
if records_continue? | |||
api_v1_list_accounts_url pagination_params(max_id: pagination_max_id) | |||
end | |||
end | |||
def prev_path | |||
unless @accounts.empty? | |||
api_v1_list_accounts_url pagination_params(since_id: pagination_since_id) | |||
end | |||
end | |||
def pagination_max_id | |||
@accounts.last.id | |||
end | |||
def pagination_since_id | |||
@accounts.first.id | |||
end | |||
def records_continue? | |||
@accounts.size == limit_param(DEFAULT_ACCOUNTS_LIMIT) | |||
end | |||
def pagination_params(core_params) | |||
params.permit(:limit).merge(core_params) | |||
end | |||
end |
@@ -0,0 +1,79 @@ | |||
# frozen_string_literal: true | |||
class Api::V1::ListsController < Api::BaseController | |||
LISTS_LIMIT = 50 | |||
before_action -> { doorkeeper_authorize! :read }, only: [:index, :show] | |||
before_action -> { doorkeeper_authorize! :write }, except: [:index, :show] | |||
before_action :require_user! | |||
before_action :set_list, except: [:index, :create] | |||
after_action :insert_pagination_headers, only: :index | |||
def index | |||
@lists = List.where(account: current_account).paginate_by_max_id(limit_param(LISTS_LIMIT), params[:max_id], params[:since_id]) | |||
render json: @lists, each_serializer: REST::ListSerializer | |||
end | |||
def show | |||
render json: @list, serializer: REST::ListSerializer | |||
end | |||
def create | |||
@list = List.create!(list_params.merge(account: current_account)) | |||
render json: @list, serializer: REST::ListSerializer | |||
end | |||
def update | |||
@list.update!(list_params) | |||
render json: @list, serializer: REST::ListSerializer | |||
end | |||
def destroy | |||
@list.destroy! | |||
render_empty | |||
end | |||
private | |||
def set_list | |||
@list = List.where(account: current_account).find(params[:id]) | |||
end | |||
def list_params | |||
params.permit(:title) | |||
end | |||
def insert_pagination_headers | |||
set_pagination_headers(next_path, prev_path) | |||
end | |||
def next_path | |||
if records_continue? | |||
api_v1_lists_url pagination_params(max_id: pagination_max_id) | |||
end | |||
end | |||
def prev_path | |||
unless @lists.empty? | |||
api_v1_lists_url pagination_params(since_id: pagination_since_id) | |||
end | |||
end | |||
def pagination_max_id | |||
@lists.last.id | |||
end | |||
def pagination_since_id | |||
@lists.first.id | |||
end | |||
def records_continue? | |||
@lists.size == limit_param(LISTS_LIMIT) | |||
end | |||
def pagination_params(core_params) | |||
params.permit(:limit).merge(core_params) | |||
end | |||
end |
@@ -31,7 +31,7 @@ class Api::V1::Timelines::HomeController < Api::BaseController | |||
end | |||
def account_home_feed | |||
Feed.new(:home, current_account) | |||
HomeFeed.new(current_account) | |||
end | |||
def insert_pagination_headers | |||
@@ -0,0 +1,66 @@ | |||
# frozen_string_literal: true | |||
class Api::V1::Timelines::ListController < Api::BaseController | |||
before_action -> { doorkeeper_authorize! :read } | |||
before_action :require_user! | |||
before_action :set_list | |||
before_action :set_statuses | |||
after_action :insert_pagination_headers, unless: -> { @statuses.empty? } | |||
def show | |||
render json: @statuses, | |||
each_serializer: REST::StatusSerializer, | |||
relationships: StatusRelationshipsPresenter.new(@statuses, current_user.account_id) | |||
end | |||
private | |||
def set_list | |||
@list = List.where(account: current_account).find(params[:id]) | |||
end | |||
def set_statuses | |||
@statuses = cached_list_statuses | |||
end | |||
def cached_list_statuses | |||
cache_collection list_statuses, Status | |||
end | |||
def list_statuses | |||
list_feed.get( | |||
limit_param(DEFAULT_STATUSES_LIMIT), | |||
params[:max_id], | |||
params[:since_id] | |||
) | |||
end | |||
def list_feed | |||
ListFeed.new(@list) | |||
end | |||
def insert_pagination_headers | |||
set_pagination_headers(next_path, prev_path) | |||
end | |||
def pagination_params(core_params) | |||
params.permit(:limit).merge(core_params) | |||
end | |||
def next_path | |||
api_v1_timelines_list_url params[:id], pagination_params(max_id: pagination_max_id) | |||
end | |||
def prev_path | |||
api_v1_timelines_list_url params[:id], pagination_params(since_id: pagination_since_id) | |||
end | |||
def pagination_max_id | |||
@statuses.last.id | |||
end | |||
def pagination_since_id | |||
@statuses.first.id | |||
end | |||
end |
@@ -26,34 +26,42 @@ class FeedManager | |||
end | |||
end | |||
def push(timeline_type, account, status) | |||
return false unless add_to_feed(timeline_type, account, status) | |||
trim(timeline_type, account.id) | |||
PushUpdateWorker.perform_async(account.id, status.id) if push_update_required?(timeline_type, account.id) | |||
def push_to_home(account, status) | |||
return false unless add_to_feed(:home, account.id, status) | |||
trim(:home, account.id) | |||
PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}") if push_update_required?("timeline:#{account.id}") | |||
true | |||
end | |||
def unpush(timeline_type, account, status) | |||
return false unless remove_from_feed(timeline_type, account, status) | |||
def unpush_from_home(account, status) | |||
return false unless remove_from_feed(:home, account.id, status) | |||
Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) | |||
true | |||
end | |||
payload = Oj.dump(event: :delete, payload: status.id.to_s) | |||
Redis.current.publish("timeline:#{account.id}", payload) | |||
def push_to_list(list, status) | |||
return false unless add_to_feed(:list, list.id, status) | |||
trim(:list, list.id) | |||
PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}") if push_update_required?("timeline:list:#{list.id}") | |||
true | |||
end | |||
def unpush_from_list(list, status) | |||
return false unless remove_from_feed(:list, list.id, status) | |||
Redis.current.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s)) | |||
true | |||
end | |||
def trim(type, account_id) | |||
timeline_key = key(type, account_id) | |||
reblog_key = key(type, account_id, 'reblogs') | |||
reblog_key = key(type, account_id, 'reblogs') | |||
# Remove any items past the MAX_ITEMS'th entry in our feed | |||
redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s) | |||
# Get the score of the REBLOG_FALLOFF'th item in our feed, and stop | |||
# tracking anything after it for deduplication purposes. | |||
falloff_rank = FeedManager::REBLOG_FALLOFF - 1 | |||
falloff_rank = FeedManager::REBLOG_FALLOFF - 1 | |||
falloff_range = redis.zrevrange(timeline_key, falloff_rank, falloff_rank, with_scores: true) | |||
falloff_score = falloff_range&.first&.last&.to_i || 0 | |||
@@ -69,10 +77,6 @@ class FeedManager | |||
end | |||
end | |||
def push_update_required?(timeline_type, account_id) | |||
timeline_type != :home || redis.get("subscribed:timeline:#{account_id}").present? | |||
end | |||
def merge_into_timeline(from_account, into_account) | |||
timeline_key = key(:home, into_account.id) | |||
query = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4) | |||
@@ -84,28 +88,28 @@ class FeedManager | |||
query.each do |status| | |||
next if status.direct_visibility? || filter?(:home, status, into_account) | |||
add_to_feed(:home, into_account, status) | |||
add_to_feed(:home, into_account.id, status) | |||
end | |||
trim(:home, into_account.id) | |||
end | |||
def unmerge_from_timeline(from_account, into_account) | |||
timeline_key = key(:home, into_account.id) | |||
timeline_key = key(:home, into_account.id) | |||
oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0 | |||
from_account.statuses.select('id, reblog_of_id').where('id > ?', oldest_home_score).reorder(nil).find_each do |status| | |||
remove_from_feed(:home, into_account, status) | |||
remove_from_feed(:home, into_account.id, status) | |||
end | |||
end | |||
def clear_from_timeline(account, target_account) | |||
timeline_key = key(:home, account.id) | |||
timeline_key = key(:home, account.id) | |||
timeline_status_ids = redis.zrange(timeline_key, 0, -1) | |||
target_statuses = Status.where(id: timeline_status_ids, account: target_account) | |||
target_statuses = Status.where(id: timeline_status_ids, account: target_account) | |||
target_statuses.each do |status| | |||
unpush(:home, account, status) | |||
unpush_from_home(account, status) | |||
end | |||
end | |||
@@ -122,7 +126,7 @@ class FeedManager | |||
statuses.each do |status| | |||
next if filter_from_home?(status, account) | |||
added += 1 if add_to_feed(:home, account, status) | |||
added += 1 if add_to_feed(:home, account.id, status) | |||
end | |||
break unless added.zero? | |||
@@ -137,6 +141,10 @@ class FeedManager | |||
Redis.current | |||
end | |||
def push_update_required?(timeline_id) | |||
redis.exists("subscribed:#{timeline_id}") | |||
end | |||
def filter_from_home?(status, receiver_id) | |||
return false if receiver_id == status.account_id | |||
return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?) | |||
@@ -182,9 +190,9 @@ class FeedManager | |||
# added, and false if it was not added to the feed. Note that this is | |||
# an internal helper: callers must call trim or push updates if | |||
# either action is appropriate. | |||
def add_to_feed(timeline_type, account, status) | |||
timeline_key = key(timeline_type, account.id) | |||
reblog_key = key(timeline_type, account.id, 'reblogs') | |||
def add_to_feed(timeline_type, account_id, status) | |||
timeline_key = key(timeline_type, account_id) | |||
reblog_key = key(timeline_type, account_id, 'reblogs') | |||
if status.reblog? | |||
# If the original status or a reblog of it is within | |||
@@ -195,6 +203,7 @@ class FeedManager | |||
return false if !rank.nil? && rank < FeedManager::REBLOG_FALLOFF | |||
reblog_rank = redis.zrevrank(reblog_key, status.reblog_of_id) | |||
if reblog_rank.nil? | |||
# This is not something we've already seen reblogged, so we | |||
# can just add it to the feed (and note that we're | |||
@@ -205,7 +214,7 @@ class FeedManager | |||
# Another reblog of the same status was already in the | |||
# REBLOG_FALLOFF most recent statuses, so we note that this | |||
# is an "extra" reblog, by storing it in reblog_set_key. | |||
reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}") | |||
reblog_set_key = key(timeline_type, account_id, "reblogs:#{status.reblog_of_id}") | |||
redis.sadd(reblog_set_key, status.id) | |||
return false | |||
end | |||
@@ -220,8 +229,8 @@ class FeedManager | |||
# with reblogs, and returning true if a status was removed. As with | |||
# `add_to_feed`, this does not trigger push updates, so callers must | |||
# do so if appropriate. | |||
def remove_from_feed(timeline_type, account, status) | |||
timeline_key = key(timeline_type, account.id) | |||
def remove_from_feed(timeline_type, account_id, status) | |||
timeline_key = key(timeline_type, account_id) | |||
if status.reblog? | |||
# 1. If the reblogging status is not in the feed, stop. | |||
@@ -229,7 +238,7 @@ class FeedManager | |||
return false if status_rank.nil? | |||
# 2. Remove reblog from set of this status's reblogs. | |||
reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}") | |||
reblog_set_key = key(timeline_type, account_id, "reblogs:#{status.reblog_of_id}") | |||
redis.srem(reblog_set_key, status.id) | |||
# 3. Re-insert another reblog or original into the feed if one | |||
@@ -244,7 +253,7 @@ class FeedManager | |||
# (outside conditional) | |||
else | |||
# If the original is getting deleted, no use for reblog references | |||
redis.del(key(timeline_type, account.id, "reblogs:#{status.id}")) | |||
redis.del(key(timeline_type, account_id, "reblogs:#{status.id}")) | |||
end | |||
redis.zrem(timeline_key, status.id) | |||
@@ -3,7 +3,7 @@ | |||
# | |||
# Table name: accounts | |||
# | |||
# id :bigint not null, primary key | |||
# id :integer not null, primary key | |||
# username :string default(""), not null | |||
# domain :string | |||
# secret :string default(""), not null | |||
@@ -53,6 +53,7 @@ class Account < ApplicationRecord | |||
include AccountInteractions | |||
include Attachmentable | |||
include Remotable | |||
include Paginable | |||
enum protocol: [:ostatus, :activitypub] | |||
@@ -95,6 +96,10 @@ class Account < ApplicationRecord | |||
has_many :account_moderation_notes, dependent: :destroy | |||
has_many :targeted_moderation_notes, class_name: 'AccountModerationNote', foreign_key: :target_account_id, dependent: :destroy | |||
# Lists | |||
has_many :list_accounts, inverse_of: :account, dependent: :destroy | |||
has_many :lists, through: :list_accounts | |||
scope :remote, -> { where.not(domain: nil) } | |||
scope :local, -> { where(domain: nil) } | |||
scope :without_followers, -> { where(followers_count: 0) } | |||
@@ -3,11 +3,11 @@ | |||
# | |||
# Table name: account_domain_blocks | |||
# | |||
# id :integer not null, primary key | |||
# domain :string | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# account_id :bigint | |||
# id :bigint not null, primary key | |||
# account_id :integer | |||
# | |||
class AccountDomainBlock < ApplicationRecord | |||
@@ -3,10 +3,10 @@ | |||
# | |||
# Table name: account_moderation_notes | |||
# | |||
# id :bigint not null, primary key | |||
# id :integer not null, primary key | |||
# content :text not null | |||
# account_id :bigint not null | |||
# target_account_id :bigint not null | |||
# account_id :integer not null | |||
# target_account_id :integer not null | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# | |||
@@ -3,11 +3,11 @@ | |||
# | |||
# Table name: blocks | |||
# | |||
# id :integer not null, primary key | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# account_id :bigint not null | |||
# id :bigint not null, primary key | |||
# target_account_id :bigint not null | |||
# account_id :integer not null | |||
# target_account_id :integer not null | |||
# | |||
class Block < ApplicationRecord | |||
@@ -3,7 +3,7 @@ | |||
# | |||
# Table name: conversations | |||
# | |||
# id :bigint not null, primary key | |||
# id :integer not null, primary key | |||
# uri :string | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
@@ -3,9 +3,9 @@ | |||
# | |||
# Table name: conversation_mutes | |||
# | |||
# conversation_id :bigint not null | |||
# account_id :bigint not null | |||
# id :bigint not null, primary key | |||
# id :integer not null, primary key | |||
# conversation_id :integer not null | |||
# account_id :integer not null | |||
# | |||
class ConversationMute < ApplicationRecord | |||
@@ -3,7 +3,7 @@ | |||
# | |||
# Table name: custom_emojis | |||
# | |||
# id :bigint not null, primary key | |||
# id :integer not null, primary key | |||
# shortcode :string default(""), not null | |||
# domain :string | |||
# image_file_name :string | |||
@@ -3,12 +3,12 @@ | |||
# | |||
# Table name: domain_blocks | |||
# | |||
# id :integer not null, primary key | |||
# domain :string default(""), not null | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# severity :integer default("silence") | |||
# reject_media :boolean default(FALSE), not null | |||
# id :bigint not null, primary key | |||
# | |||
class DomainBlock < ApplicationRecord | |||
@@ -3,7 +3,7 @@ | |||
# | |||
# Table name: email_domain_blocks | |||
# | |||
# id :bigint not null, primary key | |||
# id :integer not null, primary key | |||
# domain :string default(""), not null | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
@@ -3,11 +3,11 @@ | |||
# | |||
# Table name: favourites | |||
# | |||
# id :integer not null, primary key | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# account_id :bigint not null | |||
# id :bigint not null, primary key | |||
# status_id :bigint not null | |||
# account_id :integer not null | |||
# status_id :integer not null | |||
# | |||
class Favourite < ApplicationRecord | |||
@@ -1,36 +1,27 @@ | |||
# frozen_string_literal: true | |||
class Feed | |||
def initialize(type, account) | |||
@type = type | |||
@account = account | |||
def initialize(type, id) | |||
@type = type | |||
@id = id | |||
end | |||
def get(limit, max_id = nil, since_id = nil) | |||
if redis.exists("account:#{@account.id}:regeneration") | |||
from_database(limit, max_id, since_id) | |||
else | |||
from_redis(limit, max_id, since_id) | |||
end | |||
from_redis(limit, max_id, since_id) | |||
end | |||
private | |||
protected | |||
def from_redis(limit, max_id, since_id) | |||
max_id = '+inf' if max_id.blank? | |||
since_id = '-inf' if since_id.blank? | |||
unhydrated = redis.zrevrangebyscore(key, "(#{max_id}", "(#{since_id}", limit: [0, limit], with_scores: true).map(&:first).map(&:to_i) | |||
Status.where(id: unhydrated).cache_ids | |||
end | |||
def from_database(limit, max_id, since_id) | |||
Status.as_home_timeline(@account) | |||
.paginate_by_max_id(limit, max_id, since_id) | |||
.reject { |status| FeedManager.instance.filter?(:home, status, @account.id) } | |||
Status.where(id: unhydrated).cache_ids | |||
end | |||
def key | |||
FeedManager.instance.key(@type, @account.id) | |||
FeedManager.instance.key(@type, @id) | |||
end | |||
def redis | |||
@@ -3,11 +3,11 @@ | |||
# | |||
# Table name: follows | |||
# | |||
# id :integer not null, primary key | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# account_id :bigint not null | |||
# id :bigint not null, primary key | |||
# target_account_id :bigint not null | |||
# account_id :integer not null | |||
# target_account_id :integer not null | |||
# | |||
class Follow < ApplicationRecord | |||
@@ -3,11 +3,11 @@ | |||
# | |||
# Table name: follow_requests | |||
# | |||
# id :integer not null, primary key | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# account_id :bigint not null | |||
# id :bigint not null, primary key | |||
# target_account_id :bigint not null | |||
# account_id :integer not null | |||
# target_account_id :integer not null | |||
# | |||
class FollowRequest < ApplicationRecord | |||
@@ -0,0 +1,25 @@ | |||
# frozen_string_literal: true | |||
class HomeFeed < Feed | |||
def initialize(account) | |||
@type = :home | |||
@id = account.id | |||
@account = account | |||
end | |||
def get(limit, max_id = nil, since_id = nil) | |||
if redis.exists("account:#{@account.id}:regeneration") | |||
from_database(limit, max_id, since_id) | |||
else | |||
super | |||
end | |||
end | |||
private | |||
def from_database(limit, max_id, since_id) | |||
Status.as_home_timeline(@account) | |||
.paginate_by_max_id(limit, max_id, since_id) | |||
.reject { |status| FeedManager.instance.filter?(:home, status, @account.id) } | |||
end | |||
end |
@@ -3,6 +3,7 @@ | |||
# | |||
# Table name: imports | |||
# | |||
# id :integer not null, primary key | |||
# type :integer not null | |||
# approved :boolean default(FALSE), not null | |||
# created_at :datetime not null | |||
@@ -11,8 +12,7 @@ | |||
# data_content_type :string | |||
# data_file_size :integer | |||
# data_updated_at :datetime | |||
# account_id :bigint not null | |||
# id :bigint not null, primary key | |||
# account_id :integer not null | |||
# | |||
class Import < ApplicationRecord | |||
@@ -0,0 +1,22 @@ | |||
# frozen_string_literal: true | |||
# == Schema Information | |||
# | |||
# Table name: lists | |||
# | |||
# id :integer not null, primary key | |||
# account_id :integer | |||
# title :string default(""), not null | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# | |||
class List < ApplicationRecord | |||
include Paginable | |||
belongs_to :account | |||
has_many :list_accounts, inverse_of: :list, dependent: :destroy | |||
has_many :accounts, through: :list_accounts | |||
validates :title, presence: true | |||
end |
@@ -0,0 +1,24 @@ | |||
# frozen_string_literal: true | |||
# == Schema Information | |||
# | |||
# Table name: list_accounts | |||
# | |||
# id :integer not null, primary key | |||
# list_id :integer not null | |||
# account_id :integer not null | |||
# follow_id :integer not null | |||
# | |||
class ListAccount < ApplicationRecord | |||
belongs_to :list, required: true | |||
belongs_to :account, required: true | |||
belongs_to :follow, required: true | |||
before_validation :set_follow | |||
private | |||
def set_follow | |||
self.follow = Follow.find_by(account_id: list.account_id, target_account_id: account.id) | |||
end | |||
end |
@@ -0,0 +1,8 @@ | |||
# frozen_string_literal: true | |||
class ListFeed < Feed | |||
def initialize(list) | |||
@type = :list | |||
@id = list.id | |||
end | |||
end |
@@ -3,19 +3,19 @@ | |||
# | |||
# Table name: media_attachments | |||
# | |||
# id :bigint not null, primary key | |||
# status_id :bigint | |||
# id :integer not null, primary key | |||
# status_id :integer | |||
# file_file_name :string | |||
# file_content_type :string | |||
# file_file_size :integer | |||
# file_updated_at :datetime | |||
# remote_url :string default(""), not null | |||
# account_id :bigint | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# shortcode :string | |||
# type :integer default("image"), not null | |||
# file_meta :json | |||
# account_id :integer | |||
# description :text | |||
# | |||
@@ -3,11 +3,11 @@ | |||
# | |||
# Table name: mentions | |||
# | |||
# status_id :bigint | |||
# id :integer not null, primary key | |||
# status_id :integer | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# account_id :bigint | |||
# id :bigint not null, primary key | |||
# account_id :integer | |||
# | |||
class Mention < ApplicationRecord | |||
@@ -3,13 +3,13 @@ | |||
# | |||
# Table name: notifications | |||
# | |||
# id :bigint not null, primary key | |||
# account_id :bigint | |||
# activity_id :bigint | |||
# id :integer not null, primary key | |||
# activity_id :integer | |||
# activity_type :string | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# from_account_id :bigint | |||
# account_id :integer | |||
# from_account_id :integer | |||
# | |||
class Notification < ApplicationRecord | |||
@@ -3,7 +3,7 @@ | |||
# | |||
# Table name: preview_cards | |||
# | |||
# id :bigint not null, primary key | |||
# id :integer not null, primary key | |||
# url :string default(""), not null | |||
# title :string default(""), not null | |||
# description :string default(""), not null | |||
@@ -3,15 +3,15 @@ | |||
# | |||
# Table name: reports | |||
# | |||
# id :integer not null, primary key | |||
# status_ids :integer default([]), not null, is an Array | |||
# comment :text default(""), not null | |||
# action_taken :boolean default(FALSE), not null | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# account_id :bigint not null | |||
# action_taken_by_account_id :bigint | |||
# id :bigint not null, primary key | |||
# target_account_id :bigint not null | |||
# account_id :integer not null | |||
# action_taken_by_account_id :integer | |||
# target_account_id :integer not null | |||
# | |||
class Report < ApplicationRecord | |||
@@ -3,15 +3,15 @@ | |||
# | |||
# Table name: session_activations | |||
# | |||
# id :bigint not null, primary key | |||
# user_id :bigint not null | |||
# id :integer not null, primary key | |||
# session_id :string not null | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# user_agent :string default(""), not null | |||
# ip :inet | |||
# access_token_id :bigint | |||
# web_push_subscription_id :bigint | |||
# access_token_id :integer | |||
# user_id :integer not null | |||
# web_push_subscription_id :integer | |||
# | |||
# id :bigint not null, primary key | |||
@@ -3,13 +3,13 @@ | |||
# | |||
# Table name: settings | |||
# | |||
# id :integer not null, primary key | |||
# var :string not null | |||
# value :text | |||
# thing_type :string | |||
# created_at :datetime | |||
# updated_at :datetime | |||
# id :bigint not null, primary key | |||
# thing_id :bigint | |||
# thing_id :integer | |||
# | |||
class Setting < RailsSettings::Base | |||
@@ -3,7 +3,7 @@ | |||
# | |||
# Table name: site_uploads | |||
# | |||
# id :bigint not null, primary key | |||
# id :integer not null, primary key | |||
# var :string default(""), not null | |||
# file_file_name :string | |||
# file_content_type :string | |||
@@ -3,26 +3,26 @@ | |||
# | |||
# Table name: statuses | |||
# | |||
# id :bigint not null, primary key | |||
# id :integer not null, primary key | |||
# uri :string | |||
# account_id :bigint not null | |||
# text :text default(""), not null | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# in_reply_to_id :bigint | |||
# reblog_of_id :bigint | |||
# in_reply_to_id :integer | |||
# reblog_of_id :integer | |||
# url :string | |||
# sensitive :boolean default(FALSE), not null | |||
# visibility :integer default("public"), not null | |||
# in_reply_to_account_id :bigint | |||
# application_id :bigint | |||
# spoiler_text :text default(""), not null | |||
# reply :boolean default(FALSE), not null | |||
# favourites_count :integer default(0), not null | |||
# reblogs_count :integer default(0), not null | |||
# language :string | |||
# conversation_id :bigint | |||
# conversation_id :integer | |||
# local :boolean | |||
# account_id :integer not null | |||
# application_id :integer | |||
# in_reply_to_account_id :integer | |||
# | |||
class Status < ApplicationRecord | |||
@@ -3,9 +3,9 @@ | |||
# | |||
# Table name: status_pins | |||
# | |||
# id :bigint not null, primary key | |||
# account_id :bigint not null | |||
# status_id :bigint not null | |||
# id :integer not null, primary key | |||
# account_id :integer not null | |||
# status_id :integer not null | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# | |||
@@ -3,13 +3,13 @@ | |||
# | |||
# Table name: stream_entries | |||
# | |||
# activity_id :bigint | |||
# id :integer not null, primary key | |||
# activity_id :integer | |||
# activity_type :string | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# hidden :boolean default(FALSE), not null | |||
# account_id :bigint | |||
# id :bigint not null, primary key | |||
# account_id :integer | |||
# | |||
class StreamEntry < ApplicationRecord | |||
@@ -3,6 +3,7 @@ | |||
# | |||
# Table name: subscriptions | |||
# | |||
# id :integer not null, primary key | |||
# callback_url :string default(""), not null | |||
# secret :string | |||
# expires_at :datetime | |||
@@ -11,8 +12,7 @@ | |||
# updated_at :datetime not null | |||
# last_successful_delivery_at :datetime | |||
# domain :string | |||
# account_id :bigint not null | |||
# id :bigint not null, primary key | |||
# account_id :integer not null | |||
# | |||
class Subscription < ApplicationRecord | |||
@@ -3,7 +3,7 @@ | |||
# | |||
# Table name: tags | |||
# | |||
# id :bigint not null, primary key | |||
# id :integer not null, primary key | |||
# name :string default(""), not null | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
@@ -3,7 +3,7 @@ | |||
# | |||
# Table name: users | |||
# | |||
# id :bigint not null, primary key | |||
# id :integer not null, primary key | |||
# email :string default(""), not null | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
@@ -30,7 +30,7 @@ | |||
# last_emailed_at :datetime | |||
# otp_backup_codes :string is an Array | |||
# filtered_languages :string default([]), not null, is an Array | |||
# account_id :bigint not null | |||
# account_id :integer not null | |||
# disabled :boolean default(FALSE), not null | |||
# moderator :boolean default(FALSE), not null | |||
# | |||
@@ -3,7 +3,7 @@ | |||
# | |||
# Table name: web_push_subscriptions | |||
# | |||
# id :bigint not null, primary key | |||
# id :integer not null, primary key | |||
# endpoint :string not null | |||
# key_p256dh :string not null | |||
# key_auth :string not null | |||
@@ -3,11 +3,11 @@ | |||
# | |||
# Table name: web_settings | |||
# | |||
# id :integer not null, primary key | |||
# data :json | |||
# created_at :datetime not null | |||
# updated_at :datetime not null | |||
# id :bigint not null, primary key | |||
# user_id :bigint | |||
# user_id :integer | |||
# | |||
class Web::Setting < ApplicationRecord | |||
@@ -0,0 +1,5 @@ | |||
# frozen_string_literal: true | |||
class REST::ListSerializer < ActiveModel::Serializer | |||
attributes :id, :title | |||
end |
@@ -30,6 +30,7 @@ class BatchedRemoveStatusService < BaseService | |||
account = account_statuses.first.account | |||
unpush_from_home_timelines(account, account_statuses) | |||
unpush_from_list_timelines(account, account_statuses) | |||
if account.local? | |||
batch_stream_entries(account, account_statuses) | |||
@@ -79,7 +80,15 @@ class BatchedRemoveStatusService < BaseService | |||
recipients.each do |follower| | |||
statuses.each do |status| | |||
FeedManager.instance.unpush(:home, follower, status) | |||
FeedManager.instance.unpush_from_home(follower, status) | |||
end | |||
end | |||
end | |||
def unpush_from_list_timelines(account, statuses) | |||
account.lists.select(:id, :account_id).each do |list| | |||
statuses.each do |status| | |||
FeedManager.instance.unpush_from_list(list, status) | |||
end | |||
end | |||
end | |||
@@ -14,6 +14,7 @@ class FanOutOnWriteService < BaseService | |||
deliver_to_mentioned_followers(status) | |||
else | |||
deliver_to_followers(status) | |||
deliver_to_lists(status) | |||
end | |||
return if status.account.silenced? || !status.public_visibility? || status.reblog? | |||
@@ -30,7 +31,7 @@ class FanOutOnWriteService < BaseService | |||
def deliver_to_self(status) | |||
Rails.logger.debug "Delivering status #{status.id} to author" | |||
FeedManager.instance.push(:home, status.account, status) | |||
FeedManager.instance.push_to_home(status.account, status) | |||
end | |||
def deliver_to_followers(status) | |||
@@ -38,7 +39,17 @@ class FanOutOnWriteService < BaseService | |||
status.account.followers.where(domain: nil).joins(:user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_in_batches do |followers| | |||
FeedInsertWorker.push_bulk(followers) do |follower| | |||
[status.id, follower.id] | |||
[status.id, follower.id, :home] | |||
end | |||
end | |||
end | |||
def deliver_to_lists(status) | |||
Rails.logger.debug "Delivering status #{status.id} to lists" | |||
status.account.lists.joins(account: :user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_in_batches do |lists| | |||
FeedInsertWorker.push_bulk(lists) do |list| | |||
[status.id, list.id, :list] | |||
end | |||
end | |||
end | |||
@@ -49,7 +60,7 @@ class FanOutOnWriteService < BaseService | |||
status.mentions.includes(:account).each do |mention| | |||
mentioned_account = mention.account | |||
next if !mentioned_account.local? || !mentioned_account.following?(status.account) || FeedManager.instance.filter?(:home, status, mention.account_id) | |||
FeedManager.instance.push(:home, mentioned_account, status) | |||
FeedManager.instance.push_to_home(mentioned_account, status) | |||
end | |||
end | |||
@@ -14,6 +14,7 @@ class RemoveStatusService < BaseService | |||
remove_from_self if status.account.local? | |||
remove_from_followers | |||
remove_from_lists | |||
remove_from_affected | |||
remove_reblogs | |||
remove_from_hashtags | |||
@@ -30,12 +31,18 @@ class RemoveStatusService < BaseService | |||
private | |||
def remove_from_self | |||
unpush(:home, @account, @status) | |||
FeedManager.instance.unpush_from_home(@account, @status) | |||
end | |||
def remove_from_followers | |||
@account.followers.local.find_each do |follower| | |||
unpush(:home, follower, @status) | |||
FeedManager.instance.unpush_from_home(follower, @status) | |||
end | |||
end | |||
def remove_from_lists | |||
@account.lists.select(:id, :account_id).find_each do |list| | |||
FeedManager.instance.unpush_from_list(list, @status) | |||
end | |||
end | |||
@@ -101,10 +108,6 @@ class RemoveStatusService < BaseService | |||
end | |||
end | |||
def unpush(type, receiver, status) | |||
FeedManager.instance.unpush(type, receiver, status) | |||
end | |||
def remove_from_hashtags | |||
return unless @status.public_visibility? | |||
@@ -3,34 +3,41 @@ | |||
class FeedInsertWorker | |||
include Sidekiq::Worker | |||
attr_reader :status, :follower | |||
def perform(status_id, follower_id) | |||
@status = Status.find_by(id: status_id) | |||
@follower = Account.find_by(id: follower_id) | |||
def perform(status_id, id, type = :home) | |||
@type = type.to_sym | |||
@status = Status.find(status_id) | |||
case @type | |||
when :home | |||
@follower = Account.find(id) | |||
when :list | |||
@list = List.find(id) | |||
@follower = @list.account | |||
end | |||
check_and_insert | |||
rescue ActiveRecord::RecordNotFound | |||
true | |||
end | |||
private | |||
def check_and_insert | |||
if records_available? | |||
perform_push unless feed_filtered? | |||
else | |||
true | |||
end | |||
end | |||
def records_available? | |||
status.present? && follower.present? | |||
perform_push unless feed_filtered? | |||
end | |||
def feed_filtered? | |||
FeedManager.instance.filter?(:home, status, follower.id) | |||
# Note: Lists are a variation of home, so the filtering rules | |||
# of home apply to both | |||
FeedManager.instance.filter?(:home, @status, @follower.id) | |||
end | |||
def perform_push | |||
FeedManager.instance.push(:home, follower, status) | |||
case @type | |||
when :home | |||
FeedManager.instance.push_to_home(@follower, @status) | |||
when :list | |||
FeedManager.instance.push_to_list(@list, @status) | |||
end | |||
end | |||
end |
@@ -3,12 +3,13 @@ | |||
class PushUpdateWorker | |||
include Sidekiq::Worker | |||
def perform(account_id, status_id) | |||
account = Account.find(account_id) | |||
status = Status.find(status_id) | |||
message = InlineRenderer.render(status, account, :status) | |||
def perform(account_id, status_id, timeline_id = nil) | |||
account = Account.find(account_id) | |||
status = Status.find(status_id) | |||
message = InlineRenderer.render(status, account, :status) | |||
timeline_id = "timeline:#{account.id}" if timeline_id.nil? | |||
Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :update, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i)) | |||
Redis.current.publish(timeline_id, Oj.dump(event: :update, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i)) | |||
rescue ActiveRecord::RecordNotFound | |||
true | |||
end | |||
@@ -212,6 +212,7 @@ Rails.application.routes.draw do | |||
resource :home, only: :show, controller: :home | |||
resource :public, only: :show, controller: :public | |||
resources :tag, only: :show | |||
resources :list, only: :show | |||
end | |||
resources :streaming, only: [:index] | |||
@@ -270,6 +271,10 @@ Rails.application.routes.draw do | |||
post :unmute | |||
end | |||
end | |||
resources :lists, only: [:index, :create, :show, :update, :destroy] do | |||
resource :accounts, only: [:show, :create, :destroy], controller: 'lists/accounts' | |||
end | |||
end | |||
namespace :web do | |||
@@ -0,0 +1,10 @@ | |||
class CreateLists < ActiveRecord::Migration[5.1] | |||
def change | |||
create_table :lists do |t| | |||
t.references :account, foreign_key: { on_delete: :cascade } | |||
t.string :title, null: false, default: '' | |||
t.timestamps | |||
end | |||
end | |||
end |
@@ -0,0 +1,12 @@ | |||
class CreateListAccounts < ActiveRecord::Migration[5.1] | |||
def change | |||
create_table :list_accounts do |t| | |||
t.belongs_to :list, foreign_key: { on_delete: :cascade }, null: false | |||
t.belongs_to :account, foreign_key: { on_delete: :cascade }, null: false | |||
t.belongs_to :follow, foreign_key: { on_delete: :cascade }, null: false | |||
end | |||
add_index :list_accounts, [:account_id, :list_id], unique: true | |||
add_index :list_accounts, [:list_id, :account_id] | |||
end | |||
end |
@@ -10,7 +10,7 @@ | |||
# | |||
# It's strongly recommended that you check this file into your version control system. | |||
ActiveRecord::Schema.define(version: 20171114080328) do | |||
ActiveRecord::Schema.define(version: 20171116161857) do | |||
# These are extensions that must be enabled in order to support this database | |||
enable_extension "plpgsql" | |||
@@ -170,6 +170,25 @@ ActiveRecord::Schema.define(version: 20171114080328) do | |||
t.bigint "account_id", null: false | |||
end | |||
create_table "list_accounts", force: :cascade do |t| | |||
t.bigint "list_id", null: false | |||
t.bigint "account_id", null: false | |||
t.bigint "follow_id", null: false | |||
t.index ["account_id", "list_id"], name: "index_list_accounts_on_account_id_and_list_id", unique: true | |||
t.index ["account_id"], name: "index_list_accounts_on_account_id" | |||
t.index ["follow_id"], name: "index_list_accounts_on_follow_id" | |||
t.index ["list_id", "account_id"], name: "index_list_accounts_on_list_id_and_account_id" | |||
t.index ["list_id"], name: "index_list_accounts_on_list_id" | |||
end | |||
create_table "lists", force: :cascade do |t| | |||
t.bigint "account_id" | |||
t.string "title", default: "", null: false | |||
t.datetime "created_at", null: false | |||
t.datetime "updated_at", null: false | |||
t.index ["account_id"], name: "index_lists_on_account_id" | |||
end | |||
create_table "media_attachments", force: :cascade do |t| | |||
t.bigint "status_id" | |||
t.string "file_file_name" | |||
@@ -478,6 +497,10 @@ ActiveRecord::Schema.define(version: 20171114080328) do | |||
add_foreign_key "follows", "accounts", column: "target_account_id", name: "fk_745ca29eac", on_delete: :cascade | |||
add_foreign_key "follows", "accounts", name: "fk_32ed1b5560", on_delete: :cascade | |||
add_foreign_key "imports", "accounts", name: "fk_6db1b6e408", on_delete: :cascade | |||
add_foreign_key "list_accounts", "accounts", on_delete: :cascade | |||
add_foreign_key "list_accounts", "follows", on_delete: :cascade | |||
add_foreign_key "list_accounts", "lists", on_delete: :cascade | |||
add_foreign_key "lists", "accounts", on_delete: :cascade | |||
add_foreign_key "media_attachments", "accounts", name: "fk_96dd81e81b", on_delete: :nullify | |||
add_foreign_key "media_attachments", "statuses", on_delete: :nullify | |||
add_foreign_key "mentions", "accounts", name: "fk_970d43f9d1", on_delete: :cascade | |||
@@ -0,0 +1,54 @@ | |||
require 'rails_helper' | |||
describe Api::V1::Lists::AccountsController do | |||
render_views | |||
let(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) } | |||
let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read write') } | |||
let(:list) { Fabricate(:list, account: user.account) } | |||
before do | |||
follow = Fabricate(:follow, account: user.account) | |||
list.accounts << follow.target_account | |||
allow(controller).to receive(:doorkeeper_token) { token } | |||
end | |||
describe 'GET #index' do | |||
it 'returns http success' do | |||
get :show, params: { list_id: list.id } | |||
expect(response).to have_http_status(:success) | |||
end | |||
end | |||
describe 'POST #create' do | |||
let(:bob) { Fabricate(:account, username: 'bob') } | |||
before do | |||
user.account.follow!(bob) | |||
post :create, params: { list_id: list.id, account_ids: [bob.id] } | |||
end | |||
it 'returns http success' do | |||
expect(response).to have_http_status(:success) | |||
end | |||
it 'adds account to the list' do | |||
expect(list.accounts.include?(bob)).to be true | |||
end | |||
end | |||
describe 'DELETE #destroy' do | |||
before do | |||
delete :destroy, params: { list_id: list.id, account_ids: [list.accounts.first.id] } | |||
end | |||
it 'returns http success' do | |||
expect(response).to have_http_status(:success) | |||
end | |||
it 'removes account from the list' do | |||
expect(list.accounts.count).to eq 0 | |||
end | |||
end | |||
end |
@@ -0,0 +1,68 @@ | |||
require 'rails_helper' | |||
RSpec.describe Api::V1::ListsController, type: :controller do | |||
render_views | |||
let!(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) } | |||
let!(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read write') } | |||
let!(:list) { Fabricate(:list, account: user.account) } | |||
before { allow(controller).to receive(:doorkeeper_token) { token } } | |||
describe 'GET #index' do | |||
it 'returns http success' do | |||
get :index | |||
expect(response).to have_http_status(:success) | |||
end | |||
end | |||
describe 'GET #show' do | |||
it 'returns http success' do | |||
get :show, params: { id: list.id } | |||
expect(response).to have_http_status(:success) | |||
end | |||
end | |||
describe 'POST #create' do | |||
before do | |||
post :create, params: { title: 'Foo bar' } | |||
end | |||
it 'returns http success' do | |||
expect(response).to have_http_status(:success) | |||
end | |||
it 'creates list' do | |||
expect(List.where(account: user.account).count).to eq 2 | |||
expect(List.last.title).to eq 'Foo bar' | |||
end | |||
end | |||
describe 'PUT #update' do | |||
before do | |||
put :update, params: { id: list.id, title: 'Updated title' } | |||
end | |||
it 'returns http success' do | |||
expect(response).to have_http_status(:success) | |||
end | |||
it 'updates the list' do | |||
expect(list.reload.title).to eq 'Updated title' | |||
end | |||
end | |||
describe 'DELETE #destroy' do | |||
before do | |||
delete :destroy, params: { id: list.id } | |||
end | |||
it 'returns http success' do | |||
expect(response).to have_http_status(:success) | |||
end | |||
it 'deletes the list' do | |||
expect(List.find_by(id: list.id)).to be_nil | |||
end | |||
end | |||
end |
@@ -0,0 +1,56 @@ | |||
# frozen_string_literal: true | |||
require 'rails_helper' | |||
describe Api::V1::Timelines::ListController do | |||
render_views | |||
let(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) } | |||
let(:list) { Fabricate(:list, account: user.account) } | |||
before do | |||
allow(controller).to receive(:doorkeeper_token) { token } | |||
end | |||
context 'with a user context' do | |||
let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read') } | |||
describe 'GET #show' do | |||
before do | |||
follow = Fabricate(:follow, account: user.account) | |||
list.accounts << follow.target_account | |||
PostStatusService.new.call(follow.target_account, 'New status for user home timeline.') | |||
end | |||
it 'returns http success' do | |||
get :show, params: { id: list.id } | |||
expect(response).to have_http_status(:success) | |||
end | |||
end | |||
end | |||
context 'with the wrong user context' do | |||
let(:other_user) { Fabricate(:user, account: Fabricate(:account, username: 'bob')) } | |||
let(:token) { Fabricate(:accessible_access_token, resource_owner_id: other_user.id, scopes: 'read') } | |||
describe 'GET #show' do | |||
it 'returns http not found' do | |||
get :show, params: { id: list.id } | |||
expect(response).to have_http_status(:not_found) | |||
end | |||
end | |||
end | |||
context 'without a user context' do | |||
let(:token) { Fabricate(:accessible_access_token, resource_owner_id: nil, scopes: 'read') } | |||
describe 'GET #show' do | |||
it 'returns http unprocessable entity' do | |||
get :show, params: { id: list.id } | |||
expect(response).to have_http_status(:unprocessable_entity) | |||
expect(response.headers['Link']).to be_nil | |||
end | |||
end | |||
end | |||
end |
@@ -5,7 +5,7 @@ require 'rails_helper' | |||
describe Api::V1::Timelines::TagController do | |||
render_views | |||
let(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) } | |||
let(:user) { Fabricate(:user, account: Fabricate(:account, username: 'alice')) } | |||
before do | |||
allow(controller).to receive(:doorkeeper_token) { token } | |||
@@ -0,0 +1,5 @@ | |||
Fabricator(:list_account) do | |||
list nil | |||
account nil | |||
follow nil | |||
end |
@@ -0,0 +1,4 @@ | |||
Fabricator(:list) do | |||
account nil | |||
title "MyString" | |||
end |
@@ -148,21 +148,11 @@ RSpec.describe FeedManager do | |||
account = Fabricate(:account) | |||
status = Fabricate(:status) | |||
members = FeedManager::MAX_ITEMS.times.map { |count| [count, count] } | |||
Redis.current.zadd("feed:type:#{account.id}", members) | |||
Redis.current.zadd("feed:home:#{account.id}", members) | |||
FeedManager.instance.push('type', account, status) | |||
FeedManager.instance.push_to_home(account, status) | |||
expect(Redis.current.zcard("feed:type:#{account.id}")).to eq FeedManager::MAX_ITEMS | |||
end | |||
it 'sends push updates for non-home timelines' do | |||
account = Fabricate(:account) | |||
status = Fabricate(:status) | |||
allow(Redis.current).to receive_messages(publish: nil) | |||
FeedManager.instance.push('type', account, status) | |||
expect(Redis.current).to have_received(:publish).with("timeline:#{account.id}", any_args).at_least(:once) | |||
expect(Redis.current.zcard("feed:home:#{account.id}")).to eq FeedManager::MAX_ITEMS | |||
end | |||
context 'reblogs' do | |||
@@ -171,7 +161,7 @@ RSpec.describe FeedManager do | |||
reblogged = Fabricate(:status) | |||
reblog = Fabricate(:status, reblog: reblogged) | |||
expect(FeedManager.instance.push('type', account, reblog)).to be true | |||
expect(FeedManager.instance.push_to_home(account, reblog)).to be true | |||
end | |||
it 'does not save a new reblog of a recent status' do | |||
@@ -179,9 +169,9 @@ RSpec.describe FeedManager do | |||
reblogged = Fabricate(:status) | |||
reblog = Fabricate(:status, reblog: reblogged) | |||
FeedManager.instance.push('type', account, reblogged) | |||
FeedManager.instance.push_to_home(account, reblogged) | |||
expect(FeedManager.instance.push('type', account, reblog)).to be false | |||
expect(FeedManager.instance.push_to_home(account, reblog)).to be false | |||
end | |||
it 'saves a new reblog of an old status' do | |||
@@ -189,14 +179,14 @@ RSpec.describe FeedManager do | |||
reblogged = Fabricate(:status) | |||
reblog = Fabricate(:status, reblog: reblogged) | |||
FeedManager.instance.push('type', account, reblogged) | |||
FeedManager.instance.push_to_home(account, reblogged) | |||
# Fill the feed with intervening statuses | |||
FeedManager::REBLOG_FALLOFF.times do | |||
FeedManager.instance.push('type', account, Fabricate(:status)) | |||
FeedManager.instance.push_to_home(account, Fabricate(:status)) | |||
end | |||
expect(FeedManager.instance.push('type', account, reblog)).to be true | |||
expect(FeedManager.instance.push_to_home(account, reblog)).to be true | |||
end | |||
it 'does not save a new reblog of a recently-reblogged status' do | |||
@@ -205,10 +195,10 @@ RSpec.describe FeedManager do | |||
reblogs = 2.times.map { Fabricate(:status, reblog: reblogged) } | |||
# The first reblog will be accepted | |||
FeedManager.instance.push('type', account, reblogs.first) | |||
FeedManager.instance.push_to_home(account, reblogs.first) | |||
# The second reblog should be ignored | |||
expect(FeedManager.instance.push('type', account, reblogs.last)).to be false | |||
expect(FeedManager.instance.push_to_home(account, reblogs.last)).to be false | |||
end | |||
it 'does not save a new reblog of a multiply-reblogged-then-unreblogged status' do | |||
@@ -217,14 +207,14 @@ RSpec.describe FeedManager do | |||
reblogs = 3.times.map { Fabricate(:status, reblog: reblogged) } | |||
# Accept the reblogs | |||
FeedManager.instance.push('type', account, reblogs[0]) | |||
FeedManager.instance.push('type', account, reblogs[1]) | |||
FeedManager.instance.push_to_home(account, reblogs[0]) | |||
FeedManager.instance.push_to_home(account, reblogs[1]) | |||
# Unreblog the first one | |||
FeedManager.instance.unpush('type', account, reblogs[0]) | |||
FeedManager.instance.unpush_from_home(account, reblogs[0]) | |||
# The last reblog should still be ignored | |||
expect(FeedManager.instance.push('type', account, reblogs.last)).to be false | |||
expect(FeedManager.instance.push_to_home(account, reblogs.last)).to be false | |||
end | |||
it 'saves a new reblog of a long-ago-reblogged status' do | |||
@@ -233,15 +223,15 @@ RSpec.describe FeedManager do | |||
reblogs = 2.times.map { Fabricate(:status, reblog: reblogged) } | |||
# The first reblog will be accepted | |||
FeedManager.instance.push('type', account, reblogs.first) | |||
FeedManager.instance.push_to_home(account, reblogs.first) | |||
# Fill the feed with intervening statuses | |||
FeedManager::REBLOG_FALLOFF.times do | |||
FeedManager.instance.push('type', account, Fabricate(:status)) | |||
FeedManager.instance.push_to_home(account, Fabricate(:status)) | |||
end | |||
# The second reblog should also be accepted | |||
expect(FeedManager.instance.push('type', account, reblogs.last)).to be true | |||
expect(FeedManager.instance.push_to_home(account, reblogs.last)).to be true | |||
end | |||
end | |||
end | |||
@@ -253,11 +243,11 @@ RSpec.describe FeedManager do | |||
reblogged = Fabricate(:status) | |||
status = Fabricate(:status, reblog: reblogged) | |||
another_status = Fabricate(:status, reblog: reblogged) | |||
reblogs_key = FeedManager.instance.key('type', receiver.id, 'reblogs') | |||
reblog_set_key = FeedManager.instance.key('type', receiver.id, "reblogs:#{reblogged.id}") | |||
reblogs_key = FeedManager.instance.key('home', receiver.id, 'reblogs') | |||
reblog_set_key = FeedManager.instance.key('home', receiver.id, "reblogs:#{reblogged.id}") | |||
FeedManager.instance.push('type', receiver, status) | |||
FeedManager.instance.push('type', receiver, another_status) | |||
FeedManager.instance.push_to_home(receiver, status) | |||
FeedManager.instance.push_to_home(receiver, another_status) | |||
# We should have a tracking set and an entry in reblogs. | |||
expect(Redis.current.exists(reblog_set_key)).to be true | |||
@@ -265,12 +255,12 @@ RSpec.describe FeedManager do | |||
# Push everything off the end of the feed. | |||
FeedManager::MAX_ITEMS.times do | |||
FeedManager.instance.push('type', receiver, Fabricate(:status)) | |||
FeedManager.instance.push_to_home(receiver, Fabricate(:status)) | |||
end | |||
# `trim` should be called automatically, but do it anyway, as | |||
# we're testing `trim`, not side effects of `push`. | |||
FeedManager.instance.trim('type', receiver.id) | |||
FeedManager.instance.trim('home', receiver.id) | |||
# We should not have any reblog tracking data. | |||
expect(Redis.current.exists(reblog_set_key)).to be false | |||
@@ -285,32 +275,32 @@ RSpec.describe FeedManager do | |||
reblogged = Fabricate(:status) | |||
status = Fabricate(:status, reblog: reblogged) | |||
FeedManager.instance.push('type', receiver, reblogged) | |||
FeedManager::REBLOG_FALLOFF.times { FeedManager.instance.push('type', receiver, Fabricate(:status)) } | |||
FeedManager.instance.push('type', receiver, status) | |||
FeedManager.instance.push_to_home(receiver, reblogged) | |||
FeedManager::REBLOG_FALLOFF.times { FeedManager.instance.push_to_home(receiver, Fabricate(:status)) } | |||
FeedManager.instance.push_to_home(receiver, status) | |||
# The reblogging status should show up under normal conditions. | |||
expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to include(status.id.to_s) | |||
expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to include(status.id.to_s) | |||
FeedManager.instance.unpush('type', receiver, status) | |||
FeedManager.instance.unpush_from_home(receiver, status) | |||
# Restore original status | |||
expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to_not include(status.id.to_s) | |||
expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to include(reblogged.id.to_s) | |||
expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to_not include(status.id.to_s) | |||
expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to include(reblogged.id.to_s) | |||
end | |||
it 'removes a reblogged status if it was only reblogged once' do | |||
reblogged = Fabricate(:status) | |||
status = Fabricate(:status, reblog: reblogged) | |||
FeedManager.instance.push('type', receiver, status) | |||
FeedManager.instance.push_to_home(receiver, status) | |||
# The reblogging status should show up under normal conditions. | |||
expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to eq [status.id.to_s] | |||
expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to eq [status.id.to_s] | |||
FeedManager.instance.unpush('type', receiver, status) | |||
FeedManager.instance.unpush_from_home(receiver, status) | |||
expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to be_empty | |||
expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to be_empty | |||
end | |||
it 'leaves a multiply-reblogged status if another reblog was in feed' do | |||
@@ -318,26 +308,26 @@ RSpec.describe FeedManager do | |||
reblogs = 3.times.map { Fabricate(:status, reblog: reblogged) } | |||
reblogs.each do |reblog| | |||
FeedManager.instance.push('type', receiver, reblog) | |||
FeedManager.instance.push_to_home(receiver, reblog) | |||
end | |||
# The reblogging status should show up under normal conditions. | |||
expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to eq [reblogs.first.id.to_s] | |||
expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to eq [reblogs.first.id.to_s] | |||
reblogs[0...-1].each do |reblog| | |||
FeedManager.instance.unpush('type', receiver, reblog) | |||
FeedManager.instance.unpush_from_home(receiver, reblog) | |||
end | |||
expect(Redis.current.zrange("feed:type:#{receiver.id}", 0, -1)).to eq [reblogs.last.id.to_s] | |||
expect(Redis.current.zrange("feed:home:#{receiver.id}", 0, -1)).to eq [reblogs.last.id.to_s] | |||
end | |||
it 'sends push updates' do | |||
status = Fabricate(:status) | |||
FeedManager.instance.push('type', receiver, status) | |||
FeedManager.instance.push_to_home(receiver, status) | |||
allow(Redis.current).to receive_messages(publish: nil) | |||
FeedManager.instance.unpush('type', receiver, status) | |||
FeedManager.instance.unpush_from_home(receiver, status) | |||
deletion = Oj.dump(event: :delete, payload: status.id.to_s) | |||
expect(Redis.current).to have_received(:publish).with("timeline:#{receiver.id}", deletion) | |||
@@ -1,5 +1,5 @@ | |||
require 'rails_helper' | |||
RSpec.describe AccountModerationNote, type: :model do | |||
pending "add some examples to (or delete) #{__FILE__}" | |||
end |
@@ -1,9 +1,9 @@ | |||
require 'rails_helper' | |||
RSpec.describe Feed, type: :model do | |||
RSpec.describe HomeFeed, type: :model do | |||
let(:account) { Fabricate(:account) } | |||
subject { described_class.new(:home, account) } | |||
subject { described_class.new(account) } | |||
describe '#get' do | |||
before do |
@@ -0,0 +1,5 @@ | |||
require 'rails_helper' | |||
RSpec.describe ListAccount, type: :model do | |||
end |
@@ -0,0 +1,5 @@ | |||
require 'rails_helper' | |||
RSpec.describe List, type: :model do | |||
end |
@@ -18,8 +18,8 @@ RSpec.describe AfterBlockService do | |||
end | |||
it "clears account's statuses" do | |||
FeedManager.instance.push(:home, account, status) | |||
FeedManager.instance.push(:home, account, other_account_status) | |||
FeedManager.instance.push_to_home(account, status) | |||
FeedManager.instance.push_to_home(account, other_account_status) | |||
is_expected.to change { | |||
Redis.current.zrange(home_timeline_key, 0, -1) | |||
@@ -30,11 +30,11 @@ RSpec.describe BatchedRemoveStatusService do | |||
end | |||
it 'removes statuses from author\'s home feed' do | |||
expect(Feed.new(:home, alice).get(10)).to_not include([status1.id, status2.id]) | |||
expect(HomeFeed.new(alice).get(10)).to_not include([status1.id, status2.id]) | |||
end | |||
it 'removes statuses from local follower\'s home feed' do | |||
expect(Feed.new(:home, jeff).get(10)).to_not include([status1.id, status2.id]) | |||
expect(HomeFeed.new(jeff).get(10)).to_not include([status1.id, status2.id]) | |||
end | |||
it 'notifies streaming API of followers' do | |||
@@ -19,12 +19,12 @@ RSpec.describe FanOutOnWriteService do | |||
end | |||
it 'delivers status to home timeline' do | |||
expect(Feed.new(:home, author).get(10).map(&:id)).to include status.id | |||
expect(HomeFeed.new(author).get(10).map(&:id)).to include status.id | |||
end | |||
it 'delivers status to local followers' do | |||
pending 'some sort of problem in test environment causes this to sometimes fail' | |||
expect(Feed.new(:home, follower).get(10).map(&:id)).to include status.id | |||
expect(HomeFeed.new(follower).get(10).map(&:id)).to include status.id | |||
end | |||
it 'delivers status to hashtag' do | |||
@@ -18,8 +18,8 @@ RSpec.describe MuteService do | |||
end | |||
it "clears account's statuses" do | |||
FeedManager.instance.push(:home, account, status) | |||
FeedManager.instance.push(:home, account, other_account_status) | |||
FeedManager.instance.push_to_home(account, status) | |||
FeedManager.instance.push_to_home(account, other_account_status) | |||
is_expected.to change { | |||
Redis.current.zrange(home_timeline_key, 0, -1) | |||
@@ -25,11 +25,11 @@ RSpec.describe RemoveStatusService do | |||
end | |||
it 'removes status from author\'s home feed' do | |||
expect(Feed.new(:home, alice).get(10)).to_not include(@status.id) | |||
expect(HomeFeed.new(alice).get(10)).to_not include(@status.id) | |||
end | |||
it 'removes status from local follower\'s home feed' do | |||
expect(Feed.new(:home, jeff).get(10)).to_not include(@status.id) | |||
expect(HomeFeed.new(jeff).get(10)).to_not include(@status.id) | |||
end | |||
it 'sends PuSH update to PuSH subscribers' do | |||
@@ -11,41 +11,41 @@ describe FeedInsertWorker do | |||
context 'when there are no records' do | |||
it 'skips push with missing status' do | |||
instance = double(push: nil) | |||
instance = double(push_to_home: nil) | |||
allow(FeedManager).to receive(:instance).and_return(instance) | |||
result = subject.perform(nil, follower.id) | |||
expect(result).to eq true | |||
expect(instance).not_to have_received(:push) | |||
expect(instance).not_to have_received(:push_to_home) | |||
end | |||
it 'skips push with missing account' do | |||
instance = double(push: nil) | |||
instance = double(push_to_home: nil) | |||
allow(FeedManager).to receive(:instance).and_return(instance) | |||
result = subject.perform(status.id, nil) | |||
expect(result).to eq true | |||
expect(instance).not_to have_received(:push) | |||
expect(instance).not_to have_received(:push_to_home) | |||
end | |||
end | |||
context 'when there are real records' do | |||
it 'skips the push when there is a filter' do | |||
instance = double(push: nil, filter?: true) | |||
instance = double(push_to_home: nil, filter?: true) | |||
allow(FeedManager).to receive(:instance).and_return(instance) | |||
result = subject.perform(status.id, follower.id) | |||
expect(result).to be_nil | |||
expect(instance).not_to have_received(:push) | |||
expect(instance).not_to have_received(:push_to_home) | |||
end | |||
it 'pushes the status onto the home timeline without filter' do | |||
instance = double(push: nil, filter?: false) | |||
instance = double(push_to_home: nil, filter?: false) | |||
allow(FeedManager).to receive(:instance).and_return(instance) | |||
result = subject.perform(status.id, follower.id) | |||
expect(result).to be_nil | |||
expect(instance).to have_received(:push).with(:home, follower, status) | |||
expect(instance).to have_received(:push_to_home).with(follower, status) | |||
end | |||
end | |||
end | |||
@@ -254,6 +254,26 @@ const startWorker = (workerId) => { | |||
const placeholders = (arr, shift = 0) => arr.map((_, i) => `$${i + 1 + shift}`).join(', '); | |||
const authorizeListAccess = (id, req, next) => { | |||
pgPool.connect((err, client, done) => { | |||
if (err) { | |||
next(false); | |||
return; | |||
} | |||
client.query('SELECT id, account_id FROM lists WHERE id = $1 LIMIT 1', [id], (err, result) => { | |||
done(); | |||
if (err || result.rows.length === 0 || result.rows[0].account_id !== req.accountId) { | |||
next(false); | |||
return; | |||
} | |||
next(true); | |||
}); | |||
}); | |||
}; | |||
const streamFrom = (id, req, output, attachCloseHandler, needsFiltering = false, notificationOnly = false) => { | |||
const streamType = notificationOnly ? ' (notification)' : ''; | |||
log.verbose(req.requestId, `Starting stream from ${id} for ${req.accountId}${streamType}`); | |||
@@ -410,7 +430,22 @@ const startWorker = (workerId) => { | |||
streamFrom(`timeline:hashtag:${req.query.tag.toLowerCase()}:local`, req, streamToHttp(req, res), streamHttpEnd(req), true); | |||
}); | |||
const wss = new WebSocket.Server({ server, verifyClient: wsVerifyClient }); | |||
app.get('/api/v1/streaming/list', (req, res) => { | |||
const listId = req.query.list; | |||
authorizeListAccess(listId, req, authorized => { | |||
if (!authorized) { | |||
res.writeHead(404, { 'Content-Type': 'application/json' }); | |||
res.end(JSON.stringify({ error: 'Not found' })); | |||
return; | |||
} | |||
const channel = `timeline:list:${listId}`; | |||
streamFrom(channel, req, streamToHttp(req, res), streamHttpEnd(req, subscriptionHeartbeat(channel))); | |||
}); | |||
}); | |||
const wss = new WebSocket.Server({ server, verifyClient: wsVerifyClient }); | |||
wss.on('connection', ws => { | |||
const req = ws.upgradeReq; | |||
@@ -443,6 +478,19 @@ const startWorker = (workerId) => { | |||
case 'hashtag:local': | |||
streamFrom(`timeline:hashtag:${location.query.tag.toLowerCase()}:local`, req, streamToWs(req, ws), streamWsEnd(req, ws), true); | |||
break; | |||
case 'list': | |||
const listId = location.query.list; | |||
authorizeListAccess(listId, req, authorized => { | |||
if (!authorized) { | |||
ws.close(); | |||
return; | |||
} | |||
const channel = `timeline:list:${listId}`; | |||
streamFrom(channel, req, streamToWs(req, ws), streamWsEnd(req, ws, subscriptionHeartbeat(channel))); | |||
}); | |||
break; | |||
default: | |||
ws.close(); | |||
} | |||