@@ -16,7 +16,11 @@ module JsonLdHelper | |||
def fetch_resource(uri) | |||
response = build_request(uri).perform | |||
return if response.code != 200 | |||
Oj.load(response.to_s, mode: :strict) | |||
body_to_json(response.to_s) | |||
end | |||
def body_to_json(body) | |||
body.nil? ? nil : Oj.load(body, mode: :strict) | |||
rescue Oj::ParseError | |||
nil | |||
end | |||
@@ -25,7 +29,7 @@ module JsonLdHelper | |||
def build_request(uri) | |||
request = Request.new(:get, uri) | |||
request.add_headers('Accept' => 'application/activity+json') | |||
request.add_headers('Accept' => 'application/activity+json, application/ld+json') | |||
request | |||
end | |||
end |
@@ -5,8 +5,8 @@ class ActivityPub::FetchRemoteAccountService < BaseService | |||
# Should be called when uri has already been checked for locality | |||
# Does a WebFinger roundtrip on each call | |||
def call(uri) | |||
@json = fetch_resource(uri) | |||
def call(uri, prefetched_json = nil) | |||
@json = body_to_json(prefetched_json) || fetch_resource(uri) | |||
return unless supported_context? && expected_type? | |||
@@ -4,8 +4,8 @@ class ActivityPub::FetchRemoteStatusService < BaseService | |||
include JsonLdHelper | |||
# Should be called when uri has already been checked for locality | |||
def call(uri) | |||
@json = fetch_resource(uri) | |||
def call(uri, prefetched_json = nil) | |||
@json = body_to_json(prefetched_json) || fetch_resource(uri) | |||
return unless supported_context? && expected_type? | |||
@@ -4,18 +4,10 @@ class FetchAtomService < BaseService | |||
def call(url) | |||
return if url.blank? | |||
response = Request.new(:head, url).perform | |||
@url = url | |||
Rails.logger.debug "Remote status HEAD request returned code #{response.code}" | |||
response = Request.new(:get, url).perform if response.code == 405 | |||
Rails.logger.debug "Remote status GET request returned code #{response.code}" | |||
return nil if response.code != 200 | |||
return [url, fetch(url)] if response.mime_type == 'application/atom+xml' | |||
return process_headers(url, response) if response['Link'].present? | |||
process_html(fetch(url)) | |||
perform_request | |||
process_response | |||
rescue OpenSSL::SSL::SSLError => e | |||
Rails.logger.debug "SSL error: #{e}" | |||
nil | |||
@@ -26,27 +18,57 @@ class FetchAtomService < BaseService | |||
private | |||
def process_html(body) | |||
Rails.logger.debug 'Processing HTML' | |||
def perform_request | |||
@response = Request.new(:get, @url) | |||
.add_headers('Accept' => 'application/activity+json, application/ld+json, application/atom+xml, text/html') | |||
.perform | |||
end | |||
page = Nokogiri::HTML(body) | |||
alternate_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' } | |||
def process_response(terminal = false) | |||
return nil if @response.code != 200 | |||
return nil if alternate_link.nil? | |||
[alternate_link['href'], fetch(alternate_link['href'])] | |||
if @response.mime_type == 'application/atom+xml' | |||
[@url, @response.to_s, :ostatus] | |||
elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(@response.mime_type) | |||
[@url, @response.to_s, :activitypub] | |||
elsif @response['Link'] && !terminal | |||
process_headers | |||
elsif @response.mime_type == 'text/html' && !terminal | |||
process_html | |||
end | |||
end | |||
def process_headers(url, response) | |||
Rails.logger.debug 'Processing link header' | |||
def process_html | |||
page = Nokogiri::HTML(@response.to_s) | |||
link_header = LinkHeader.parse(response['Link'].is_a?(Array) ? response['Link'].first : response['Link']) | |||
alternate_link = link_header.find_link(%w(rel alternate), %w(type application/atom+xml)) | |||
json_link = page.xpath('//link[@rel="alternate"]').find { |link| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link['type']) } | |||
atom_link = page.xpath('//link[@rel="alternate"]').find { |link| link['type'] == 'application/atom+xml' } | |||
return process_html(fetch(url)) if alternate_link.nil? | |||
[alternate_link.href, fetch(alternate_link.href)] | |||
if !json_link.nil? | |||
@url = json_link['href'] | |||
perform_request | |||
process_response(true) | |||
elsif !atom_link.nil? | |||
@url = atom_link['href'] | |||
perform_request | |||
process_response(true) | |||
end | |||
end | |||
def fetch(url) | |||
Request.new(:get, url).perform.to_s | |||
def process_headers | |||
link_header = LinkHeader.parse(@response['Link'].is_a?(Array) ? @response['Link'].first : @response['Link']) | |||
json_link = link_header.find_link(%w(rel alternate), %w(type application/activity+json)) || link_header.find_link(%w(rel alternate), ['type', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"']) | |||
atom_link = link_header.find_link(%w(rel alternate), %w(type application/atom+xml)) | |||
if !json_link.nil? | |||
@url = json_link.href | |||
perform_request | |||
process_response(true) | |||
elsif !atom_link.nil? | |||
@url = atom_link.href | |||
perform_request | |||
process_response(true) | |||
end | |||
end | |||
end |
@@ -5,14 +5,19 @@ class FetchRemoteAccountService < BaseService | |||
def call(url, prefetched_body = nil) | |||
if prefetched_body.nil? | |||
atom_url, body = FetchAtomService.new.call(url) | |||
resource_url, body, protocol = FetchAtomService.new.call(url) | |||
else | |||
atom_url = url | |||
body = prefetched_body | |||
resource_url = url | |||
body = prefetched_body | |||
protocol = :ostatus | |||
end | |||
return nil if atom_url.nil? | |||
process_atom(atom_url, body) | |||
case protocol | |||
when :ostatus | |||
process_atom(resource_url, body) | |||
when :activitypub | |||
ActivityPub::FetchRemoteAccountService.new.call(resource_url, body) | |||
end | |||
end | |||
private | |||
@@ -5,14 +5,19 @@ class FetchRemoteStatusService < BaseService | |||
def call(url, prefetched_body = nil) | |||
if prefetched_body.nil? | |||
atom_url, body = FetchAtomService.new.call(url) | |||
resource_url, body, protocol = FetchAtomService.new.call(url) | |||
else | |||
atom_url = url | |||
body = prefetched_body | |||
resource_url = url | |||
body = prefetched_body | |||
protocol = :ostatus | |||
end | |||
return nil if atom_url.nil? | |||
process_atom(atom_url, body) | |||
case protocol | |||
when :ostatus | |||
process_atom(resource_url, body) | |||
when :activitypub | |||
ActivityPub::FetchRemoteStatusService.new.call(resource_url, body) | |||
end | |||
end | |||
private | |||
@@ -38,19 +38,19 @@ RSpec.describe Api::SubscriptionsController, type: :controller do | |||
before do | |||
stub_request(:post, "https://quitter.no/main/push/hub").to_return(:status => 200, :body => "", :headers => {}) | |||
stub_request(:get, "https://quitter.no/avatar/7477-300-20160211190340.png").to_return(request_fixture('avatar.txt')) | |||
stub_request(:head, "https://quitter.no/notice/1269244").to_return(status: 404) | |||
stub_request(:head, "https://quitter.no/notice/1265331").to_return(status: 404) | |||
stub_request(:head, "https://community.highlandarrow.com/notice/54411").to_return(status: 404) | |||
stub_request(:head, "https://community.highlandarrow.com/notice/53857").to_return(status: 404) | |||
stub_request(:head, "https://community.highlandarrow.com/notice/51852").to_return(status: 404) | |||
stub_request(:head, "https://social.umeahackerspace.se/notice/424348").to_return(status: 404) | |||
stub_request(:head, "https://community.highlandarrow.com/notice/50467").to_return(status: 404) | |||
stub_request(:head, "https://quitter.no/notice/1243309").to_return(status: 404) | |||
stub_request(:head, "https://quitter.no/user/7477").to_return(status: 404) | |||
stub_request(:head, "https://community.highlandarrow.com/user/1").to_return(status: 404) | |||
stub_request(:head, "https://social.umeahackerspace.se/user/2").to_return(status: 404) | |||
stub_request(:head, "https://gs.kawa-kun.com/user/2").to_return(status: 404) | |||
stub_request(:head, "https://mastodon.social/users/Gargron").to_return(status: 404) | |||
stub_request(:get, "https://quitter.no/notice/1269244").to_return(status: 404) | |||
stub_request(:get, "https://quitter.no/notice/1265331").to_return(status: 404) | |||
stub_request(:get, "https://community.highlandarrow.com/notice/54411").to_return(status: 404) | |||
stub_request(:get, "https://community.highlandarrow.com/notice/53857").to_return(status: 404) | |||
stub_request(:get, "https://community.highlandarrow.com/notice/51852").to_return(status: 404) | |||
stub_request(:get, "https://social.umeahackerspace.se/notice/424348").to_return(status: 404) | |||
stub_request(:get, "https://community.highlandarrow.com/notice/50467").to_return(status: 404) | |||
stub_request(:get, "https://quitter.no/notice/1243309").to_return(status: 404) | |||
stub_request(:get, "https://quitter.no/user/7477").to_return(status: 404) | |||
stub_request(:any, "https://community.highlandarrow.com/user/1").to_return(status: 404) | |||
stub_request(:any, "https://social.umeahackerspace.se/user/2").to_return(status: 404) | |||
stub_request(:any, "https://gs.kawa-kun.com/user/2").to_return(status: 404) | |||
stub_request(:any, "https://mastodon.social/users/Gargron").to_return(status: 404) | |||
request.env['HTTP_X_HUB_SIGNATURE'] = "sha1=#{OpenSSL::HMAC.hexdigest('sha1', 'abc', feed)}" | |||
request.env['RAW_POST_DATA'] = feed | |||
@@ -124,8 +124,7 @@ RSpec.describe ProcessFeedService do | |||
</entry> | |||
XML | |||
stub_request(:head, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 200, headers: { 'Content-Type' => 'application/atom+xml' }) | |||
stub_request(:get, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 200, body: real_body) | |||
stub_request(:get, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 200, body: real_body, headers: { 'Content-Type' => 'application/atom+xml' }) | |||
bad_actor = Fabricate(:account, username: 'sombra', domain: 'talon.xyz') | |||
@@ -168,7 +167,7 @@ XML | |||
end | |||
it 'ignores reblogs if it failed to retreive reblogged statuses' do | |||
stub_request(:head, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 404) | |||
stub_request(:get, 'https://overwatch.com/users/tracer/updates/1').to_return(status: 404) | |||
actor = Fabricate(:account, username: 'tracer', domain: 'overwatch.com') | |||