diff --git a/app/services/activitypub/fetch_featured_collections_collection_service.rb b/app/services/activitypub/fetch_featured_collections_collection_service.rb new file mode 100644 index 0000000000..99a45fac3e --- /dev/null +++ b/app/services/activitypub/fetch_featured_collections_collection_service.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +class ActivityPub::FetchFeaturedCollectionsCollectionService < BaseService + include JsonLdHelper + + MAX_PAGES = 10 + MAX_ITEMS = 50 + + def call(account, request_id: nil) + return if account.collections_url.blank? || account.suspended? || account.local? + + @request_id = request_id + @account = account + @items, = collection_items(@account.collections_url, max_pages: MAX_PAGES, reference_uri: @account.uri) + process_items(@items) + end + + private + + def process_items(items) + return if items.nil? + + items.take(MAX_ITEMS).each do |collection_json| + if collection_json.is_a?(String) + ActivityPub::FetchRemoteFeaturedCollectionService.new.call(collection_json, request_id: @request_id) + else + ActivityPub::ProcessFeaturedCollectionService.new.call(@account, collection_json, request_id: @request_id) + end + end + end +end diff --git a/app/services/activitypub/fetch_remote_featured_collection_service.rb b/app/services/activitypub/fetch_remote_featured_collection_service.rb index e9858fe34d..babad143e1 100644 --- a/app/services/activitypub/fetch_remote_featured_collection_service.rb +++ b/app/services/activitypub/fetch_remote_featured_collection_service.rb @@ -3,7 +3,7 @@ class ActivityPub::FetchRemoteFeaturedCollectionService < BaseService include JsonLdHelper - def call(uri, on_behalf_of = nil) + def call(uri, request_id: nil, on_behalf_of: nil) json = fetch_resource(uri, true, on_behalf_of) return unless supported_context?(json) @@ -17,6 +17,6 @@ class ActivityPub::FetchRemoteFeaturedCollectionService < BaseService existing_collection = account.collections.find_by(uri:) return existing_collection if existing_collection.present? - ActivityPub::ProcessFeaturedCollectionService.new.call(account, json) + ActivityPub::ProcessFeaturedCollectionService.new.call(account, json, request_id:) end end diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index fb26992bf4..dd0eeadaa0 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -60,6 +60,7 @@ class ActivityPub::ProcessAccountService < BaseService unless @options[:only_key] || @account.suspended? check_featured_collection! if @json['featured'].present? check_featured_tags_collection! if @json['featuredTags'].present? + check_featured_collections_collection! if @json['featuredCollections'].present? && Mastodon::Feature.collections_federation_enabled? check_links! if @account.fields.any?(&:requires_verification?) end @@ -201,6 +202,10 @@ class ActivityPub::ProcessAccountService < BaseService ActivityPub::SynchronizeFeaturedTagsCollectionWorker.perform_async(@account.id, @json['featuredTags']) end + def check_featured_collections_collection! + ActivityPub::SynchronizeFeaturedCollectionsCollectionWorker.perform_async(@account.id, @options[:request_id]) + end + def check_links! VerifyAccountLinksWorker.perform_in(rand(10.minutes.to_i), @account.id) end diff --git a/app/services/activitypub/process_featured_collection_service.rb b/app/services/activitypub/process_featured_collection_service.rb index 73db0d6699..91c15bdce2 100644 --- a/app/services/activitypub/process_featured_collection_service.rb +++ b/app/services/activitypub/process_featured_collection_service.rb @@ -7,9 +7,10 @@ class ActivityPub::ProcessFeaturedCollectionService ITEMS_LIMIT = 150 - def call(account, json) + def call(account, json, request_id: nil) @account = account @json = json + @request_id = request_id return if non_matching_uri_hosts?(@account.uri, @json['id']) with_redis_lock("collection:#{@json['id']}") do @@ -46,7 +47,7 @@ class ActivityPub::ProcessFeaturedCollectionService def process_items! @json['orderedItems'].take(ITEMS_LIMIT).each do |item_json| - ActivityPub::ProcessFeaturedItemWorker.perform_async(@collection.id, item_json) + ActivityPub::ProcessFeaturedItemWorker.perform_async(@collection.id, item_json, @request_id) end end end diff --git a/app/services/activitypub/process_featured_item_service.rb b/app/services/activitypub/process_featured_item_service.rb index b69cfc54e8..c0d2bfd206 100644 --- a/app/services/activitypub/process_featured_item_service.rb +++ b/app/services/activitypub/process_featured_item_service.rb @@ -5,7 +5,8 @@ class ActivityPub::ProcessFeaturedItemService include Lockable include Redisable - def call(collection, uri_or_object) + def call(collection, uri_or_object, request_id: nil) + @request_id = request_id item_json = uri_or_object.is_a?(String) ? fetch_resource(uri_or_object, true) : uri_or_object return if non_matching_uri_hosts?(collection.uri, item_json['id']) @@ -35,8 +36,8 @@ class ActivityPub::ProcessFeaturedItemService private def verify_authorization! - ActivityPub::VerifyFeaturedItemService.new.call(@collection_item, @approval_uri) + ActivityPub::VerifyFeaturedItemService.new.call(@collection_item, @approval_uri, request_id: @request_id) rescue Mastodon::RecursionLimitExceededError, Mastodon::UnexpectedResponseError, *Mastodon::HTTP_CONNECTION_ERRORS - ActivityPub::VerifyFeaturedItemWorker.perform_in(rand(30..600).seconds, @collection_item.id, @approval_uri) + ActivityPub::VerifyFeaturedItemWorker.perform_in(rand(30..600).seconds, @collection_item.id, @approval_uri, @request_id) end end diff --git a/app/services/activitypub/verify_featured_item_service.rb b/app/services/activitypub/verify_featured_item_service.rb index ecebdf2075..6ce524870c 100644 --- a/app/services/activitypub/verify_featured_item_service.rb +++ b/app/services/activitypub/verify_featured_item_service.rb @@ -3,7 +3,7 @@ class ActivityPub::VerifyFeaturedItemService include JsonLdHelper - def call(collection_item, approval_uri) + def call(collection_item, approval_uri, request_id: nil) @collection_item = collection_item @authorization = fetch_resource(approval_uri, true, raise_on_error: :temporary) @@ -16,7 +16,7 @@ class ActivityPub::VerifyFeaturedItemService return unless matching_type? && matching_collection_uri? account = Account.where(uri: @collection_item.object_uri).first - account ||= ActivityPub::FetchRemoteAccountService.new.call(@collection_item.object_uri) + account ||= ActivityPub::FetchRemoteAccountService.new.call(@collection_item.object_uri, request_id:) return if account.blank? @collection_item.update!(account:, approval_uri:, state: :accepted) diff --git a/app/workers/activitypub/process_featured_item_worker.rb b/app/workers/activitypub/process_featured_item_worker.rb index dd765e7df6..f50ff50b40 100644 --- a/app/workers/activitypub/process_featured_item_worker.rb +++ b/app/workers/activitypub/process_featured_item_worker.rb @@ -6,10 +6,10 @@ class ActivityPub::ProcessFeaturedItemWorker sidekiq_options queue: 'pull', retry: 3 - def perform(collection_id, id_or_json) + def perform(collection_id, id_or_json, request_id = nil) collection = Collection.find(collection_id) - ActivityPub::ProcessFeaturedItemService.new.call(collection, id_or_json) + ActivityPub::ProcessFeaturedItemService.new.call(collection, id_or_json, request_id:) rescue ActiveRecord::RecordNotFound true end diff --git a/app/workers/activitypub/synchronize_featured_collections_collection_worker.rb b/app/workers/activitypub/synchronize_featured_collections_collection_worker.rb new file mode 100644 index 0000000000..e24fe59b07 --- /dev/null +++ b/app/workers/activitypub/synchronize_featured_collections_collection_worker.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class ActivityPub::SynchronizeFeaturedCollectionsCollectionWorker + include Sidekiq::Worker + + sidekiq_options queue: 'pull', lock: :until_executed, lock_ttl: 1.day.to_i + + def perform(account_id, request_id = nil) + account = Account.find(account_id) + + ActivityPub::FetchFeaturedCollectionsCollectionService.new.call(account, request_id:) + rescue ActiveRecord::RecordNotFound + true + end +end diff --git a/app/workers/activitypub/verify_featured_item_worker.rb b/app/workers/activitypub/verify_featured_item_worker.rb index cd47f5ee21..3e96b81c09 100644 --- a/app/workers/activitypub/verify_featured_item_worker.rb +++ b/app/workers/activitypub/verify_featured_item_worker.rb @@ -7,10 +7,10 @@ class ActivityPub::VerifyFeaturedItemWorker sidekiq_options queue: 'pull', retry: 5 - def perform(collection_item_id, approval_uri) + def perform(collection_item_id, approval_uri, request_id = nil) collection_item = CollectionItem.find(collection_item_id) - ActivityPub::VerifyFeaturedItemService.new.call(collection_item, approval_uri) + ActivityPub::VerifyFeaturedItemService.new.call(collection_item, approval_uri, request_id:) rescue ActiveRecord::RecordNotFound # Do nothing nil diff --git a/spec/services/activitypub/fetch_featured_collections_collection_service_spec.rb b/spec/services/activitypub/fetch_featured_collections_collection_service_spec.rb new file mode 100644 index 0000000000..37a78dbf41 --- /dev/null +++ b/spec/services/activitypub/fetch_featured_collections_collection_service_spec.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe ActivityPub::FetchFeaturedCollectionsCollectionService do + subject { described_class.new } + + let(:account) { Fabricate(:remote_account, collections_url: 'https://example.com/account/featured_collections') } + let(:featured_collection_one) do + { + '@context' => 'https://www.w3.org/ns/activitystreams', + 'id' => 'https://example.com/featured_collections/1', + 'type' => 'FeaturedCollection', + 'name' => 'Incredible people', + 'summary' => 'These are really amazing', + 'attributedTo' => account.uri, + 'sensitive' => false, + 'discoverable' => true, + 'totalItems' => 0, + } + end + let(:featured_collection_two) do + { + '@context' => 'https://www.w3.org/ns/activitystreams', + 'id' => 'https://example.com/featured_collections/2', + 'type' => 'FeaturedCollection', + 'name' => 'Even cooler people', + 'summary' => 'These are just as amazing', + 'attributedTo' => account.uri, + 'sensitive' => false, + 'discoverable' => true, + 'totalItems' => 0, + } + end + let(:items) { [featured_collection_one, featured_collection_two] } + let(:collection_json) do + { + '@context' => 'https://www.w3.org/ns/activitystreams', + 'type' => 'Collection', + 'id' => account.collections_url, + 'items' => items, + } + end + + describe '#call' do + subject { described_class.new.call(account) } + + before do + stub_request(:get, account.collections_url) + .to_return_json(status: 200, body: collection_json, headers: { 'Content-Type': 'application/activity+json' }) + end + + shared_examples 'collection creation' do + it 'creates the expected collections' do + expect { subject }.to change(account.collections, :count).by(2) + expect(account.collections.pluck(:name)).to contain_exactly('Incredible people', 'Even cooler people') + end + end + + context 'when the endpoint is not paginated' do + context 'when all items are inlined' do + it_behaves_like 'collection creation' + end + + context 'when items are URIs' do + let(:items) { [featured_collection_one['id'], featured_collection_two['id']] } + + before do + [featured_collection_one, featured_collection_two].each do |featured_collection| + stub_request(:get, featured_collection['id']) + .to_return_json(status: 200, body: featured_collection, headers: { 'Content-Type': 'application/activity+json' }) + end + end + + it_behaves_like 'collection creation' + end + end + + context 'when the endpoint is a paginated Collection' do + let(:first_page) do + { + '@context' => 'https://www.w3.org/ns/activitystreams', + 'type' => 'CollectionPage', + 'partOf' => account.collections_url, + 'id' => 'https://example.com/featured_collections/1/1', + 'items' => [featured_collection_one], + 'next' => second_page['id'], + } + end + let(:second_page) do + { + '@context' => 'https://www.w3.org/ns/activitystreams', + 'type' => 'CollectionPage', + 'partOf' => account.collections_url, + 'id' => 'https://example.com/featured_collections/1/2', + 'items' => [featured_collection_two], + } + end + let(:collection_json) do + { + '@context' => 'https://www.w3.org/ns/activitystreams', + 'type' => 'Collection', + 'id' => account.collections_url, + 'first' => first_page['id'], + } + end + + before do + [first_page, second_page].each do |page| + stub_request(:get, page['id']) + .to_return_json(status: 200, body: page, headers: { 'Content-Type': 'application/activity+json' }) + end + end + + it_behaves_like 'collection creation' + end + end +end diff --git a/spec/services/activitypub/process_account_service_spec.rb b/spec/services/activitypub/process_account_service_spec.rb index 1d8d8f8ac5..832f4e9729 100644 --- a/spec/services/activitypub/process_account_service_spec.rb +++ b/spec/services/activitypub/process_account_service_spec.rb @@ -63,7 +63,7 @@ RSpec.describe ActivityPub::ProcessAccountService do end end - context 'with collection URIs' do + context 'with collection URIs', feature: :collections_federation do let(:payload) do { 'id' => 'https://foo.test', @@ -81,13 +81,16 @@ RSpec.describe ActivityPub::ProcessAccountService do .to_return(status: 200, body: '', headers: {}) end - it 'parses and sets the URIs' do + it 'parses and sets the URIs, queues jobs to synchronize' do account = subject.call('alice', 'example.com', payload) expect(account.featured_collection_url).to eq 'https://foo.test/featured' expect(account.followers_url).to eq 'https://foo.test/followers' expect(account.following_url).to eq 'https://foo.test/following' expect(account.collections_url).to eq 'https://foo.test/featured_collections' + + expect(ActivityPub::SynchronizeFeaturedCollectionWorker).to have_enqueued_sidekiq_job + expect(ActivityPub::SynchronizeFeaturedCollectionsCollectionWorker).to have_enqueued_sidekiq_job end end diff --git a/spec/services/activitypub/verify_featured_item_service_spec.rb b/spec/services/activitypub/verify_featured_item_service_spec.rb index 5698c23155..f0f1661b6d 100644 --- a/spec/services/activitypub/verify_featured_item_service_spec.rb +++ b/spec/services/activitypub/verify_featured_item_service_spec.rb @@ -55,7 +55,7 @@ RSpec.describe ActivityPub::VerifyFeaturedItemService do let(:stubbed_service) { instance_double(ActivityPub::FetchRemoteAccountService) } before do - allow(stubbed_service).to receive(:call).with('https://example.com/actor/1') { featured_account } + allow(stubbed_service).to receive(:call).with('https://example.com/actor/1', request_id: nil) { featured_account } allow(ActivityPub::FetchRemoteAccountService).to receive(:new).and_return(stubbed_service) end diff --git a/spec/workers/activitypub/process_featured_item_worker_spec.rb b/spec/workers/activitypub/process_featured_item_worker_spec.rb index f27ec21c35..02a9edfc47 100644 --- a/spec/workers/activitypub/process_featured_item_worker_spec.rb +++ b/spec/workers/activitypub/process_featured_item_worker_spec.rb @@ -19,7 +19,7 @@ RSpec.describe ActivityPub::ProcessFeaturedItemWorker do it 'calls the service to process the item' do subject.perform(collection.id, object) - expect(stubbed_service).to have_received(:call).with(collection, object) + expect(stubbed_service).to have_received(:call).with(collection, object, request_id: nil) end end end diff --git a/spec/workers/activitypub/synchronize_featured_collections_collection_worker_spec.rb b/spec/workers/activitypub/synchronize_featured_collections_collection_worker_spec.rb new file mode 100644 index 0000000000..6fcb9d02b2 --- /dev/null +++ b/spec/workers/activitypub/synchronize_featured_collections_collection_worker_spec.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe ActivityPub::SynchronizeFeaturedCollectionsCollectionWorker do + let(:worker) { described_class.new } + let(:service) { instance_double(ActivityPub::FetchFeaturedCollectionsCollectionService, call: true) } + + describe '#perform' do + before do + allow(ActivityPub::FetchFeaturedCollectionsCollectionService).to receive(:new).and_return(service) + end + + let(:account) { Fabricate(:account) } + + it 'sends the account to the service' do + worker.perform(account.id) + + expect(service).to have_received(:call).with(account, request_id: nil) + end + + it 'returns true for non-existent record' do + result = worker.perform(123_123_123) + + expect(result).to be(true) + end + end +end diff --git a/spec/workers/activitypub/verify_featured_item_worker_spec.rb b/spec/workers/activitypub/verify_featured_item_worker_spec.rb index bbea044c34..d7d31b3510 100644 --- a/spec/workers/activitypub/verify_featured_item_worker_spec.rb +++ b/spec/workers/activitypub/verify_featured_item_worker_spec.rb @@ -14,7 +14,7 @@ RSpec.describe ActivityPub::VerifyFeaturedItemWorker do it 'sends the status to the service' do worker.perform(collection_item.id, 'https://example.com/authorizations/1') - expect(service).to have_received(:call).with(collection_item, 'https://example.com/authorizations/1') + expect(service).to have_received(:call).with(collection_item, 'https://example.com/authorizations/1', request_id: nil) end it 'returns nil for non-existent record' do