diff --git a/api/base/utils.py b/api/base/utils.py index 31bc35fd8d4..6b967760e32 100644 --- a/api/base/utils.py +++ b/api/base/utils.py @@ -155,7 +155,11 @@ def get_object_or_error(model_or_qs, query_or_pk=None, request=None, display_nam if display_name is None: raise Gone else: - raise Gone(detail=f'The requested {display_name} is no longer available.') + AbstractNode = apps.get_model('osf', 'AbstractNode') + spammy_node = isinstance(obj, AbstractNode) and obj.is_spammy + raise Gone( + detail=f'The requested {display_name} is no longer available.', meta={'flagged_content': True} if spammy_node else {}, + ) return obj diff --git a/api/files/views.py b/api/files/views.py index c19aec53d2c..d96289d4010 100644 --- a/api/files/views.py +++ b/api/files/views.py @@ -51,7 +51,7 @@ def get_file(self, check_permissions=True): raise Gone(detail='The requested file is no longer available.') if getattr(obj.target, 'deleted', None): - raise Gone(detail='The requested file is no longer available') + raise Gone(detail='The requested file is no longer available.', meta={'flagged_content': getattr(obj.target, 'is_spammy', False)}) if getattr(obj.target, 'is_retracted', False): raise Gone(detail='The requested file is no longer available.') diff --git a/api/preprints/views.py b/api/preprints/views.py index 27986428958..35bc61aa9d5 100644 --- a/api/preprints/views.py +++ b/api/preprints/views.py @@ -22,7 +22,7 @@ from api.actions.serializers import ReviewActionSerializer from api.actions.views import get_review_actions_queryset from api.base.pagination import PreprintContributorPagination -from api.base.exceptions import Conflict +from api.base.exceptions import Conflict, Gone from api.base.views import JSONAPIBaseView, WaterButlerMixin from api.base.filters import ListFilterMixin, PreprintAsTargetFilterMixin, PreprintFilterMixin from api.base.parsers import ( @@ -126,16 +126,19 @@ def get_preprint(self, check_object_permissions=True, ignore_404=False): qs = Preprint.published_objects.filter(versioned_guids__guid___id=base_guid_id).order_by('-versioned_guids__version') preprint = qs.select_for_update().first() if check_select_for_update(self.request) else qs.select_related('node').first() + user = self.request.user if not preprint: sentry.log_message(f'Preprint not found: [guid={base_guid_id}, version={preprint_version}]') if ignore_404: return raise NotFound if preprint.deleted is not None: - sentry.log_message(f'Preprint deleted: [guid={base_guid_id}, version={preprint_version}]') - raise NotFound + if preprint.is_spammy: + raise Gone(detail='The requested preprint is no longer available.', meta={'flagged_content': True}) + else: + sentry.log_message(f'Preprint deleted: [guid={base_guid_id}, version={preprint_version}]') + raise Gone(detail='The requested preprint is no longer available.') - user = self.request.user if isinstance(user, AnonymousUser): user_is_reviewer = user_is_contributor = False else: diff --git a/api/registrations/views.py b/api/registrations/views.py index caab1e134de..c1bb1ba42a4 100644 --- a/api/registrations/views.py +++ b/api/registrations/views.py @@ -122,7 +122,7 @@ def get_node(self, check_object_permissions=True, **annotations): raise NotFound if node.deleted: - raise Gone(detail='The requested registration is no longer available.') + raise Gone(detail='The requested registration is no longer available.', meta={'flagged_content': node.is_spammy}) if check_object_permissions: self.check_object_permissions(self.request, node) diff --git a/api_tests/files/views/test_file_detail.py b/api_tests/files/views/test_file_detail.py index 45d3888dcc3..e9f432d60d1 100644 --- a/api_tests/files/views/test_file_detail.py +++ b/api_tests/files/views/test_file_detail.py @@ -607,6 +607,24 @@ def test_folder_files_relationships_contains_guid_not_id( assert node._id in split_href assert node.id not in split_href + def test_spammed_node_file_detail_gone(self, app, node, file_url, user): + node.confirm_spam(save=True, train_spam_services=False) + res = app.get(file_url, expect_errors=True) + assert res.status_code == 410 + error = res.json['errors'][0] + assert error['detail'] == 'The requested file is no longer available.' + assert 'meta' in error + assert error['meta']['flagged_content'] + + def test_deleted_file_not_spammed_gone(self, app, user, file, file_url): + file.delete(user=user, save=True) + res = app.get(file_url, expect_errors=True) + assert res.status_code == 410 + error = res.json['errors'][0] + assert error['detail'] == 'The requested file is no longer available.' + assert 'meta' in error + assert not error['meta'].get('flagged_content', False) + @pytest.mark.django_db class TestFileVersionView: diff --git a/api_tests/identifiers/views/test_identifier_list.py b/api_tests/identifiers/views/test_identifier_list.py index 95277a38a13..7a117b4e6f4 100644 --- a/api_tests/identifiers/views/test_identifier_list.py +++ b/api_tests/identifiers/views/test_identifier_list.py @@ -380,26 +380,26 @@ def test_preprint_identifier_list_permissions_deleted( # test_unpublished_preprint_identifier_unauthenticated res = app.get(url_preprint_identifier, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # test_unpublished_preprint_identifier_noncontrib_authenticated non_contrib = AuthUserFactory() res = app.get(url_preprint_identifier, auth=non_contrib.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # test_unpublished_preprint_identifier_admin_authenticated res = app.get(url_preprint_identifier, auth=user.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # test_unpublished_preprint_identifier_readcontrib_authenticated read_user = AuthUserFactory() preprint.add_contributor(read_user, READ, save=True) res = app.get(url_preprint_identifier, auth=read_user.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # test_published_preprint_identifier_unauthenticated res = app.get(url_preprint_identifier, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 def test_preprint_identifier_list_permissions_abandoned( self, app, user, data_preprint_identifier, preprint, url_preprint_identifier): diff --git a/api_tests/nodes/views/test_node_detail.py b/api_tests/nodes/views/test_node_detail.py index 5fa0cd8e150..e03cff83ca9 100644 --- a/api_tests/nodes/views/test_node_detail.py +++ b/api_tests/nodes/views/test_node_detail.py @@ -571,3 +571,22 @@ def test_current_user_permissions_vol(self, app, user, url_public, project_publi private_link.save() res = app.get(f'{url_public}?view_only={private_link.key}', auth=user.auth) assert [permissions.READ] == res.json['data']['attributes']['current_user_permissions'] + + def test_spammed_node_detail_gone(self, app, user, project_public, url_public): + project_public.confirm_spam(save=True, train_spam_services=False) + res = app.get(url_public, expect_errors=True) + assert res.status_code == 410 + error = res.json['errors'][0] + assert error['detail'] == 'The requested node is no longer available.' + assert 'meta' in error + assert error['meta']['flagged_content'] + + def test_not_spammed_deleted_node_detail_gone(self, app, user, project_public, url_public): + project_public.is_deleted = True + project_public.save() + res = app.get(url_public, expect_errors=True) + assert res.status_code == 410 + error = res.json['errors'][0] + assert error['detail'] == 'The requested node is no longer available.' + assert 'meta' in error + assert not error['meta'].get('flagged_content', False) diff --git a/api_tests/preprints/views/test_preprint_citations.py b/api_tests/preprints/views/test_preprint_citations.py index a95f7527d2f..4fa6fe69be3 100644 --- a/api_tests/preprints/views/test_preprint_citations.py +++ b/api_tests/preprints/views/test_preprint_citations.py @@ -122,21 +122,21 @@ def test_deleted_preprint_citations(self): # Unauthenticated res = self.app.get(self.published_preprint_url, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # Non contrib res = self.app.get(self.published_preprint_url, auth=self.other_contrib.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # Write contrib self.published_preprint.add_contributor(self.other_contrib, WRITE, save=True) res = self.app.get(self.published_preprint_url, auth=self.other_contrib.auth, expect_errors=True) # Really because preprint is in initial machine state - assert res.status_code == 404 + assert res.status_code == 410 # Admin contrib res = self.app.get(self.published_preprint_url, auth=self.admin_contributor.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 def test_abandoned_preprint_citations(self): self.published_preprint.machine_state = DefaultStates.INITIAL.value @@ -243,21 +243,21 @@ def test_deleted_preprint_citations(self): # Unauthenticated res = self.app.get(self.published_preprint_url, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # Non contrib res = self.app.get(self.published_preprint_url, auth=self.other_contrib.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # Write contrib self.published_preprint.add_contributor(self.other_contrib, WRITE, save=True) res = self.app.get(self.published_preprint_url, auth=self.other_contrib.auth, expect_errors=True) # Really because preprint is in initial machine state - assert res.status_code == 404 + assert res.status_code == 410 # Admin contrib res = self.app.get(self.published_preprint_url, auth=self.admin_contributor.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 def test_abandoned_preprint_citations(self): self.published_preprint.machine_state = DefaultStates.INITIAL.value diff --git a/api_tests/preprints/views/test_preprint_contributors_detail.py b/api_tests/preprints/views/test_preprint_contributors_detail.py index 726656fda75..e052c4eb62d 100644 --- a/api_tests/preprints/views/test_preprint_contributors_detail.py +++ b/api_tests/preprints/views/test_preprint_contributors_detail.py @@ -194,21 +194,21 @@ def test_preprint_contributor_deleted( # Unauthenticated res = app.get(url_published, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # Noncontrib user_two = AuthUserFactory() res = app.get(url_published, auth=user_two.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # Write contrib preprint_published.add_contributor(user_two, permissions.WRITE, save=True) res = app.get(url_published, auth=user_two.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # Admin contrib res = app.get(url_published, auth=user.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 def test_preprint_contributor_private( self, app, user, preprint_published, url_published): diff --git a/api_tests/preprints/views/test_preprint_contributors_list.py b/api_tests/preprints/views/test_preprint_contributors_list.py index e069ec7d9d9..53da11cb453 100644 --- a/api_tests/preprints/views/test_preprint_contributors_list.py +++ b/api_tests/preprints/views/test_preprint_contributors_list.py @@ -205,20 +205,20 @@ def test_return_preprint_contributors_deleted_preprint( # test_deleted_preprint_contributors_logged_out res = app.get(url_published, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # test_deleted_preprint_contributor_non_contrib res = app.get(url_published, auth=user_two.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # test_deleted_preprint_contributors_read_contrib_logged_out preprint_published.add_contributor(user_two, permissions.READ, save=True) res = app.get(url_published, auth=user_two.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # test_deleted_preprint_contributors_admin res = app.get(url_published, auth=user.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 def test_return_preprint_contributors_abandoned_preprint( self, app, user, user_two, preprint_published, url_published): diff --git a/api_tests/preprints/views/test_preprint_detail.py b/api_tests/preprints/views/test_preprint_detail.py index 283f7029c9f..1c08e34bc85 100644 --- a/api_tests/preprints/views/test_preprint_detail.py +++ b/api_tests/preprints/views/test_preprint_detail.py @@ -179,6 +179,26 @@ def test_return_affiliated_institutions(self, app, user, preprint, institution, relationship_link = res.json['data']['relationships']['affiliated_institutions']['links']['self']['href'] assert f'/v2/preprints/{preprint._id}/relationships/institutions/' in relationship_link + def test_spammed_preprint_detail_gone(self, app, preprint, user, url): + preprint.confirm_spam(save=True, train_spam_services=False) + url = f'/{API_BASE}preprints/{preprint._id}/' + res = app.get(url, expect_errors=True) + assert res.status_code == 410 + error = res.json['errors'][0] + assert error['detail'] == 'The requested preprint is no longer available.' + assert 'meta' in error + assert error['meta']['flagged_content'] is True + + def test_not_spammed_deleted_preprint_detail_gone(self, app, preprint, user, url): + preprint.deleted = timezone.now() + preprint.save() + res = app.get(url, expect_errors=True) + assert res.status_code == 410 + error = res.json['errors'][0] + assert error['detail'] == 'The requested preprint is no longer available.' + assert 'meta' in error + assert not error['meta'].get('flagged_content', False) + @pytest.mark.django_db class TestPreprintDelete: diff --git a/api_tests/preprints/views/test_preprint_files_list.py b/api_tests/preprints/views/test_preprint_files_list.py index 6b7af924e60..2443f429111 100644 --- a/api_tests/preprints/views/test_preprint_files_list.py +++ b/api_tests/preprints/views/test_preprint_files_list.py @@ -107,20 +107,20 @@ def test_deleted_preprint_files(self): # Unauthenticated res = self.app.get(self.url, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # Noncontrib res = self.app.get(self.url, auth=self.user_two.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # Write contributor self.preprint.add_contributor(self.user_two, WRITE, save=True) res = self.app.get(self.url, auth=self.user_two.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 # Admin contrib res = self.app.get(self.url, auth=self.user.auth, expect_errors=True) - assert res.status_code == 404 + assert res.status_code == 410 def test_withdrawn_preprint_files(self): self.preprint.date_withdrawn = timezone.now() diff --git a/api_tests/registrations/views/test_registration_detail.py b/api_tests/registrations/views/test_registration_detail.py index 17649ad7b8d..9d90703ed75 100644 --- a/api_tests/registrations/views/test_registration_detail.py +++ b/api_tests/registrations/views/test_registration_detail.py @@ -233,6 +233,29 @@ def test_registration_detail( expected_url = f'{public_url}relationships/subjects/' assert urlparse(self_url).path == expected_url + def test_spammed_registration_detail_gone(self, app, user, public_registration, private_registration): + for registration in (public_registration, private_registration): + registration.confirm_spam(save=True, train_spam_services=False) + url = f'/{API_BASE}registrations/{registration._id}/' + res = app.get(url, expect_errors=True) + assert res.status_code == 410 + error = res.json['errors'][0] + assert error['detail'] == 'The requested registration is no longer available.' + assert 'meta' in error + assert error['meta']['flagged_content'] + + def test_not_spammed_detailed_registration_detail_gone(self, app, user, public_registration, private_registration): + for registration in (public_registration, private_registration): + registration.deleted = timezone.now() + registration.save() + url = f'/{API_BASE}registrations/{registration._id}/' + res = app.get(url, expect_errors=True) + assert res.status_code == 410 + error = res.json['errors'][0] + assert error['detail'] == 'The requested registration is no longer available.' + assert 'meta' in error + assert not error['meta'].get('flagged_content', False) + class TestRegistrationUpdateTestCase: diff --git a/osf_tests/test_generate_sitemap.py b/osf_tests/test_generate_sitemap.py index a84bb1bdcfd..675901b7922 100644 --- a/osf_tests/test_generate_sitemap.py +++ b/osf_tests/test_generate_sitemap.py @@ -125,6 +125,33 @@ def preprint_other(self, project_preprint_other, user_admin_project_public, prov creator=user_admin_project_public, provider=provider_other) + @pytest.fixture(autouse=True) + def project_spammed(self, user_admin_project_public): + project = ProjectFactory(creator=user_admin_project_public, is_public=True) + project.confirm_spam(save=True, train_spam_services=False) + return project + + @pytest.fixture(autouse=True) + def preprint_spammed(self, project_preprint_osf, user_admin_project_public, provider_osf): + preprint = PreprintFactory( + project=project_preprint_osf, + creator=user_admin_project_public, + provider=provider_osf + ) + preprint.confirm_spam(save=True, train_spam_services=False) + return preprint + + @pytest.fixture(autouse=True) + def registration_spammed(self, user_admin_project_public, project_registration_public): + registration = RegistrationFactory( + project=project_registration_public, + creator=user_admin_project_public, + is_public=True + ) + # Flag the registration as spam + registration.confirm_spam(save=True, train_spam_services=False) + return registration + @pytest.fixture(autouse=True) def all_included_links(self, user_admin_project_public, user_admin_project_private, project_registration_public, project_preprint_osf, project_preprint_other, @@ -199,3 +226,23 @@ def test_deleted_project_link_not_included(self, project_deleted, create_tmp_dir urls = get_all_sitemap_urls() assert urljoin(settings.DOMAIN, project_deleted.url + 'overview') not in urls + + def test_spammed_project_link_not_included(self, project_spammed, create_tmp_directory): + with mock.patch('website.settings.STATIC_FOLDER', create_tmp_directory): + urls = get_all_sitemap_urls() + + assert urljoin(settings.DOMAIN, project_spammed.url + 'overview') not in urls + + def test_spammed_preprint_link_not_included(self, preprint_spammed, provider_osf, create_tmp_directory): + with mock.patch('website.settings.STATIC_FOLDER', create_tmp_directory): + urls = get_all_sitemap_urls() + + spammed_url = urljoin(settings.DOMAIN, f'/preprints/{provider_osf._id}/{preprint_spammed._id}') + assert spammed_url not in urls + + def test_spammed_registration_link_not_included(self, registration_spammed, create_tmp_directory): + with mock.patch('website.settings.STATIC_FOLDER', create_tmp_directory): + urls = get_all_sitemap_urls() + + # Verify the spammed registration's overview page does not make it into the XML + assert urljoin(settings.DOMAIN, registration_spammed.url + 'overview') not in urls diff --git a/scripts/generate_sitemap.py b/scripts/generate_sitemap.py index 6039eb56025..7ab62c42a99 100644 --- a/scripts/generate_sitemap.py +++ b/scripts/generate_sitemap.py @@ -19,6 +19,7 @@ from framework.celery_tasks import app as celery_app from django.db.models import Q, F, OuterRef, Subquery from osf.models import OSFUser, AbstractNode, Preprint, PreprintProvider +from osf.models.spam import SpamStatus from osf.utils.workflows import DefaultStates from scripts import utils as script_utils from website import settings @@ -185,7 +186,7 @@ def generate(self): # AbstractNode urls (Nodes and Registrations, no Collections) objs = (AbstractNode.objects .filter(is_public=True, is_deleted=False, retraction_id__isnull=True) - .exclude(type__in=['osf.collection']) + .exclude(type__in=['osf.collection'], spam_status__in=[SpamStatus.SPAM, SpamStatus.FLAGGED]) .values('guids___id', 'modified')) progress.start(objs.count(), 'NODE: ') for obj in objs: @@ -198,8 +199,8 @@ def generate(self): self.log_errors('NODE', obj['guids___id'], e) progress.increment() progress.stop() - #Removed previous logic as it blocked withdrawn preprints to get in sitemap generator - objs = Preprint.objects.filter(date_published__isnull=False).annotate( + objs = Preprint.objects.filter( + date_published__isnull=False).exclude(spam_status__in=[SpamStatus.SPAM, SpamStatus.FLAGGED]).annotate( most_recent_non_withdrawn=Subquery( Preprint.objects.filter( guids=OuterRef('guids')