[ENG-10309] Add meta to API preprint detail view to indicate spam status

api/base/utils.py

@@ -155,7 +155,11 @@ def get_object_or_error(model_or_qs, query_or_pk=None, request=None, display_nam
         if display_name is None:
             raise Gone
         else:
-            raise Gone(detail=f'The requested {display_name} is no longer available.')
+            AbstractNode = apps.get_model('osf', 'AbstractNode')
+            spammy_node = isinstance(obj, AbstractNode) and obj.is_spammy
+            raise Gone(
+                detail=f'The requested {display_name} is no longer available.', meta={'flagged_content': True} if spammy_node else {},
+            )
     return obj
 
 

api/files/views.py

@@ -51,7 +51,7 @@ def get_file(self, check_permissions=True):
                 raise Gone(detail='The requested file is no longer available.')
 
         if getattr(obj.target, 'deleted', None):
-            raise Gone(detail='The requested file is no longer available')
+            raise Gone(detail='The requested file is no longer available.', meta={'flagged_content': getattr(obj.target, 'is_spammy', False)})
 
         if getattr(obj.target, 'is_retracted', False):
             raise Gone(detail='The requested file is no longer available.')

api/preprints/views.py

@@ -22,7 +22,7 @@
 from api.actions.serializers import ReviewActionSerializer
 from api.actions.views import get_review_actions_queryset
 from api.base.pagination import PreprintContributorPagination
-from api.base.exceptions import Conflict
+from api.base.exceptions import Conflict, Gone
 from api.base.views import JSONAPIBaseView, WaterButlerMixin
 from api.base.filters import ListFilterMixin, PreprintAsTargetFilterMixin, PreprintFilterMixin
 from api.base.parsers import (
@@ -126,16 +126,19 @@ def get_preprint(self, check_object_permissions=True, ignore_404=False):
                 qs = Preprint.published_objects.filter(versioned_guids__guid___id=base_guid_id).order_by('-versioned_guids__version')
                 preprint = qs.select_for_update().first() if check_select_for_update(self.request) else qs.select_related('node').first()
 
+        user = self.request.user
         if not preprint:
             sentry.log_message(f'Preprint not found: [guid={base_guid_id}, version={preprint_version}]')
             if ignore_404:
                 return
             raise NotFound
         if preprint.deleted is not None:
-            sentry.log_message(f'Preprint deleted: [guid={base_guid_id}, version={preprint_version}]')
-            raise NotFound
+            if preprint.is_spammy:
+                raise Gone(detail='The requested preprint is no longer available.', meta={'flagged_content': True})
+            else:
+                sentry.log_message(f'Preprint deleted: [guid={base_guid_id}, version={preprint_version}]')
+                raise Gone(detail='The requested preprint is no longer available.')
 
-        user = self.request.user
         if isinstance(user, AnonymousUser):
             user_is_reviewer = user_is_contributor = False
         else:

api/registrations/views.py

@@ -122,7 +122,7 @@ def get_node(self, check_object_permissions=True, **annotations):
             raise NotFound
 
         if node.deleted:
-            raise Gone(detail='The requested registration is no longer available.')
+            raise Gone(detail='The requested registration is no longer available.', meta={'flagged_content': node.is_spammy})
 
         if check_object_permissions:
             self.check_object_permissions(self.request, node)

api_tests/files/views/test_file_detail.py

@@ -607,6 +607,24 @@ def test_folder_files_relationships_contains_guid_not_id(
         assert node._id in split_href
         assert node.id not in split_href
 
+    def test_spammed_node_file_detail_gone(self, app, node, file_url, user):
+        node.confirm_spam(save=True, train_spam_services=False)
+        res = app.get(file_url, expect_errors=True)
+        assert res.status_code == 410
+        error = res.json['errors'][0]
+        assert error['detail'] == 'The requested file is no longer available.'
+        assert 'meta' in error
+        assert error['meta']['flagged_content']
+
+    def test_deleted_file_not_spammed_gone(self, app, user, file, file_url):
+        file.delete(user=user, save=True)
+        res = app.get(file_url, expect_errors=True)
+        assert res.status_code == 410
+        error = res.json['errors'][0]
+        assert error['detail'] == 'The requested file is no longer available.'
+        assert 'meta' in error
+        assert not error['meta'].get('flagged_content', False)
+
 
 @pytest.mark.django_db
 class TestFileVersionView:

api_tests/identifiers/views/test_identifier_list.py

@@ -380,26 +380,26 @@ def test_preprint_identifier_list_permissions_deleted(
 
         # test_unpublished_preprint_identifier_unauthenticated
         res = app.get(url_preprint_identifier, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # test_unpublished_preprint_identifier_noncontrib_authenticated
         non_contrib = AuthUserFactory()
         res = app.get(url_preprint_identifier, auth=non_contrib.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # test_unpublished_preprint_identifier_admin_authenticated
         res = app.get(url_preprint_identifier, auth=user.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # test_unpublished_preprint_identifier_readcontrib_authenticated
         read_user = AuthUserFactory()
         preprint.add_contributor(read_user, READ, save=True)
         res = app.get(url_preprint_identifier, auth=read_user.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # test_published_preprint_identifier_unauthenticated
         res = app.get(url_preprint_identifier, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
     def test_preprint_identifier_list_permissions_abandoned(
             self, app, user, data_preprint_identifier, preprint, url_preprint_identifier):

api_tests/nodes/views/test_node_detail.py

@@ -571,3 +571,22 @@ def test_current_user_permissions_vol(self, app, user, url_public, project_publi
         private_link.save()
         res = app.get(f'{url_public}?view_only={private_link.key}', auth=user.auth)
         assert [permissions.READ] == res.json['data']['attributes']['current_user_permissions']
+
+    def test_spammed_node_detail_gone(self, app, user, project_public, url_public):
+        project_public.confirm_spam(save=True, train_spam_services=False)
+        res = app.get(url_public, expect_errors=True)
+        assert res.status_code == 410
+        error = res.json['errors'][0]
+        assert error['detail'] == 'The requested node is no longer available.'
+        assert 'meta' in error
+        assert error['meta']['flagged_content']
+
+    def test_not_spammed_deleted_node_detail_gone(self, app, user, project_public, url_public):
+        project_public.is_deleted = True
+        project_public.save()
+        res = app.get(url_public, expect_errors=True)
+        assert res.status_code == 410
+        error = res.json['errors'][0]
+        assert error['detail'] == 'The requested node is no longer available.'
+        assert 'meta' in error
+        assert not error['meta'].get('flagged_content', False)

api_tests/preprints/views/test_preprint_citations.py

@@ -122,21 +122,21 @@ def test_deleted_preprint_citations(self):
 
         # Unauthenticated
         res = self.app.get(self.published_preprint_url, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # Non contrib
         res = self.app.get(self.published_preprint_url, auth=self.other_contrib.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # Write contrib
         self.published_preprint.add_contributor(self.other_contrib, WRITE, save=True)
         res = self.app.get(self.published_preprint_url, auth=self.other_contrib.auth, expect_errors=True)
         # Really because preprint is in initial machine state
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # Admin contrib
         res = self.app.get(self.published_preprint_url, auth=self.admin_contributor.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
     def test_abandoned_preprint_citations(self):
         self.published_preprint.machine_state = DefaultStates.INITIAL.value
@@ -243,21 +243,21 @@ def test_deleted_preprint_citations(self):
 
         # Unauthenticated
         res = self.app.get(self.published_preprint_url, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # Non contrib
         res = self.app.get(self.published_preprint_url, auth=self.other_contrib.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # Write contrib
         self.published_preprint.add_contributor(self.other_contrib, WRITE, save=True)
         res = self.app.get(self.published_preprint_url, auth=self.other_contrib.auth, expect_errors=True)
         # Really because preprint is in initial machine state
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # Admin contrib
         res = self.app.get(self.published_preprint_url, auth=self.admin_contributor.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
     def test_abandoned_preprint_citations(self):
         self.published_preprint.machine_state = DefaultStates.INITIAL.value

api_tests/preprints/views/test_preprint_contributors_detail.py

@@ -194,21 +194,21 @@ def test_preprint_contributor_deleted(
 
         # Unauthenticated
         res = app.get(url_published, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # Noncontrib
         user_two = AuthUserFactory()
         res = app.get(url_published, auth=user_two.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # Write contrib
         preprint_published.add_contributor(user_two, permissions.WRITE, save=True)
         res = app.get(url_published, auth=user_two.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # Admin contrib
         res = app.get(url_published, auth=user.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
     def test_preprint_contributor_private(
             self, app, user, preprint_published, url_published):

api_tests/preprints/views/test_preprint_contributors_list.py

@@ -205,20 +205,20 @@ def test_return_preprint_contributors_deleted_preprint(
 
         # test_deleted_preprint_contributors_logged_out
         res = app.get(url_published, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # test_deleted_preprint_contributor_non_contrib
         res = app.get(url_published, auth=user_two.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # test_deleted_preprint_contributors_read_contrib_logged_out
         preprint_published.add_contributor(user_two, permissions.READ, save=True)
         res = app.get(url_published, auth=user_two.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # test_deleted_preprint_contributors_admin
         res = app.get(url_published, auth=user.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
     def test_return_preprint_contributors_abandoned_preprint(
             self, app, user, user_two, preprint_published, url_published):

api_tests/preprints/views/test_preprint_detail.py

@@ -179,6 +179,26 @@ def test_return_affiliated_institutions(self, app, user, preprint, institution,
         relationship_link = res.json['data']['relationships']['affiliated_institutions']['links']['self']['href']
         assert f'/v2/preprints/{preprint._id}/relationships/institutions/' in relationship_link
 
+    def test_spammed_preprint_detail_gone(self, app, preprint, user, url):
+        preprint.confirm_spam(save=True, train_spam_services=False)
+        url = f'/{API_BASE}preprints/{preprint._id}/'
+        res = app.get(url, expect_errors=True)
+        assert res.status_code == 410
+        error = res.json['errors'][0]
+        assert error['detail'] == 'The requested preprint is no longer available.'
+        assert 'meta' in error
+        assert error['meta']['flagged_content'] is True
+
+    def test_not_spammed_deleted_preprint_detail_gone(self, app, preprint, user, url):
+        preprint.deleted = timezone.now()
+        preprint.save()
+        res = app.get(url, expect_errors=True)
+        assert res.status_code == 410
+        error = res.json['errors'][0]
+        assert error['detail'] == 'The requested preprint is no longer available.'
+        assert 'meta' in error
+        assert not error['meta'].get('flagged_content', False)
+
 
 @pytest.mark.django_db
 class TestPreprintDelete:

api_tests/preprints/views/test_preprint_files_list.py

@@ -107,20 +107,20 @@ def test_deleted_preprint_files(self):
 
         # Unauthenticated
         res = self.app.get(self.url, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # Noncontrib
         res = self.app.get(self.url, auth=self.user_two.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # Write contributor
         self.preprint.add_contributor(self.user_two, WRITE, save=True)
         res = self.app.get(self.url, auth=self.user_two.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
         # Admin contrib
         res = self.app.get(self.url, auth=self.user.auth, expect_errors=True)
-        assert res.status_code == 404
+        assert res.status_code == 410
 
     def test_withdrawn_preprint_files(self):
         self.preprint.date_withdrawn = timezone.now()

api_tests/registrations/views/test_registration_detail.py

@@ -233,6 +233,29 @@ def test_registration_detail(
         expected_url = f'{public_url}relationships/subjects/'
         assert urlparse(self_url).path == expected_url
 
+    def test_spammed_registration_detail_gone(self, app, user, public_registration, private_registration):
+        for registration in (public_registration, private_registration):
+            registration.confirm_spam(save=True, train_spam_services=False)
+            url = f'/{API_BASE}registrations/{registration._id}/'
+            res = app.get(url, expect_errors=True)
+            assert res.status_code == 410
+            error = res.json['errors'][0]
+            assert error['detail'] == 'The requested registration is no longer available.'
+            assert 'meta' in error
+            assert error['meta']['flagged_content']
+
+    def test_not_spammed_detailed_registration_detail_gone(self, app, user, public_registration, private_registration):
+        for registration in (public_registration, private_registration):
+            registration.deleted = timezone.now()
+            registration.save()
+            url = f'/{API_BASE}registrations/{registration._id}/'
+            res = app.get(url, expect_errors=True)
+            assert res.status_code == 410
+            error = res.json['errors'][0]
+            assert error['detail'] == 'The requested registration is no longer available.'
+            assert 'meta' in error
+            assert not error['meta'].get('flagged_content', False)
+
 
 class TestRegistrationUpdateTestCase:
 

osf_tests/test_generate_sitemap.py

@@ -125,6 +125,33 @@ def preprint_other(self, project_preprint_other, user_admin_project_public, prov
                                              creator=user_admin_project_public,
                                              provider=provider_other)
 
+    @pytest.fixture(autouse=True)
+    def project_spammed(self, user_admin_project_public):
+        project = ProjectFactory(creator=user_admin_project_public, is_public=True)
+        project.confirm_spam(save=True, train_spam_services=False)
+        return project
+
+    @pytest.fixture(autouse=True)
+    def preprint_spammed(self, project_preprint_osf, user_admin_project_public, provider_osf):
+        preprint = PreprintFactory(
+            project=project_preprint_osf,
+            creator=user_admin_project_public,
+            provider=provider_osf
+        )
+        preprint.confirm_spam(save=True, train_spam_services=False)
+        return preprint
+
+    @pytest.fixture(autouse=True)
+    def registration_spammed(self, user_admin_project_public, project_registration_public):
+        registration = RegistrationFactory(
+            project=project_registration_public,
+            creator=user_admin_project_public,
+            is_public=True
+        )
+        # Flag the registration as spam
+        registration.confirm_spam(save=True, train_spam_services=False)
+        return registration
+
     @pytest.fixture(autouse=True)
     def all_included_links(self, user_admin_project_public, user_admin_project_private, project_registration_public,
                              project_preprint_osf, project_preprint_other,
@@ -199,3 +226,23 @@ def test_deleted_project_link_not_included(self, project_deleted, create_tmp_dir
             urls = get_all_sitemap_urls()
 
         assert urljoin(settings.DOMAIN, project_deleted.url + 'overview') not in urls
+
+    def test_spammed_project_link_not_included(self, project_spammed, create_tmp_directory):
+        with mock.patch('website.settings.STATIC_FOLDER', create_tmp_directory):
+            urls = get_all_sitemap_urls()
+
+        assert urljoin(settings.DOMAIN, project_spammed.url + 'overview') not in urls
+
+    def test_spammed_preprint_link_not_included(self, preprint_spammed, provider_osf, create_tmp_directory):
+        with mock.patch('website.settings.STATIC_FOLDER', create_tmp_directory):
+            urls = get_all_sitemap_urls()
+
+        spammed_url = urljoin(settings.DOMAIN, f'/preprints/{provider_osf._id}/{preprint_spammed._id}')
+        assert spammed_url not in urls
+
+    def test_spammed_registration_link_not_included(self, registration_spammed, create_tmp_directory):
+        with mock.patch('website.settings.STATIC_FOLDER', create_tmp_directory):
+            urls = get_all_sitemap_urls()
+
+        # Verify the spammed registration's overview page does not make it into the XML
+        assert urljoin(settings.DOMAIN, registration_spammed.url + 'overview') not in urls