Merge branch 'release/23.12.0'

.pre-commit-config.yaml

@@ -12,7 +12,7 @@ repos:
   hooks:
   - id: double-quote-string-fixer
   - id: trailing-whitespace
-    exclude: website/static/vendor/*
+    exclude: "^(website/static/vendor/|osf/metadata/definitions/.*\\.xsd)"
   - id: flake8
     additional_dependencies: ["flake8==3.6.0", "flake8-mutable==1.2.0"]
 - repo: https://github.com/pre-commit/mirrors-jshint

CHANGELOG

@@ -2,6 +2,11 @@
 
 We follow the CalVer (https://calver.org/) versioning scheme: YY.MINOR.MICRO.
 
+23.12.0 (2023-8-22)
+===================
+- Search Improvements
+- Serialize IRI-centric metadata for SHARE
+
 23.11.0 (2023-8-17)
 ===================
 - Allow folder-level connections with S3

addons/osfstorage/models.py

@@ -255,6 +255,41 @@ def last_known_metadata(self):
             'last_seen': self.modified
         }
 
+    @property
+    def should_update_search(self):
+        target = self.target
+        qa_tags = set(website_settings.DO_NOT_INDEX_LIST['tags'])
+        if qa_tags.intersection(self.tags.all().values_list('name', flat=True)):
+            return False
+        if qa_tags.intersection(target.tags.all().values_list('name', flat=True)):
+            return False
+        if any(qa_substring in target.title for qa_substring in website_settings.DO_NOT_INDEX_LIST['titles']):
+            return False
+
+        if not self.name or self.is_deleted:
+            return False
+
+        spam_check_field = (
+            'is_spammy'
+            if website_settings.SPAM_FLAGGED_REMOVE_FROM_SEARCH
+            else 'is_spam'
+        )
+        target_check_fields = [
+            ('is_public', True),
+            ('deleted', None),
+            ('is_deleted', False),
+            ('is_retracted', False),
+            (spam_check_field, False),
+            ('archiving', False),
+            ('verified_publishable', True),
+            ('primary_file', self)
+        ]
+        for field_name, expected_value in target_check_fields:
+            if hasattr(target, field_name) and getattr(target, field_name, expected_value) != expected_value:
+                return False
+
+        return True
+
     def touch(self, bearer, version=None, revision=None, **kwargs):
         try:
             return self.get_version(revision or version)
@@ -392,17 +427,14 @@ def remove_tags(self, tags, auth, save=True):
             self.remove_tag(tag, auth, save)
 
     def delete(self, user=None, **kwargs):
-        from website.search import search
-
-        search.update_file(self, delete=True)
-        return super().delete(user, **kwargs)
+        ret = super().delete(user, **kwargs)
+        self.update_search()
+        return ret
 
     def save(self, skip_search=False, *args, **kwargs):
-        from website.search import search
-
         ret = super(OsfStorageFile, self).save()
         if not skip_search:
-            search.update_file(self)
+            self.update_search()
         return ret
 
 

admin/nodes/views.py

@@ -508,9 +508,7 @@ class NodeReindexShare(NodeMixin, View):
 
     def post(self, request, *args, **kwargs):
         node = self.get_object()
-        if settings.SHARE_ENABLED:
-            update_share(node)
-
+        update_share(node)
         update_admin_log(
             user_id=self.request.user.id,
             object_id=node._id,

admin/preprint_providers/views.py

@@ -384,7 +384,7 @@ def get(self, request, *args, **kwargs):
         except ValidationError as e:
             messages.error(request, e.message)
 
-        return redirect(reverse_lazy('preprint_providers:share_source', kwargs={'preprint_provider_id': provider.id}))
+        return redirect(reverse_lazy('preprint_providers:detail', kwargs={'preprint_provider_id': provider.id}))
 
 class SubjectDynamicUpdateView(PermissionRequiredMixin, View):
     permission_required = 'osf.change_preprintprovider'

admin/preprints/views.py

@@ -42,7 +42,7 @@
     UNFLAG_SPAM,
 )
 
-from website import search, settings
+from website import search
 
 
 class PreprintMixin(PermissionRequiredMixin):
@@ -141,8 +141,7 @@ class PreprintReindexShare(PreprintMixin, View):
 
     def post(self, request, *args, **kwargs):
         preprint = self.get_object()
-        if settings.SHARE_ENABLED:
-            update_share(preprint)
+        update_share(preprint)
         update_admin_log(
             user_id=self.request.user.id,
             object_id=preprint._id,

admin_tests/nodes/test_views.py

@@ -1,10 +1,8 @@
 import datetime as dt
 import pytest
-import json
 import mock
 import pytz
 import datetime
-import responses
 
 from osf.models import AdminLogEntry, NodeLog, AbstractNode
 from admin.nodes.views import (
@@ -22,6 +20,7 @@
     RemoveStuckRegistrationsView
 )
 from admin_tests.utilities import setup_log_view, setup_view
+from api_tests.share._utils import mock_update_share
 from website import settings
 from nose import tools as nt
 from django.utils import timezone
@@ -265,32 +264,19 @@ def test_reindex_node_share(self):
         count = AdminLogEntry.objects.count()
         view = NodeReindexShare()
         view = setup_log_view(view, self.request, guid=self.node._id)
-        with mock.patch('api.share.utils.settings.SHARE_ENABLED', True):
-            with mock.patch('api.share.utils.settings.SHARE_API_TOKEN', 'mock-api-token'):
-                with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps:
-                    rsps.add(responses.POST, 'https://share.osf.io/api/v2/normalizeddata/')
-                    view.post(self.request)
-                    data = json.loads(rsps.calls[-1].request.body.decode())
-
-                    share_graph = data['data']['attributes']['data']['@graph']
-                    identifier_node = next(n for n in share_graph if n['@type'] == 'workidentifier')
-                    assert identifier_node['creative_work']['@type'] == 'project'
-                    nt.assert_equal(AdminLogEntry.objects.count(), count + 1)
+        with mock_update_share() as _shmock:
+            view.post(self.request)
+            assert _shmock.called_once_with(self.node)
+        nt.assert_equal(AdminLogEntry.objects.count(), count + 1)
 
     def test_reindex_registration_share(self):
         count = AdminLogEntry.objects.count()
         view = NodeReindexShare()
         view = setup_log_view(view, self.request, guid=self.registration._id)
-        with mock.patch('api.share.utils.settings.SHARE_ENABLED', True):
-            with mock.patch('api.share.utils.settings.SHARE_API_TOKEN', 'mock-api-token'):
-                with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps:
-                    rsps.add(responses.POST, 'https://share.osf.io/api/v2/normalizeddata/')
-                    view.post(self.request)
-                    data = json.loads(rsps.calls[-1].request.body.decode())
-
-                    assert any(graph for graph in data['data']['attributes']['data']['@graph']
-                               if graph['@type'] == self.registration.provider.share_publish_type.lower())
-                    nt.assert_equal(AdminLogEntry.objects.count(), count + 1)
+        with mock_update_share() as _shmock:
+            view.post(self.request)
+            assert _shmock.called_once_with(self.registration)
+        nt.assert_equal(AdminLogEntry.objects.count(), count + 1)
 
     @mock.patch('website.search.search.update_node')
     def test_reindex_node_elastic(self, mock_update_node):

admin_tests/preprints/test_views.py

@@ -1,7 +1,5 @@
 import pytest
 import mock
-import json
-import responses
 
 from django.test import RequestFactory
 from django.urls import reverse
@@ -334,22 +332,17 @@ def test_change_preprint_provider_subjects_change_permissions(self, plain_view,
 @pytest.mark.enable_implicit_clean
 class TestPreprintReindex:
 
-    def test_reindex_preprint_share(self, preprint, req):
+    def test_reindex_preprint_share(self, preprint, req, mock_update_share):
         preprint.provider.access_token = 'totally real access token I bought from a guy wearing a trenchcoat in the summer'
         preprint.provider.save()
 
         count = AdminLogEntry.objects.count()
         view = views.PreprintReindexShare()
         view = setup_log_view(view, req, guid=preprint._id)
-
-        with mock.patch('api.share.utils.settings.SHARE_ENABLED', True):
-            with mock.patch('api.share.utils.settings.SHARE_API_TOKEN', 'mock-api-token'):
-                with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps:
-                    rsps.add(responses.POST, 'https://share.osf.io/api/v2/normalizeddata/')
-                    view.post(req)
-                    data = json.loads(rsps.calls[0].request.body.decode())
-                    assert data['data']['type'] == 'NormalizedData'
-                    assert AdminLogEntry.objects.count() == count + 1
+        mock_update_share.reset_mock()
+        view.post(req)
+        mock_update_share.assert_called_once_with(preprint)
+        assert AdminLogEntry.objects.count() == count + 1
 
     @mock.patch('website.search.search.update_preprint')
     def test_reindex_preprint_elastic(self, mock_update_search, preprint, req):

admin_tests/registration_providers/test_views.py

@@ -144,9 +144,9 @@ def view(self, req, provider):
         view = views.ShareSourceRegistrationProvider()
         return setup_view(view, req, registration_provider_id=provider.id)
 
-    def test_share_source(self, mock_share, view, provider, req):
-        mock_share.reset()
-        mock_share.add(
+    def test_share_source(self, mock_share_responses, view, provider, req):
+        mock_share_responses.reset()
+        mock_share_responses.add(
             responses.POST,
             f'{settings.SHARE_URL}api/v2/sources/',
             json.dumps(
@@ -172,9 +172,9 @@ def test_share_source(self, mock_share, view, provider, req):
         assert provider.access_token == 'test access token'
 
     @mock.patch.object(settings, 'SHARE_PROVIDER_PREPEND', 'testenv')
-    def test_share_source_prefix(self, mock_share, view, provider, req):
-        mock_share.reset()
-        mock_share.add(
+    def test_share_source_prefix(self, mock_share_responses, view, provider, req):
+        mock_share_responses.reset()
+        mock_share_responses.add(
             responses.POST,
             f'{settings.SHARE_URL}api/v2/sources/',
             json.dumps(

api/base/storage.py

@@ -3,7 +3,7 @@
 from storages.backends.gcloud import GoogleCloudStorage
 from future.moves.urllib.parse import urljoin
 
-from website.settings import DOMAIN
+from website.settings import INTERNAL_DOMAIN
 
 
 class DevFileSystemStorage(FileSystemStorage):
@@ -14,7 +14,7 @@ def url(self, name):
         url = filepath_to_uri(name)
         if url is not None:
             url = url.lstrip('/')
-        url = urljoin(DOMAIN, url)
+        url = urljoin(INTERNAL_DOMAIN, url)
         return urljoin(self.base_url, url)
 
 class RequestlessURLGoogleCloudStorage(GoogleCloudStorage):