Merge branch 'develop' into feature/peace-corpora

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,66 @@
+---
+name: Bug report
+description: Let us know that something isn't working right
+labels:
+  - bug
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thank you for making a bug report! Please fill in this information so we can get to the
+        bottom of your issue.
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What went wrong?
+      description: Please describe what happened.
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: What did you expect to happen?
+    validations:
+      required: true
+  - type: textarea
+    id: screenshot
+    attributes:
+      label: Screenshot
+      description: If you can make a screenshot of the issue, please include it!
+    validations:
+      required: false
+  - type: checkboxes
+    id: instance
+    attributes:
+      label: Where did you find the bug?
+      description: Please add where you found the bug.
+      options:
+        - label: https://ianalyzer.hum.uu.nl
+        - label: https://peopleandparliament.hum.uu.nl
+        - label: https://peace.sites.uu.nl
+        - label: a server hosted elsewhere (i.e. not by the research software lab)
+        - label: a local server
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: Version
+      description: |
+        For third-party and local servers, please add information about the version of the
+        software, if you know it. A version number (e.g "1.2.3") is great. For a pre-release
+        build, you can provide the branch or commit hash.
+    validations:
+      required: false
+  - type: textarea
+    id: to-reproduce
+    attributes:
+      label: Steps to reproduce
+      description: |
+        How can a developer replicate the issue? Please provide any information you can. For
+        example: "I went to
+        https://ianalyzer.hum.uu.nl/search/troonredes?date=1814-01-01:1972-01-01 and then
+        clicked on Download CSV. I pressed cancel and then I clicked Download CSV again."
+    validations:
+      required: true
+---

CITATION.cff

@@ -35,5 +35,5 @@ keywords:
   - elasticsearch
   - natural language processing
 license: MIT
-version: 4.2.0
-date-released: '2023-09-28'
+version: 5.2.0
+date-released: '2023-10-31'

backend/ianalyzer/settings_saml.py

@@ -171,3 +171,5 @@
         'url': [('https://dig.hum.uu.nl', 'en')],
         },
 }
+
+SAML_GROUP_NAME = 'uu'

backend/tag/migrations/0003_taggeddocument_unique_document_id_for_corpus.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.1.9 on 2023-08-10 10:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('tag', '0002_taggeddocument_delete_taginstance'),
+    ]
+
+    operations = [
+        migrations.AddConstraint(
+            model_name='taggeddocument',
+            constraint=models.UniqueConstraint(fields=('corpus', 'doc_id'), name='unique_document_ID_for_corpus'),
+        ),
+    ]

backend/tag/models.py

@@ -42,3 +42,11 @@ class TaggedDocument(models.Model):
         to=Tag,
         related_name='tagged_docs'
     )
+
+    class Meta:
+        constraints = [
+            UniqueConstraint(
+                fields=['corpus', 'doc_id'],
+                name='unique_document_ID_for_corpus'
+            )
+        ]

backend/tag/tests/test_views.py

@@ -105,6 +105,39 @@ def test_patch_document_tags(auth_client, auth_user_tag, mock_corpus, auth_user_
     assert status.is_success(response.status_code)
     assert auth_user_tag.count == 0
 
+def test_assign_multiple_tags_at_once(auth_client, multiple_tags, mock_corpus, auth_user_corpus_acces):
+    doc = 'test'
+    patch_request = lambda data: auth_client.patch(
+        f'/api/tag/document_tags/{mock_corpus}/{doc}',
+        data,
+        content_type='application/json'
+    )
+
+    response = patch_request({
+        'tags': [tag.id for tag in multiple_tags]
+    })
+    assert status.is_success(response.status_code)
+    doc = TaggedDocument.objects.get(doc_id=doc)
+    assert doc.tags.count() == len(multiple_tags)
+
+def test_assign_multiple_tags_one_by_one(auth_client, multiple_tags, mock_corpus, auth_user_corpus_acces):
+    doc = 'test'
+    patch_request = lambda data: auth_client.patch(
+        f'/api/tag/document_tags/{mock_corpus}/{doc}',
+        data,
+        content_type='application/json'
+    )
+
+    for i in range(len(multiple_tags)):
+        response = patch_request({
+            'tags': [tag.id for tag in multiple_tags][:i+1]
+        })
+
+        assert status.is_success(response.status_code)
+        doc = TaggedDocument.objects.get(doc_id=doc)
+        n_tags = doc.tags.count()
+        assert doc.tags.count() == i + 1
+
 def test_patch_tags_contamination(auth_client, auth_user_tag, admin_user_tag, mock_corpus, mock_corpus_obj, auth_user_corpus_acces):
     '''
     Verify that patching tags does not affect the tags of other users

backend/users/migrations/0005_saml_user_group.py

@@ -0,0 +1,26 @@
+from django.db import migrations
+from users.saml import saml_user_group
+
+def add_saml_users_to_group(apps, schema_editor):
+    CustomUser = apps.get_model('users', 'CustomUser')
+
+    saml_users = CustomUser.objects.filter(saml = True)
+    saml_group = saml_user_group()
+
+    if saml_group:
+        for user in saml_users:
+            user.groups.add(saml_group.id)
+            user.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0004_userprofile'),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            add_saml_users_to_group,
+            reverse_code=migrations.RunPython.noop
+        )
+    ]

backend/users/saml.py

@@ -1,7 +1,20 @@
 from djangosaml2.backends import Saml2Backend
+from django.contrib.auth.models import Group
+from django.conf import settings
 
 class CustomSaml2Backend(Saml2Backend):
     def get_or_create_user(self, *args, **kwargs):
         user, created = super().get_or_create_user(*args, **kwargs)
         user.saml = True
+
+        saml_group = saml_user_group()
+        if saml_group:
+            user.groups.add(saml_group)
+
         return user, created
+
+def saml_user_group():
+    group_name = getattr(settings, 'SAML_GROUP_NAME', None)
+    if group_name:
+        group, _ = Group.objects.get_or_create(name=group_name)
+        return group

backend/visualization/query.py

@@ -131,7 +131,11 @@ def get_filters(query):
 
 def is_date_filter(filter):
     """Checks if a filter object is a date filter"""
-    return has_path(filter, 'range', 'date')
+    range_filters = filter.get('range', dict()).keys()
+    return any(
+        has_path(filter, 'range', filter_name, 'format')
+        for filter_name in range_filters
+    )
 
 
 def parse_date(datestring):
@@ -145,7 +149,11 @@ def get_date_range(query: Dict):
         datefilters = list(filter(is_date_filter, filters))
 
         if len(datefilters):
-            parameters = [f['range']['date'] for f in datefilters]
+            parameters = [
+                data
+                for f in datefilters
+                for data in f['range'].values()
+            ]
             min_dates = [parse_date(p['gte'])
                          for p in parameters if 'gte' in p]
             max_dates = [parse_date(p['lte'])

backend/visualization/tests/test_query.py

@@ -79,3 +79,40 @@ def test_manipulation_is_pure(basic_query):
     assert query.get_date_range(basic_query) == (None, None)
     assert query.get_date_range(narrow_query) == narrow_timeframe
     assert query.get_date_range(wide_query) == wide_timeframe
+
+def test_is_date_filter():
+    assert query.is_date_filter({
+        'range': {
+            'date': {
+                'gte': '1950-01-01',
+                'lte': '1959-12-31',
+                'format': 'yyyy-MM-dd'
+            }
+        }
+    })
+
+    assert query.is_date_filter({
+        'range': {
+            'publication_date': {
+                'gte': '1950-01-01',
+                'lte': '1959-12-31',
+                'format': 'yyyy-MM-dd'
+            }
+        }
+    })
+
+    assert not query.is_date_filter({
+        'range': {
+            'year': {
+                'gte': 1950,
+                'lte': 1959,
+            }
+        }
+    })
+
+def test_get_date_range():
+    min_date = datetime(1800, 1, 1)
+    max_date = datetime(1900, 1, 1)
+    date_filter = query.make_date_filter(min_date, max_date, 'publication_date')
+    q = query.add_filter(query.MATCH_ALL, date_filter)
+    assert query.get_date_range(q) == (min_date, max_date)

documentation/SAML.md

@@ -1,7 +1,13 @@
+# SAML
+
 In order to login with Solis ID, I-analyzer has SAML integration with ITS. For this, it uses the [djangosaml2 library](https://djangosaml2.readthedocs.io/). More information on working with SAML, setting up a local environment to test the SAML integration, etc. can be found [here](https://github.com/UUDigitalHumanitieslab/dh-info/blob/master/SAML.md)
 
 The urls exposed by DjangoSaml2 are included as part of our `users` application, e.g., `<hostname>/users/saml2/login`. DjangoSaml2 takes care of consuming the response from the Identity Provider and logging in the user. The `SAML_ATTRIBUTE_MAPPING` variable contains a dictionary of the data coming in from the identity provider, e.g., `uushortid`, and translating that to the corresponding column in the user table, e.g., `username`. Moreover, the setting `SAML_CREATE_UNKNOWN_USER = True` makes sure that we create a user in our database if it's not present yet.
 
 The only tweaks added on top of the DjangoSaml2 package are:
 - the logic to set the `saml` column to `True` for a user logging in with SAML. The `CustomSaml2Backend` overrides DjangoSaml2's `get_or_create_user` function to take care of this. Note that in the future, we could also turn this field into a `CharField` to keep track of multiple identity providers here.
 - overriding DjangoSaml2's `LogoutView` to make its `post` method `csrf_exempt`. The response from the ITS Identity Provider does not send the csrf cookie in a way that it can be consumed by Django at the moment.
+
+### Authorisation
+
+If you define a `SAML_GROUP_NAME` in settings, SAML users will always be added to a group with that name when they create an account. (The group will be created if it does not exist.) This can be used to give permissions to SAML users. The group is not used to handle authentication, so you can add non-SAML users to it as well.

frontend/package.json

@@ -37,12 +37,11 @@
     "balloon-css": "^0.5.2",
     "bulma": "^0.5.1",
     "bulma-switch": "^2.0.0",
-    "chart.js": "^3.7.1",
-    "chartjs-adapter-moment": "^1.0.0",
-    "chartjs-plugin-zoom": "^1.2.0",
+    "chart.js": "^4.4.0",
+    "chartjs-adapter-moment": "^1.0.1",
+    "chartjs-chart-wordcloud": "^4.3.2",
+    "chartjs-plugin-zoom": "^2.0.1",
     "core-js": "^2.6.11",
-    "d3": "^7.8.5",
-    "d3-cloud": "^1.2.7",
     "file-saver": "^2.0.5",
     "font-awesome": "^4.7.0",
     "html-to-image": "^1.9.0",
@@ -72,8 +71,7 @@
     "@angular-eslint/template-parser": "13.5.0",
     "@angular/compiler-cli": "^13.2.2",
     "@angular/language-service": "^13.2.2",
-    "@types/chart.js": "^2.9.35",
-    "@types/d3": "^7.4.0",
+    "@types/chart.js": "^2.9.37",
     "@types/jasmine": "~3.6.0",
     "@types/jasminewd2": "^2.0.8",
     "@types/node": "^6.14.10",
@@ -104,7 +102,6 @@
   "resolutions": {
     "loader-utils": "^2.0.4",
     "terser": "^5.14.2",
-    "d3-color": "^3.1.0",
     "semver": "^7.5.2"
   }
 }

frontend/src/app/common-test-bed.ts

@@ -20,6 +20,8 @@ import { WordmodelsService } from './services/wordmodels.service';
 import { WordmodelsServiceMock } from '../mock-data/wordmodels';
 import { VisualizationService } from './services/visualization.service';
 import { visualizationServiceMock } from '../mock-data/visualization';
+import { TagService } from './services/tag.service';
+import { TagServiceMock } from '../mock-data/tag';
 
 export const commonTestBed = () => {
     const filteredImports = imports.filter(value => !(value in [HttpClientModule]));
@@ -59,6 +61,10 @@ export const commonTestBed = () => {
         {
             provide: VisualizationService,
             useValue: new visualizationServiceMock(),
+        },
+        {
+            provide: TagService,
+            useValue: new TagServiceMock(),
         }
     );
 

frontend/src/app/corpus-header/corpus-header.component.html

@@ -1,7 +1,7 @@
 <section class="section title-section">
     <div class="container is-readable">
         <div class="level" *ngIf="corpus">
-            <div class="level-right">
+            <div class="level-left">
                 <div class="level-item">
                     <h1 class="title">
                         <span *ngIf="currentPage === 'search'">Search</span>
@@ -12,7 +12,7 @@ <h1 class="title">
                     </h1>
                 </div>
             </div>
-            <div class="level-left">
+            <nav class="level-right" aria-label="secondary navigation">
                 <div class="field has-addons">
                     <div class="control">
                         <a class="button is-medium" [routerLink]="['/search', corpus.name]"
@@ -42,7 +42,7 @@ <h1 class="title">
                         </a>
                     </div>
                 </div>
-            </div>
+            </nav>
         </div>
     </div>
 </section>