diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 2b87893d4..15b83ca73 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -1,4 +1,4 @@ -# This workflow will run backend tests on the Python version defined in the Dockerfiles +# This workflow will run backend tests on the Python version defined in the backend/Dockerfile name: Backend unit tests @@ -13,15 +13,45 @@ on: - 'hotfix/**' - 'release/**' - 'dependabot/**' - paths-ignore: - - 'frontend/**' - - '**.md' + paths: + - 'backend/**' + - '.github/workflows/backend*' + - 'docker-compose.yaml' jobs: backend-test: name: Test Backend runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push Elasticsearch image + uses: docker/build-push-action@v6 + with: + context: . + file: DockerfileElastic + push: true + tags: ghcr.io/uudigitalhumanitieslab/ianalyzer-elastic:latest + cache-from: type=registry,ref=ghcr.io/uudigitalhumanitieslab/ianalyzer-elastic:latest + cache-to: type=inline + - name: Build and push Backend + uses: docker/build-push-action@v6 + with: + context: backend/. + push: true + tags: ghcr.io/uudigitalhumanitieslab/ianalyzer-backend:latest + cache-from: type=registry,ref=ghcr.io/uudigitalhumanitieslab/ianalyzer-backend:latest + cache-to: type=inline - name: Run backend tests - run: sudo mkdir -p /ci-data && sudo docker-compose --env-file .env-ci run backend pytest + run: | + sudo mkdir -p /ci-data + docker compose pull elasticsearch + docker compose pull backend + docker compose --env-file .env-ci run --rm backend pytest diff --git a/.github/workflows/frontend-test.yml b/.github/workflows/frontend-test.yml index fdb14f20e..0e19cb73a 100644 --- a/.github/workflows/frontend-test.yml +++ b/.github/workflows/frontend-test.yml @@ -13,15 +13,34 @@ on: - 'hotfix/**' - 'release/**' - 'dependabot/**' - paths-ignore: - - 'backend/**' - - '**.md' + paths: + - 'frontend/**' + - '.github/workflows/frontend*' + - 'docker-compose.yaml' jobs: frontend-test: name: Test Frontend runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Run frontend tests - run: sudo docker-compose --env-file .env-ci run frontend yarn test + - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build frontend image, using cache from Github registry + uses: docker/build-push-action@v6 + with: + context: frontend/. + push: true + tags: ghcr.io/uudigitalhumanitieslab/ianalyzer-frontend:latest + cache-from: type=registry,ref=ghcr.io/uudigitalhumanitieslab/ianalyzer-frontend:latest + cache-to: type=inline + - name: Run frontend unit tests + run: | + docker compose pull frontend + docker compose --env-file .env-ci run --rm frontend yarn test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index ed2bc732e..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,25 +0,0 @@ -# This action will update the CITATION.cff file for new release or hotfix branches - -name: Release - -on: - push: - branches: - - 'release/**' - - 'hotfix/**' - -jobs: - citation-update: - name: Update CITATION.cff - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Autoformat CITATION.cff - run: | - version=`grep -o '\d\+\.\d\+\.\d\+' package.json` - today=`date +"%Y-%m-%d"` - sed -i "s/^version: [[:digit:]]\{1,\}\.[[:digit:]]\{1,\}\.[[:digit:]]\{1,\}/version: $version/" CITATION.cff - sed -i "s/[[:digit:]]\{4\}-[[:digit:]]\{2\}-[[:digit:]]\{2\}/$today/" CITATION.cff - bash ./update-citation.sh - git commit -a -m "update version and date in CITATION.cff" - diff --git a/.vscode/launch.json b/.vscode/launch.json index 786b5f0a5..953445716 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -61,6 +61,16 @@ } }, { + "name": "Python: Debug Tests", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "purpose": [ + "debug-test" + ], + "console": "internalConsole", + "justMyCode": false + }, { "name": "celery", "type": "debugpy", "request": "launch", diff --git a/CITATION.cff b/CITATION.cff index 69b99051a..413c366c9 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -35,5 +35,5 @@ keywords: - elasticsearch - natural language processing license: MIT -version: 5.9.0 -date-released: '2024-07-05' +version: 5.12.0 +date-released: '2024-08-30' diff --git a/backend/Dockerfile b/backend/Dockerfile index 2c58b766e..aefd4cd76 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -7,7 +7,6 @@ RUN apt-get -y update && apt-get -y upgrade RUN apt-get install -y pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl default-libmysqlclient-dev RUN pip install --upgrade pip -RUN pip install pip-tools # make a directory in the container WORKDIR /backend # copy requirements from the host system to the directory in the container diff --git a/backend/addcorpus/constants.py b/backend/addcorpus/constants.py index c0f19e5d2..12fb38edb 100644 --- a/backend/addcorpus/constants.py +++ b/backend/addcorpus/constants.py @@ -49,9 +49,18 @@ class VisualizationType(Enum): 'scan', 'tab-scan' 'p', + 'tags', + 'context', + 'tab', ] ''' -Field names that cannot be used because they are also query parameters in frontend routes. +Field names that cannot be used because they interfere with other functionality. -Using them would make routing ambiguous. +This is usually because they are also query parameters in frontend routes, and using them +would make routing ambiguous. + +`query` is also forbidden because it is a reserved column in CSV downloads. Likewise, +`context` is forbidden because it's used in download requests. + +`scan` and `tab-scan` are added because they interfere with element IDs in the DOM. ''' diff --git a/backend/addcorpus/models.py b/backend/addcorpus/models.py index 2d6c8927a..98e297537 100644 --- a/backend/addcorpus/models.py +++ b/backend/addcorpus/models.py @@ -268,8 +268,9 @@ def has_named_entities(self): try: mapping = client.indices.get_mapping( index=self.es_index) - fields = mapping[self.es_index].get( - 'mappings', {}).get('properties', {}).keys() + # in production, the index name can be different from the object's es_index value + index_name = list(mapping.keys())[0] + fields = mapping[index_name].get('mappings', {}).get('properties', {}).keys() if any(field.endswith(':ner') for field in fields): return True except: @@ -473,6 +474,13 @@ class PageType(models.TextChoices): help_text='markdown contents of the documentation' ) + @property + def page_index(self): + '''Numerical index to determine the order in which pages should be displayed. + Based on the order in which `PageType` choices are declared.''' + indexed_values = enumerate(__class__.PageType.values) + return next((i for (i, value) in indexed_values if value == self.type), None) + def __str__(self): return f'{self.corpus_configuration.corpus.name} - {self.type}' diff --git a/backend/addcorpus/permissions.py b/backend/addcorpus/permissions.py index 4aead5aa6..4f8df27de 100644 --- a/backend/addcorpus/permissions.py +++ b/backend/addcorpus/permissions.py @@ -1,7 +1,6 @@ from rest_framework import permissions from rest_framework.exceptions import NotFound -from users.models import CustomUser -from typing import List +from rest_framework.request import Request from addcorpus.models import Corpus def corpus_name_from_request(request): @@ -25,20 +24,7 @@ def corpus_name_from_request(request): return corpus -def filter_user_corpora(corpora: List[Corpus], user: CustomUser) -> List[Corpus]: - ''' - Filter all available corpora to only - include the ones the user has access to - ''' - - return [ - corpus - for corpus in corpora - if user.has_access(corpus.name) - ] - - -class CorpusAccessPermission(permissions.BasePermission): +class CanSearchCorpus(permissions.BasePermission): message = 'You do not have permission to access this corpus' def has_permission(self, request, view): @@ -48,9 +34,32 @@ def has_permission(self, request, view): # check if the corpus exists try: corpus = Corpus.objects.get(name=corpus_name) - assert corpus.active except: raise NotFound('Corpus does not exist') # check if the user has access - return user.has_access(corpus) + return user.can_search(corpus) + + +class IsCurator(permissions.BasePermission): + ''' + The user is permitted to use the corpus definition API. + ''' + + message = 'You do not have permission to manage corpus definitions' + + def has_permission(self, request: Request, view): + return request.user.is_staff + +class IsCuratorOrReadOnly(permissions.BasePermission): + ''' + The user is permitted to edit the corpus, or it is a read-only request. + ''' + + message = 'You do not have permission to edit this corpus' + + def has_permission(self, request: Request, view): + if request.method in permissions.SAFE_METHODS: + return True + + return request.user.is_staff diff --git a/backend/addcorpus/reader.py b/backend/addcorpus/reader.py index 2566f6fa6..ca95eea13 100644 --- a/backend/addcorpus/reader.py +++ b/backend/addcorpus/reader.py @@ -36,8 +36,6 @@ class NewReader(CSVReader): for f in corpus.configuration.fields.all()] def sources(self, *args, **kwargs): - return ( - (fn, {}) for fn in glob.glob(f'{self.data_directory}/**/*.csv', recursive=True) - ) + return glob.glob(f'{self.data_directory}/**/*.csv', recursive=True) return NewReader() diff --git a/backend/addcorpus/serializers.py b/backend/addcorpus/serializers.py index b350656a3..12e788717 100644 --- a/backend/addcorpus/serializers.py +++ b/backend/addcorpus/serializers.py @@ -65,6 +65,16 @@ def to_representation(self, value): key = super().to_representation(value) return self.choices[key] + def to_internal_value(self, data): + # If the data provides a display name, get the corresponding key. + # The browsable API sends keys instead of labels; use the original data if no + # matching label is found. + value = next( + (key for (key, label) in self.choices.items() if label == data), + data + ) + return super().to_internal_value(value) + class CorpusConfigurationSerializer(serializers.ModelSerializer): fields = FieldSerializer(many=True, read_only=True) languages = serializers.ListField(child=LanguageField()) @@ -123,11 +133,18 @@ def to_representation(self, value): class CorpusDocumentationPageSerializer(serializers.ModelSerializer): type = PrettyChoiceField(choices = CorpusDocumentationPage.PageType.choices) - content = DocumentationTemplateField() + index = serializers.IntegerField(source='page_index', read_only=True) + content = DocumentationTemplateField(read_only=True) + content_template = serializers.CharField(source='content') + corpus = serializers.SlugRelatedField( + source='corpus_configuration', + queryset=CorpusConfiguration.objects.all(), + slug_field='corpus__name', + ) class Meta: model = CorpusDocumentationPage - fields = ['corpus_configuration', 'type', 'content'] + fields = ['id', 'corpus', 'type', 'content', 'content_template', 'index'] class JSONDefinitionField(serializers.Field): diff --git a/backend/addcorpus/tests/test_corpus_access.py b/backend/addcorpus/tests/test_corpus_access.py index 9e14a4e0d..ca6a8a418 100644 --- a/backend/addcorpus/tests/test_corpus_access.py +++ b/backend/addcorpus/tests/test_corpus_access.py @@ -1,24 +1,33 @@ from users.models import CustomUser, CustomAnonymousUser +from addcorpus.models import Corpus def test_access_through_group(db, basic_mock_corpus, group_with_access): user = CustomUser.objects.create(username='nice-user', password='secret') user.groups.add(group_with_access) user.save() - assert user.has_access(basic_mock_corpus) + corpus = Corpus.objects.get(name=basic_mock_corpus) + assert user.can_search(corpus) + assert corpus in user.searchable_corpora() def test_superuser_access(basic_mock_corpus, admin_user): - assert admin_user.has_access(basic_mock_corpus) + corpus = Corpus.objects.get(name=basic_mock_corpus) + assert admin_user.can_search(corpus) + assert corpus in admin_user.searchable_corpora() def test_no_corpus_access(db, basic_mock_corpus): user = CustomUser.objects.create(username='bad-user', password='secret') - assert not user.has_access(basic_mock_corpus) - + corpus = Corpus.objects.get(name=basic_mock_corpus) + assert not user.can_search(corpus) + assert corpus not in user.searchable_corpora() def test_public_corpus_access(db, basic_corpus_public): user = CustomUser.objects.create(username='new-user', password='secret') - assert user.has_access(basic_corpus_public) + corpus = Corpus.objects.get(name=basic_corpus_public) + assert user.can_search(corpus) + assert corpus in user.searchable_corpora() anon = CustomAnonymousUser() - assert anon.has_access(basic_corpus_public) + assert anon.can_search(corpus) + assert corpus in anon.searchable_corpora() def test_api_access(db, basic_mock_corpus, group_with_access, auth_client, auth_user): # default: no access diff --git a/backend/addcorpus/tests/test_corpus_views.py b/backend/addcorpus/tests/test_corpus_views.py index d0080c053..8cd659bb4 100644 --- a/backend/addcorpus/tests/test_corpus_views.py +++ b/backend/addcorpus/tests/test_corpus_views.py @@ -3,7 +3,7 @@ from typing import Dict from users.models import CustomUser -from addcorpus.models import Corpus +from addcorpus.models import Corpus, CorpusDocumentationPage from addcorpus.python_corpora.save_corpus import load_and_save_all_corpora def test_no_corpora(db, settings, admin_client): @@ -16,23 +16,58 @@ def test_no_corpora(db, settings, admin_client): assert status.is_success(response.status_code) assert response.data == [] -def test_corpus_documentation_view(admin_client, basic_mock_corpus, settings): - response = admin_client.get(f'/api/corpus/documentation/{basic_mock_corpus}/') +def test_corpus_documentation_list_view(admin_client, basic_mock_corpus, settings): + response = admin_client.get(f'/api/corpus/documentation/') assert response.status_code == 200 pages = response.data - # check that the pages are sorted in canonical order - page_types = [page['type'] for page in pages] + # check that the pages specify canonical order + sorted_and_filtered = sorted( + (page for page in pages if page['corpus'] == basic_mock_corpus), + key=lambda page: page['index'] + ) + page_types = [page['type'] for page in sorted_and_filtered] assert page_types == ['General information', 'Citation', 'License'] # should contain citation guidelines - citation_page = next(page for page in pages if page['type'] == 'Citation') + match = { 'type': 'Citation', 'corpus': basic_mock_corpus } + citation_page = next( + page for page in response.data if match.items() <= page.items() + ) # check that the page template is rendered with context content = citation_page['content'] assert '{{ frontend_url }}' not in content assert settings.BASE_URL in content +def test_corpus_documentation_filter_list_view(admin_client, basic_mock_corpus): + response = admin_client.get(f'/api/corpus/documentation/?corpus={basic_mock_corpus}') + assert status.is_success(response.status_code) + pages = response.data + for page in pages: + assert page['corpus'] == basic_mock_corpus + + +def test_corpus_documentation_retrieve_view(admin_client: Client, basic_mock_corpus): + page = CorpusDocumentationPage.objects.first() + response = admin_client.get(f'/api/corpus/documentation/{page.pk}/') + assert status.is_success(response.status_code) + + +def test_corpus_documentation_create_view(admin_client, basic_mock_corpus): + request_data = { + 'corpus': basic_mock_corpus, + 'type': 'Terms of service', + 'content_template': 'You can do whatever you want.' + } + response = admin_client.post( + '/api/corpus/documentation/', + request_data, + content_type='application/json' + ) + assert status.is_success(response.status_code) + + def test_corpus_image_view(admin_client, basic_mock_corpus): corpus = Corpus.objects.get(name=basic_mock_corpus) assert not corpus.configuration.image @@ -47,7 +82,7 @@ def test_corpus_image_view(admin_client, basic_mock_corpus): assert response.status_code == 200 def test_nonexistent_corpus(admin_client): - response = admin_client.get(f'/api/corpus/documentation/unknown-corpus/') + response = admin_client.get(f'/api/corpus/image/unknown-corpus') assert response.status_code == 404 def test_no_corpus_access(db, client, basic_mock_corpus): @@ -55,18 +90,18 @@ def test_no_corpus_access(db, client, basic_mock_corpus): user = CustomUser.objects.create(username='bad-user', password='secret') client.force_login(user) - response = client.get(f'/api/corpus/documentation/{basic_mock_corpus}/') + response = client.get(f'/api/corpus/image/{basic_mock_corpus}') assert response.status_code == 403 -def test_corpus_documentation_unauthenticated(db, client, basic_mock_corpus): +def test_private_corpus_image_unauthenticated(db, client, basic_mock_corpus): response = client.get( - f'/api/corpus/documentation/{basic_mock_corpus}/') + f'/api/corpus/image/{basic_mock_corpus}') assert response.status_code == 401 -def test_public_corpus_documentation_unauthenticated(db, client, basic_corpus_public): +def test_public_corpus_image_unauthenticated(db, client, basic_corpus_public): response = client.get( - f'/api/corpus/documentation/{basic_corpus_public}/') + f'/api/corpus/image/{basic_corpus_public}') assert response.status_code == 200 def test_corpus_serialization(admin_client, basic_mock_corpus): diff --git a/backend/addcorpus/tests/test_models.py b/backend/addcorpus/tests/test_models.py index 4e79e2687..25ba9af6b 100644 --- a/backend/addcorpus/tests/test_models.py +++ b/backend/addcorpus/tests/test_models.py @@ -1,4 +1,4 @@ -from addcorpus.models import Corpus +from addcorpus.models import Corpus, CorpusDocumentationPage def test_corpus_model(db): corpus = Corpus(name = 'test_corpus') @@ -9,3 +9,12 @@ def test_corpus_model(db): corpus.delete() assert not Corpus.objects.filter(name = corpus) + +def test_corpus_documentation_page_model(db, basic_mock_corpus): + corpus = Corpus.objects.get(name=basic_mock_corpus) + page = CorpusDocumentationPage( + corpus_configuration=corpus.configuration, + type=CorpusDocumentationPage.PageType.LICENSE, + content='Do whatever you want.', + ) + assert page.page_index == 2 diff --git a/backend/addcorpus/urls.py b/backend/addcorpus/urls.py index 73ab78cff..b6788d6bf 100644 --- a/backend/addcorpus/urls.py +++ b/backend/addcorpus/urls.py @@ -2,8 +2,7 @@ from addcorpus.views import CorpusImageView, CorpusView, CorpusDocumentationPageViewset, CorpusDocumentView urlpatterns = [ - path('', CorpusView.as_view()), + path('', CorpusView.as_view({'get': 'list'})), path('image/', CorpusImageView.as_view()), - path('documentation//', CorpusDocumentationPageViewset.as_view({'get': 'list'})), path('document//', CorpusDocumentView.as_view()), ] diff --git a/backend/addcorpus/views.py b/backend/addcorpus/views.py index a32064011..652610c98 100644 --- a/backend/addcorpus/views.py +++ b/backend/addcorpus/views.py @@ -1,70 +1,50 @@ from rest_framework.views import APIView from addcorpus.serializers import CorpusSerializer, CorpusDocumentationPageSerializer, CorpusJSONDefinitionSerializer -from rest_framework.response import Response from addcorpus.python_corpora.load_corpus import corpus_dir, load_corpus_definition import os from django.http.response import FileResponse -from rest_framework.permissions import IsAuthenticatedOrReadOnly, IsAdminUser -from addcorpus.permissions import CorpusAccessPermission, filter_user_corpora +from addcorpus.permissions import ( + CanSearchCorpus, corpus_name_from_request, IsCurator, + IsCuratorOrReadOnly) from rest_framework.exceptions import NotFound from rest_framework import viewsets from addcorpus.models import Corpus, CorpusConfiguration, CorpusDocumentationPage -from addcorpus.permissions import corpus_name_from_request from django.conf import settings -class CorpusView(APIView): +class CorpusView(viewsets.ReadOnlyModelViewSet): ''' List all available corpora ''' - permission_classes = (IsAuthenticatedOrReadOnly,) + serializer_class = CorpusSerializer - def get(self, request, *args, **kwargs): - corpora = Corpus.objects.filter(active=True) - filtered_corpora = filter_user_corpora(corpora, request.user) - serializer = CorpusSerializer(filtered_corpora, many=True) - return Response(serializer.data) + def get_queryset(self): + return self.request.user.searchable_corpora() -def send_corpus_file(corpus='', subdir='', filename=''): +class CorpusDocumentationPageViewset(viewsets.ModelViewSet): ''' - Returns a FileResponse for a file in the corpus directory. - - E.g. arguments `(corpus='times', subdir='images', filename='times.jpeg')` will return the file - at `/images/times.jpeg` + Markdown documentation pages for corpora. ''' - path = os.path.join(corpus_dir(corpus), subdir, filename) - - if not os.path.isfile(path): - raise NotFound() - - return FileResponse(open(path, 'rb')) - - -class CorpusDocumentationPageViewset(viewsets.ModelViewSet): - permission_classes = [IsAuthenticatedOrReadOnly, CorpusAccessPermission] + permission_classes = [IsCuratorOrReadOnly] serializer_class = CorpusDocumentationPageSerializer - @staticmethod - def get_relevant_pages(pages, corpus_name): - # only include wordmodels documentation if models are present - if Corpus.objects.get(name=corpus_name).has_python_definition: - definition = load_corpus_definition(corpus_name) - if definition.word_models_present: - return pages - return pages.exclude(type=CorpusDocumentationPage.PageType.WORDMODELS) - def get_queryset(self): - corpus_name = corpus_name_from_request(self.request) - pages = CorpusDocumentationPage.objects.filter( - corpus_configuration__corpus__name=corpus_name) - relevant_pages = self.get_relevant_pages(pages, corpus_name) - canonical_order = [e.value for e in CorpusDocumentationPage.PageType] + # curators are not limited to active corpora (to allow editing) + if self.request.user.is_staff: + corpora = Corpus.objects.all() + else: + corpora = self.request.user.searchable_corpora() + + queried_corpus = self.request.query_params.get('corpus') + if queried_corpus: + corpora = corpora.filter(name=queried_corpus) - return sorted( - relevant_pages, key=lambda p: canonical_order.index(p.type)) + return CorpusDocumentationPage.objects.filter( + corpus_configuration__corpus__in=corpora + ) class CorpusImageView(APIView): @@ -72,7 +52,7 @@ class CorpusImageView(APIView): Return the image for a corpus. ''' - permission_classes = (IsAuthenticatedOrReadOnly,) + permission_classes = [CanSearchCorpus, IsCuratorOrReadOnly] def get(self, request, *args, **kwargs): corpus_name = corpus_name_from_request(request) @@ -87,17 +67,23 @@ def get(self, request, *args, **kwargs): class CorpusDocumentView(APIView): ''' - Return a document for a corpus - e.g. extra metadata. + Return a file for a corpus - e.g. extra metadata. ''' - permission_classes = [IsAuthenticatedOrReadOnly, CorpusAccessPermission] + permission_classes = [CanSearchCorpus] def get(self, request, *args, **kwargs): - return send_corpus_file(subdir='documents', **kwargs) + corpus = Corpus.objects.get(corpus_name_from_request(request)) + if not corpus.has_python_definition: + raise NotFound() + path = os.path.join(corpus_dir(corpus.name), 'documents', kwargs['filename']) + if not os.path.isfile(path): + raise NotFound() + return FileResponse(open(path, 'rb')) class CorpusDefinitionViewset(viewsets.ModelViewSet): - permission_classes = [IsAdminUser] + permission_classes = [IsCurator] serializer_class = CorpusJSONDefinitionSerializer def get_queryset(self): diff --git a/backend/corpora/dbnl/dbnl.py b/backend/corpora/dbnl/dbnl.py index b588d3bb2..76fcb4ed5 100644 --- a/backend/corpora/dbnl/dbnl.py +++ b/backend/corpora/dbnl/dbnl.py @@ -2,6 +2,7 @@ import os import re from tqdm import tqdm +from ianalyzer_readers.xml_tag import Tag, CurrentTag, TransformTag from django.conf import settings from addcorpus.python_corpora.corpus import XMLCorpusDefinition, FieldDefinition @@ -25,8 +26,8 @@ class DBNL(XMLCorpusDefinition): languages = ['nl', 'dum', 'fr', 'la', 'fy', 'lat', 'en', 'nds', 'de', 'af'] category = 'book' - tag_toplevel = 'TEI.2' - tag_entry = { 'name': 'div', 'attrs': {'type': 'chapter'} } + tag_toplevel = Tag('TEI.2') + tag_entry = Tag('div', type='chapter') document_context = { 'context_fields': ['title_id'], @@ -261,18 +262,18 @@ def _xml_files(self): Pass( Backup( XML( # get the language on chapter-level if available + CurrentTag(), attribute='lang', transform=lambda value: [value] if value else None, ), XML( # look for section-level codes - {'name': 'div', 'attrs': {'type': 'section'}}, + Tag('div', type='section'), attribute='lang', multiple=True, ), XML( # look in the top-level metadata - 'language', + Tag('language'), toplevel=True, - recursive=True, multiple=True, attribute='id' ), @@ -298,17 +299,17 @@ def _xml_files(self): extractor=Pass( Backup( XML( # get the language on chapter-level if available + CurrentTag(), attribute='lang', ), XML( # look for section-level code - {'name': 'div', 'attrs': {'type': 'section'}}, + Tag('div', type='section'), attribute='lang' ), XML( #otherwise, get the (first) language for the book - 'language', + Tag('language'), attribute='id', toplevel=True, - recursive=True, ), transform=utils.single_language_code, ), @@ -322,13 +323,11 @@ def _xml_files(self): display_name='Chapter', extractor=Backup( XML( - tag='head', - recursive=True, + Tag('head'), flatten=True, ), XML( - tag=utils.LINE_TAG, - recursive=True, + Tag(utils.LINE_TAG), flatten=True, ) ), @@ -359,11 +358,11 @@ def _xml_files(self): search_field_core=True, csv_core=True, extractor=XML( - tag=utils.LINE_TAG, - recursive=True, + Tag(utils.LINE_TAG), + TransformTag(utils.pad_content), multiple=True, flatten=True, - transform_soup_func=utils.pad_content, + transform=lambda lines: '\n'.join(lines).strip() if lines else None, ), es_mapping=main_content_mapping(token_counts=True), visualizations=['wordcloud'], diff --git a/backend/corpora/dbnl/tests/test_dbnl_extraction.py b/backend/corpora/dbnl/tests/test_dbnl_extraction.py index 2c6d976c4..bc6ed6063 100644 --- a/backend/corpora/dbnl/tests/test_dbnl_extraction.py +++ b/backend/corpora/dbnl/tests/test_dbnl_extraction.py @@ -145,12 +145,12 @@ def test_append_to_tag(xml, tag, padding, original_output, new_output): 'content': '\n'.join([ 'Register der Liedekens.', 'A.', - 'ACh gesalfde van den Heer. Pag. 30 ', - 'Als Saul, en david den vyant in\'t velt. 41 ', - 'Als ick de Son verhoogen sie. 184 ', - 'Als hem de Son begeeft. 189 ', - 'Als ick den Herfst aenschou. 194 ', - 'Als in koelt, de nacht komt overkleeden 208 ', + 'ACh gesalfde van den Heer. Pag. 30', + 'Als Saul, en david den vyant in\'t velt. 41', + 'Als ick de Son verhoogen sie. 184', + 'Als hem de Son begeeft. 189', + 'Als ick den Herfst aenschou. 194', + 'Als in koelt, de nacht komt overkleeden 208', 'Als van der meer op Eng\'le-vleug\'len vloog. 232', ]) }, { # metadata-only book @@ -194,6 +194,8 @@ def test_dbnl_extraction(dbnl_corpus): for actual, expected in zip(docs, expected_docs): # assert that actual is a superset of expected for key in expected: + if expected[key] != actual[key]: + print(key) assert expected[key] == actual[key] assert expected.items() <= actual.items() diff --git a/backend/corpora/dbnl/utils.py b/backend/corpora/dbnl/utils.py index 6a819425f..029f7388b 100644 --- a/backend/corpora/dbnl/utils.py +++ b/backend/corpora/dbnl/utils.py @@ -183,7 +183,8 @@ def append_to_tag(soup, tag, padding): def pad_content(node): pad_cells = lambda n: append_to_tag(n, 'cell', ' ') pad_linebreaks = lambda n: append_to_tag(n, 'lb', '\n') - return pad_cells(pad_linebreaks(node)) + pad_cells(pad_linebreaks(node)) + return [node] def standardize_language_code(code): if code: diff --git a/backend/corpora/dutchannualreports/dutchannualreports.py b/backend/corpora/dutchannualreports/dutchannualreports.py index e9d4993c5..32ca73396 100644 --- a/backend/corpora/dutchannualreports/dutchannualreports.py +++ b/backend/corpora/dutchannualreports/dutchannualreports.py @@ -4,6 +4,7 @@ import os.path as op import logging from datetime import datetime +from ianalyzer_readers.xml_tag import Tag from django.conf import settings @@ -20,7 +21,6 @@ class DutchAnnualReports(XMLCorpusDefinition): """ Alto XML corpus of Dutch annual reports. """ - # Data overrides from .common.Corpus (fields at bottom of class) title = "Dutch Annual Reports" description = "Annual reports of Dutch financial and non-financial institutes" min_date = datetime(year=1957, month=1, day=1) @@ -38,9 +38,8 @@ class DutchAnnualReports(XMLCorpusDefinition): mimetype = 'application/pdf' - # Data overrides from .common.XMLCorpus - tag_toplevel = 'alto' - tag_entry = 'Page' + tag_toplevel = Tag('alto') + tag_entry = Tag('Page') # New data members non_xml_msg = 'Skipping non-XML file {}' @@ -187,9 +186,8 @@ def sources(self, start=min_date, end=max_date): description='Text content of the page.', results_overview=True, extractor=XML( - tag='String', + Tag('String'), attribute='CONTENT', - recursive=True, multiple=True, transform=lambda x: ' '.join(x), ), diff --git a/backend/corpora/dutchnewspapers/dutchnewspapers_public.py b/backend/corpora/dutchnewspapers/dutchnewspapers_public.py index 19e5c4377..9d295f82b 100644 --- a/backend/corpora/dutchnewspapers/dutchnewspapers_public.py +++ b/backend/corpora/dutchnewspapers/dutchnewspapers_public.py @@ -7,6 +7,7 @@ from datetime import datetime from os.path import join, split, splitext import os +from ianalyzer_readers.xml_tag import Tag, SiblingTag from django.conf import settings @@ -43,8 +44,9 @@ class DutchNewspapersPublic(XMLCorpusDefinition): def es_settings(self): return es_settings(self.languages[:1], stopword_analysis=True, stemming_analysis=True) - tag_toplevel = 'text' - tag_entry = 'p' + tag_toplevel = Tag('text') + tag_entry = Tag('p') + external_file_tag_toplevel = Tag('DIDL') # New data members definition_pattern = re.compile(r'didl') @@ -137,18 +139,10 @@ def fields(self): description="Link to record on Delpher", display_type='url', es_mapping=keyword_mapping(), - extractor=XML(tag='identifier', - toplevel=True, - recursive=True, - multiple=False, - secondary_tag={ - 'tag': 'recordIdentifier', - 'match': 'id' - }, - external_file={ - 'xml_tag_toplevel': 'DIDL', - 'xml_tag_entry': 'dcx' - } + extractor=XML( + lambda metadata: Tag('recordIdentifier', string=metadata['id']), + SiblingTag('identifier'), + external_file=True ) ), FieldDefinition( @@ -179,13 +173,9 @@ def fields(self): 'indicator is in this range.' ) ), - extractor=XML(tag='OCRConfidencelevel', - toplevel=True, - recursive=True, - external_file={ - 'xml_tag_toplevel': 'DIDL', - 'xml_tag_entry': 'dcx' - }, + extractor=XML( + Tag('OCRConfidencelevel'), + external_file=True, transform=lambda x: float(x)*100 ), sortable=True @@ -225,19 +215,11 @@ def fields(self): description='Whether the item is an article, advertisment, etc.', csv_core=True, es_mapping={'type': 'keyword'}, - extractor=XML(tag='subject', - toplevel=True, - recursive=True, - multiple=False, - secondary_tag={ - 'tag': 'recordIdentifier', - 'match': 'id' - }, - external_file={ - 'xml_tag_toplevel': 'DIDL', - 'xml_tag_entry': 'dcx' - } - ), + extractor=XML( + lambda metadata: Tag('recordIdentifier', string=metadata['id']), + SiblingTag('subject'), + external_file=True + ), search_filter=filters.MultipleChoiceFilter( description='Accept only articles in these categories.', option_count=2, @@ -276,7 +258,7 @@ def fields(self): description='Article title', results_overview=True, search_field_core=True, - extractor=XML(tag='title', flatten=True, toplevel=True) + extractor=XML(Tag('title'), flatten=True, toplevel=True) ), FieldDefinition( name='id', @@ -320,8 +302,13 @@ def fields(self): es_mapping=main_content_mapping(True, True, True, 'nl'), results_overview=True, search_field_core=True, - extractor=XML(tag='p', multiple=True, - flatten=True, toplevel=True), + extractor=XML( + Tag('p'), + multiple=True, + flatten=True, + toplevel=True, + transform='\n'.join, + ), visualizations=["wordcloud"], language='nl', ), diff --git a/backend/corpora/ecco/ecco.py b/backend/corpora/ecco/ecco.py index 863c08433..54e095aaf 100644 --- a/backend/corpora/ecco/ecco.py +++ b/backend/corpora/ecco/ecco.py @@ -7,6 +7,7 @@ from datetime import datetime import logging import re +from ianalyzer_readers.xml_tag import Tag from django.conf import settings @@ -37,8 +38,8 @@ class Ecco(XMLCorpusDefinition): languages = ['en', 'fr', 'la', 'grc', 'de', 'it', 'cy', 'ga', 'gd'] category = 'book' - tag_toplevel = 'pageContent' - tag_entry = 'page' + tag_toplevel = Tag('pageContent') + tag_entry = Tag('page') meta_pattern = re.compile('^\d+\_DocMetadata\.xml$') @@ -153,8 +154,7 @@ def fields(self): description='Text content.', results_overview=True, search_field_core=True, - extractor=XML(tag='ocrText', - flatten=True), + extractor=XML(Tag('ocrText'), flatten=True), visualizations=['wordcloud'] ), FieldDefinition( diff --git a/backend/corpora/guardianobserver/guardianobserver.py b/backend/corpora/guardianobserver/guardianobserver.py index 54737e274..be6f0f658 100644 --- a/backend/corpora/guardianobserver/guardianobserver.py +++ b/backend/corpora/guardianobserver/guardianobserver.py @@ -11,6 +11,7 @@ from datetime import datetime from zipfile import ZipFile from io import BytesIO +from ianalyzer_readers.xml_tag import Tag from django.conf import settings @@ -46,7 +47,7 @@ class GuardianObserver(XMLCorpusDefinition): def es_settings(self): return es_settings(self.languages[:1], stopword_analysis=True, stemming_analysis=True) - tag_toplevel = 'Record' + tag_toplevel = Tag('Record') def sources(self, start=datetime.min, end=datetime.max): ''' @@ -84,7 +85,7 @@ def sources(self, start=datetime.min, end=datetime.max): ) ), extractor=extract.XML( - tag='NumericPubDate', toplevel=True, + Tag('NumericPubDate'), transform=lambda x: '{y}-{m}-{d}'.format(y=x[:4],m=x[4:6],d=x[6:]) ), sortable=True, @@ -96,30 +97,28 @@ def sources(self, start=datetime.min, end=datetime.max): csv_core=True, results_overview=True, description='Publication date as full string, as found in source file', - extractor=extract.XML( - tag='AlphaPubDate', toplevel=True - ) + extractor=extract.XML(Tag('AlphaPubDate')) ), FieldDefinition( name='id', es_mapping=keyword_mapping(), display_name='ID', description='Article identifier.', - extractor=extract.XML(tag='RecordID', toplevel=True) + extractor=extract.XML(Tag('RecordID')), ), FieldDefinition( name='pub_id', es_mapping=keyword_mapping(), display_name='Publication ID', description='Publication identifier', - extractor=extract.XML(tag='PublicationID', toplevel=True, recursive=True) + extractor=extract.XML(Tag('PublicationID')) ), FieldDefinition( name='page', es_mapping=keyword_mapping(), display_name='Page', description='Start page label, from source (1, 2, 17A, ...).', - extractor=extract.XML(tag='StartPage', toplevel=True) + extractor=extract.XML(Tag('StartPage')) ), FieldDefinition( name='title', @@ -127,14 +126,14 @@ def sources(self, start=datetime.min, end=datetime.max): search_field_core=True, visualizations=['wordcloud'], description='Article title.', - extractor=extract.XML(tag='RecordTitle', toplevel=True) + extractor=extract.XML(Tag('RecordTitle')) ), FieldDefinition( name='source-paper', es_mapping=keyword_mapping(True), display_name='Source paper', description='Credited as source.', - extractor=extract.XML(tag='Title', toplevel=True, recursive=True), + extractor=extract.XML(Tag('Title')), search_filter=filters.MultipleChoiceFilter( description='Accept only articles from these source papers.', option_count=5 @@ -145,14 +144,14 @@ def sources(self, start=datetime.min, end=datetime.max): mapping=keyword_mapping(True), display_name='Place', description='Place in which the article was published', - extractor=extract.XML(tag='Qualifier', toplevel=True, recursive=True) + extractor=extract.XML(Tag('Qualifier')) ), FieldDefinition( name='author', mapping=keyword_mapping(True), display_name='Author', description='Article author', - extractor=extract.XML(tag='PersonName', toplevel=True, recursive=True) + extractor=extract.XML(Tag('PersonName')) ), FieldDefinition( name='category', @@ -164,7 +163,7 @@ def sources(self, start=datetime.min, end=datetime.max): description='Accept only articles in these categories.', option_count=19 ), - extractor=extract.XML(tag='ObjectType', toplevel=True), + extractor=extract.XML(Tag('ObjectType')), csv_core=True ), FieldDefinition( @@ -176,7 +175,7 @@ def sources(self, start=datetime.min, end=datetime.max): description='Raw OCR\'ed text (content).', results_overview=True, search_field_core=True, - extractor=extract.XML(tag='FullText', toplevel=True, flatten=True), + extractor=extract.XML(Tag('FullText'), flatten=True), language='en', ) ] diff --git a/backend/corpora/jewishinscriptions/jewishinscriptions.py b/backend/corpora/jewishinscriptions/jewishinscriptions.py index 29371c3fe..8d307a269 100644 --- a/backend/corpora/jewishinscriptions/jewishinscriptions.py +++ b/backend/corpora/jewishinscriptions/jewishinscriptions.py @@ -3,6 +3,7 @@ import os.path as op import logging from datetime import datetime +from ianalyzer_readers.xml_tag import Tag, CurrentTag from django.conf import settings @@ -15,7 +16,6 @@ class JewishInscriptions(XMLCorpusDefinition): """ Alto XML corpus of Jewish funerary inscriptions. """ - # Data overrides from .common.Corpus (fields at bottom of class) title = "Jewish Funerary Inscriptions" description = "A collection of inscriptions on Jewish burial sites" min_date = datetime(year=769, month=1, day=1) @@ -27,9 +27,9 @@ class JewishInscriptions(XMLCorpusDefinition): languages = ['heb', 'lat'] category = 'inscription' - # Data overrides from .common.XMLCorpus - tag_toplevel = '' - tag_entry = 'TEI' + + tag_toplevel = CurrentTag() + tag_entry = Tag('TEI') # New data members filename_pattern = re.compile('\d+') @@ -60,8 +60,10 @@ def sources(self, start=min_date, end=max_date): display_name='ID', description='ID of the inscription entry.', extractor=XML( - tag=['teiHeader', 'fileDesc', 'titleStmt', 'title'], - toplevel=False, + Tag('teiHeader'), + Tag('fileDesc'), + Tag('titleStmt'), + Tag('title'), ), es_mapping=keyword_mapping() ), @@ -76,8 +78,13 @@ def sources(self, start=min_date, end=max_date): upper=max_date.year, ), extractor=XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'history', 'origin', 'origDate'], - toplevel=False, + Tag('teiHeader'), + Tag('fileDesc'), + Tag('sourceDesc'), + Tag('msDesc'), + Tag('history'), + Tag('origin'), + Tag('origDate'), ), csv_core=True, sortable=True, @@ -90,8 +97,13 @@ def sources(self, start=min_date, end=max_date): display_name='Date comments', description='Additional comments on the year.', extractor=XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'history', 'origin', 'remarksOnDate'], - toplevel=False, + Tag('teiHeader'), + Tag('fileDesc'), + Tag('sourceDesc'), + Tag('msDesc'), + Tag('history'), + Tag('origin'), + Tag('remarksOnDate'), ), ), FieldDefinition( @@ -99,8 +111,9 @@ def sources(self, start=min_date, end=max_date): display_name='Transcription', description='Text content of the inscription.', extractor=XML( - tag=['text', 'body', 'transcription'], - toplevel=False, + Tag('text'), + Tag('body'), + Tag('transcription'), flatten=True ), search_field_core=True, @@ -118,30 +131,21 @@ def sources(self, start=min_date, end=max_date): description='Search only within these incipit types.', option_count=8 ), - extractor=XML( - tag=['text', 'body', 'incipit'], - toplevel=False, - ), + extractor=XML(Tag('text'), Tag('body'), Tag('incipit')), visualizations=['resultscount', 'termfrequency'] ), FieldDefinition( name='names', display_name='Names', description='Names of the buried persons.', - extractor=XML( - tag=['text', 'body', 'namesMentioned'], - toplevel=False, - ), + extractor=XML(Tag('text'), Tag('body'), Tag('namesMentioned')), search_field_core=True ), FieldDefinition( name='names_hebrew', display_name='Names (Hebrew)', description='Names in Hebrew of the buried persons.', - extractor=XML( - tag=['text', 'body', 'namesMentionedHebrew'], - toplevel=False, - ), + extractor=XML(Tag('text'), Tag('body'), Tag('namesMentionedHebrew')), ), FieldDefinition( name='sex', @@ -152,10 +156,7 @@ def sources(self, start=min_date, end=max_date): description='Search only within these genders.', option_count=3, ), - extractor=XML( - tag=['text', 'body', 'sex'], - toplevel=False, - ), + extractor=XML(Tag('text'), Tag('body'), Tag('sex')), csv_core=True ), FieldDefinition( @@ -168,10 +169,7 @@ def sources(self, start=min_date, end=max_date): lower=0, upper=100, ), - extractor=XML( - tag=['text', 'body', 'age'], - toplevel=False, - ), + extractor=XML(Tag('text'), Tag('body'), Tag('age')), csv_core=True, sortable=True ), @@ -179,10 +177,7 @@ def sources(self, start=min_date, end=max_date): name='age_remarks', display_name='Age remarks', description='Additional comments on the age.', - extractor=XML( - tag=['text', 'body', 'remarksOnAge'], - toplevel=False, - ), + extractor=XML(Tag('text'), Tag('body'), Tag('remarksOnAge')), ), FieldDefinition( name='provenance', @@ -194,8 +189,13 @@ def sources(self, start=min_date, end=max_date): option_count = 8 ), extractor=XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'history', 'origin', 'provenance'], - toplevel=False, + Tag('teiHeader'), + Tag('fileDesc'), + Tag('sourceDesc'), + Tag('msDesc'), + Tag('history'), + Tag('origin'), + Tag('provenance'), ), visualizations=['resultscount', 'termfrequency'] ), @@ -204,10 +204,7 @@ def sources(self, start=min_date, end=max_date): display_name='Inscription type', description='Type of inscription found.', es_mapping={'type': 'keyword'}, - extractor=XML( - tag=['text', 'body', 'inscriptionType'], - toplevel=False, - ), + extractor=XML(Tag('text'), Tag('body'), Tag('inscriptionType')), csv_core=True ), FieldDefinition( @@ -219,10 +216,7 @@ def sources(self, start=min_date, end=max_date): description='Search only within these iconography types.', option_count=8 ), - extractor=XML( - tag=['text', 'body', 'iconographyType'], - toplevel=False, - ), + extractor=XML(Tag('text'), Tag('body'), Tag('iconographyType')), csv_core=True, visualizations=['resultscount', 'termfrequency'] ), @@ -230,10 +224,7 @@ def sources(self, start=min_date, end=max_date): name='iconography_desc', display_name='Iconography description', description='Description of the iconography on the inscription.', - extractor=XML( - tag=['text', 'body', 'iconographyDescription'], - toplevel=False, - ), + extractor=XML(Tag('text'), Tag('body'), Tag('iconographyDescription')), ), FieldDefinition( name='material', @@ -244,10 +235,7 @@ def sources(self, start=min_date, end=max_date): description='Search only within these material types.', option_count=8 ), - extractor=XML( - tag=['text', 'body', 'material'], - toplevel=False, - ), + extractor=XML(Tag('text'), Tag('body'), Tag('material')), csv_core=True, visualizations=['resultscount', 'termfrequency'] ), @@ -260,10 +248,7 @@ def sources(self, start=min_date, end=max_date): description='Search only within these languages.', option_count = 3 ), - extractor=XML( - tag=['text', 'body', 'language'], - toplevel=False, - ), + extractor=XML(Tag('text'), Tag('body'), Tag('language')), csv_core=True, visualizations=['resultscount', 'termfrequency'] ), @@ -278,10 +263,7 @@ def sources(self, start=min_date, end=max_date): # lower=0, # upper=100, # ), - extractor=XML( - tag=['text', 'body', 'numberOfLinesSurviving'], - toplevel=False, - ), + extractor=XML(Tag('text'), Tag('body'), Tag('numberOfLinesSurviving')), csv_core=True ), FieldDefinition( @@ -290,8 +272,12 @@ def sources(self, start=min_date, end=max_date): description='Storage location of the published work.', es_mapping=keyword_mapping(), extractor=XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'msIdentifier', 'location'], - toplevel=False, + Tag('teiHeader'), + Tag('fileDesc'), + Tag('sourceDesc'), + Tag('msDesc'), + Tag('msIdentifier'), + Tag('location'), ), csv_core=True, results_overview=True @@ -301,8 +287,12 @@ def sources(self, start=min_date, end=max_date): display_name='Publication', description='Article or book where inscription is published.', extractor=XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'msIdentifier', 'publication'], - toplevel=False, + Tag('teiHeader'), + Tag('fileDesc'), + Tag('sourceDesc'), + Tag('msDesc'), + Tag('msIdentifier'), + Tag('publication'), ), es_mapping=keyword_mapping(True) ), @@ -310,40 +300,28 @@ def sources(self, start=min_date, end=max_date): name='facsimile', display_name='Facsimile', description='Photo or facsimile of publication.', - extractor=XML( - tag=['facsimile', 'photoFacsimile'], - toplevel=False, - ), + extractor=XML(Tag('facsimile'), Tag('photoFacsimile')), es_mapping=keyword_mapping() ), FieldDefinition( name='photos_leonard', display_name='Photos (Leonard)', description='Photos by Leonard.', - extractor=XML( - tag=['facsimile', 'photosLeonard'], - toplevel=False, - ), + extractor=XML(Tag('facsimile'), Tag('photosLeonard')), es_mapping=keyword_mapping() ), FieldDefinition( name='3D_image', display_name='3D image', description='3D image of inscription.', - extractor=XML( - tag=['facsimile', 'image3D'], - toplevel=False, - ), + extractor=XML(Tag('facsimile'), Tag('image3D')), es_mapping=keyword_mapping() ), FieldDefinition( name='commentary', display_name='Commentary', description='Extra comments, questions or remarks on this inscription.', - extractor=XML( - tag=['text', 'body', 'commentary'], - toplevel=False, - ), + extractor=XML(Tag('text'), Tag('body'), Tag('commentary')), search_field_core=True, ) ] diff --git a/backend/corpora/jewishmigration/test_jewishmigration.py b/backend/corpora/jewishmigration/test_jewishmigration.py index d1dc7acdd..ef6d56fb4 100644 --- a/backend/corpora/jewishmigration/test_jewishmigration.py +++ b/backend/corpora/jewishmigration/test_jewishmigration.py @@ -136,7 +136,7 @@ def jm_corpus_settings(settings): settings.CORPORA = { 'jewishmigration': os.path.join(here, 'jewishmigration.py') } - settings.JMIG_DATA_DIR = '/corpora' + settings.JMIG_DATA_DIR = None settings.JMIG_DATA = None settings.JMIG_DATA_URL = 'http://www.example.com' settings.JMIG_INDEX = 'test-jewishmigration' diff --git a/backend/corpora/parliament/finland.py b/backend/corpora/parliament/finland.py index ad3524572..8be053707 100644 --- a/backend/corpora/parliament/finland.py +++ b/backend/corpora/parliament/finland.py @@ -1,5 +1,6 @@ from datetime import datetime from glob import glob +from ianalyzer_readers.xml_tag import Tag, FindParentTag, PreviousSiblingTag, ParentTag from addcorpus.python_corpora.corpus import XMLCorpusDefinition from addcorpus.python_corpora.extract import XML, Combined, Constant, Metadata @@ -17,24 +18,6 @@ def format_role(values): clean_id = id.replace('#', '') return roles.get(clean_id, clean_id) -def speech_metadata(speech_node): - """Gets the `note` sibling to the speech.""" - return speech_node.find_previous_sibling('note') - -def find_topic(speech_node): - return speech_node.parent.find_previous_sibling('head') - -def find_debate_node(speech_node): - return speech_node.find_parent('TEI') - -def find_debate_title(speech_node): - debate_node = find_debate_node(speech_node) - return debate_node.teiHeader.find('title') - -def find_date(speech_node): - debate_node = find_debate_node(speech_node) - return debate_node.teiHeader.find('date') - class ParliamentFinland(Parliament, XMLCorpusDefinition): title = 'People and Parliament (Finland, 1907-)' @@ -67,27 +50,31 @@ def sources(self, start, end): document_context = document_context() - tag_toplevel = 'teiCorpus' - tag_entry = 'u' + tag_toplevel = Tag('teiCorpus') + tag_entry = Tag('u') country = field_defaults.country() country.extractor = Constant('Finland') date = field_defaults.date() date.extractor = XML( - transform_soup_func = find_date, - attribute = 'when' + FindParentTag('TEI'), + Tag('teiHeader', recursive=False), + Tag('date'), + attribute='when' ) debate_id = field_defaults.debate_id() debate_id.extractor = XML( - transform_soup_func = find_debate_node, - attribute = 'xml:id' + FindParentTag('TEI'), + attribute='xml:id' ) debate_title = field_defaults.debate_title() debate_title.extractor = XML( - transform_soup_func = find_debate_title, + FindParentTag('TEI'), + Tag('teiHeader', recursive=False), + Tag('title'), transform = clean_value, ) @@ -104,7 +91,7 @@ def sources(self, start, end): role = field_defaults.parliamentary_role() role.extractor = Combined( - XML(attribute = 'ana'), + XML(attribute='ana'), Metadata('roles'), transform = format_role, ) @@ -125,26 +112,25 @@ def sources(self, start, end): speech.extractor = XML(transform = clean_value) speech_id = field_defaults.speech_id() - speech_id.extractor = XML( - attribute = 'xml:id' - ) + speech_id.extractor = XML(attribute='xml:id') speech_type = field_defaults.speech_type() speech_type.extractor = XML( - transform_soup_func = speech_metadata, + PreviousSiblingTag('note'), attribute = 'speechType' ) speech_type.language = 'fi' topic = field_defaults.topic() topic.extractor = XML( - transform_soup_func = find_topic, + ParentTag(), + PreviousSiblingTag('head'), transform = clean_value, ) url = field_defaults.url() url.extractor = XML( - transform_soup_func = speech_metadata, + PreviousSiblingTag('note'), attribute = 'link' ) diff --git a/backend/corpora/parliament/ireland.py b/backend/corpora/parliament/ireland.py index e2b3279b2..3c06238a4 100644 --- a/backend/corpora/parliament/ireland.py +++ b/backend/corpora/parliament/ireland.py @@ -6,6 +6,7 @@ from bs4 import BeautifulSoup import json import csv +from ianalyzer_readers.xml_tag import Tag, PreviousSiblingTag from addcorpus.python_corpora.corpus import CorpusDefinition, CSVCorpusDefinition, XMLCorpusDefinition from addcorpus.python_corpora.extract import Constant, CSV, XML, Metadata, Combined, Backup @@ -246,8 +247,6 @@ def extract_number_from_id(id): if match: return int(match.group(0)) -def find_topic_heading(speech_node): - return speech_node.find_previous_sibling('heading') def get_debate_id(filename): name, _ = os.path.splitext(filename) @@ -319,8 +318,8 @@ class ParliamentIrelandNew(XMLCorpusDefinition): min_date = datetime(year=2014, month=1, day=1) max_date = datetime(year=2020, month=12, day=31) - tag_toplevel = 'debate' - tag_entry = 'speech' + tag_toplevel = Tag('debate') + tag_entry = Tag('speech') def sources(self, start, end): if in_date_range(self, start, end): @@ -359,9 +358,8 @@ def sources(self, start, end): date = field_defaults.date() date.extractor = XML( - tag = 'docDate', + Tag('docDate'), attribute = 'date', - recursive = True, toplevel = True, ) @@ -388,7 +386,7 @@ def sources(self, start, end): speech = field_defaults.speech() speech.extractor = XML( - 'p', + Tag('p'), multiple = True, transform = strip_and_join_paragraphs, ) @@ -402,7 +400,7 @@ def sources(self, start, end): topic = field_defaults.topic() topic.extractor = XML( - transform_soup_func = find_topic_heading, + PreviousSiblingTag('heading'), extract_soup_func = lambda node : node.text, ) diff --git a/backend/corpora/parliament/netherlands.py b/backend/corpora/parliament/netherlands.py index 09a21dc5e..06f088d51 100644 --- a/backend/corpora/parliament/netherlands.py +++ b/backend/corpora/parliament/netherlands.py @@ -4,10 +4,11 @@ from bs4 import BeautifulSoup from os.path import join from django.conf import settings +from ianalyzer_readers.xml_tag import Tag, FindParentTag, PreviousTag, TransformTag import bs4 from addcorpus.python_corpora.corpus import XMLCorpusDefinition -from addcorpus.python_corpora.extract import XML, Constant, Combined, Choice +from addcorpus.python_corpora.extract import XML, Constant, Combined, Choice, Order from corpora.parliament.utils.parlamint import extract_all_party_data, extract_people_data, extract_role_data, party_attribute_extractor, person_attribute_extractor from corpora.utils.formatting import format_page_numbers from corpora.parliament.parliament import Parliament @@ -29,8 +30,7 @@ def format_role(role): else: return role.title() if type(role) == str else role -def find_topic(speech): - return speech.find_parent('topic') + def format_house(house): if house == 'senate': @@ -53,21 +53,6 @@ def format_house_recent(url): else: return 'Tweede Kamer' -def find_last_pagebreak(node): - "find the last pagebreak node before the start of the current node" - is_tag = lambda x : type(x) == bs4.element.Tag - - #look for pagebreaks in previous nodes - for prev_node in node.previous_siblings: - if is_tag(prev_node): - breaks = prev_node.find_all('pagebreak') - if breaks: - return breaks[-1] - - #if none was found, go up a level - parent = node.parent - if parent: - return find_last_pagebreak(parent) def format_pages(pages): topic_start, topic_end, prev_break, last_break = pages @@ -90,10 +75,10 @@ def format_party(data): def get_party_full(speech_node): party_ref = speech_node.attrs.get(':party-ref') if not party_ref: - return None + return [] parents = list(speech_node.parents) party_node = parents[-1].find('organization', attrs={'pm:ref':party_ref}) - return party_node + return [party_node] def get_source(meta_node): if type(meta_node) == bs4.element.Tag: @@ -103,9 +88,6 @@ def get_source(meta_node): return '' -def get_sequence(node, tag_entry): - previous = node.find_all_previous(tag_entry) - return len(previous) + 1 # start from 1 def is_old(metadata): return metadata['dataset'] == 'old' @@ -134,8 +116,8 @@ class ParliamentNetherlands(Parliament, XMLCorpusDefinition): image = 'netherlands.jpg' description_page = 'netherlands.md' citation_page = 'netherlands.md' - tag_toplevel = lambda _, metadata: 'root' if is_old(metadata) else 'TEI' - tag_entry = lambda _, metadata: 'speech' if is_old(metadata) else 'u' + tag_toplevel = lambda metadata: Tag('root') if is_old(metadata) else Tag('TEI') + tag_entry = lambda metadata: Tag('speech') if is_old(metadata) else Tag('u') languages = ['nl'] category = 'parliament' @@ -183,12 +165,17 @@ def sources(self, start, end): date = field_defaults.date() date.extractor = Choice( XML( - tag=['meta','dc:date'], + Tag('meta'), + Tag('dc:date'), toplevel=True, applicable=is_old ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc','bibl', 'date'], + Tag('teiHeader'), + Tag('fileDesc'), + Tag('sourceDesc'), + Tag('bibl'), + Tag('date'), toplevel=True ) ) @@ -197,14 +184,20 @@ def sources(self, start, end): chamber = field_defaults.chamber() chamber.extractor = Choice( XML( - tag=['meta','dc:subject', 'pm:house'], + Tag('meta'), + Tag('dc:subject'), + Tag('pm:house'), attribute='pm:house', toplevel=True, transform=format_house, applicable=is_old ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc','bibl','idno'], + Tag('teiHeader'), + Tag('fileDesc'), + Tag('sourceDesc'), + Tag('bibl'), + Tag('idno'), toplevel=True, transform=format_house_recent ) @@ -214,12 +207,16 @@ def sources(self, start, end): debate_title = field_defaults.debate_title() debate_title.extractor = Choice( XML( - tag=['meta', 'dc:title'], + Tag('meta'), + Tag('dc:title'), toplevel=True, applicable=is_old ), XML( - tag=['teiHeader', 'fileDesc', 'titleStmt', 'title'], + Tag('teiHeader'), + Tag('fileDesc'), + Tag('titleStmt'), + Tag('title'), multiple=True, toplevel=True, transform=lambda titles: titles[-2] if len(titles) else titles @@ -230,12 +227,12 @@ def sources(self, start, end): debate_id = field_defaults.debate_id() debate_id.extractor = Choice( XML( - tag=['meta', 'dc:identifier'], + Tag('meta'), + Tag('dc:identifier'), toplevel=True, applicable=is_old ), XML( - tag=None, attribute='xml:id', toplevel=True, ) @@ -244,14 +241,13 @@ def sources(self, start, end): topic = field_defaults.topic() topic.extractor = Choice( XML( - transform_soup_func = find_topic, + FindParentTag('topic'), attribute='title', applicable=is_old, ), XML( - tag=['note'], + Tag('note'), toplevel=True, - recursive=True ) ) topic.language = 'nl' @@ -259,16 +255,17 @@ def sources(self, start, end): speech = field_defaults.speech(language='nl') speech.extractor = Choice( XML( - tag='p', + Tag('p'), multiple=True, flatten=True, - applicable=is_old + applicable=is_old, ), XML( - tag=['seg'], + Tag('seg'), multiple=True, flatten=True, - ) + ), + transform='\n'.join, ) speech_id = field_defaults.speech_id() @@ -278,7 +275,6 @@ def sources(self, start, end): applicable=is_old ), XML( - tag=None, attribute='xml:id' ) ) @@ -317,10 +313,9 @@ def sources(self, start, end): XML( attribute='role', transform=format_role, - applicable = is_old, + applicable=is_old, ), XML( - tag=None, attribute='ana', transform=lambda x: x[1:].title() ) @@ -351,8 +346,8 @@ def sources(self, start, end): party_full = field_defaults.party_full() party_full.extractor = Choice( XML( + TransformTag(get_party_full), attribute='pm:name', - transform_soup_func=get_party_full, applicable = is_old, ), party_attribute_extractor('full_name') @@ -362,16 +357,18 @@ def sources(self, start, end): page = field_defaults.page() page.extractor = Choice( Combined( - XML(transform_soup_func=find_topic, + XML(FindParentTag('topic'), attribute='source-start-page' ), - XML(transform_soup_func=find_topic, + XML(FindParentTag('topic'), attribute='source-end-page' ), - XML(transform_soup_func=find_last_pagebreak, + XML(PreviousTag('pagebreak'), attribute='originalpagenr', ), - XML(tag=['stage-direction', 'pagebreak'], + XML( + Tag('stage-direction'), + Tag('pagebreak'), attribute='originalpagenr', multiple=True, transform=lambda pages : pages[-1] if pages else pages @@ -383,25 +380,16 @@ def sources(self, start, end): url = field_defaults.url() url.extractor = XML( - tag=['meta', 'dc:source'], - transform_soup_func=get_source, + Tag('meta'), + Tag('dc:source'), + Tag('pm:link'), toplevel=True, attribute='pm:source', applicable = is_old, ) sequence = field_defaults.sequence() - sequence.extractor = Choice( - XML( - extract_soup_func = lambda node : get_sequence(node, 'speech'), - applicable = is_old - ), - XML( - tag=None, - attribute='xml:id', - transform = get_sequence_recent, - ) - ) + sequence.extractor = Order(transform=lambda value: value + 1) source_archive = field_defaults.source_archive() source_archive.extractor = Choice( diff --git a/backend/corpora/peaceportal/FIJI/fiji.py b/backend/corpora/peaceportal/FIJI/fiji.py index 8bc5ce6c4..5c30f63b3 100644 --- a/backend/corpora/peaceportal/FIJI/fiji.py +++ b/backend/corpora/peaceportal/FIJI/fiji.py @@ -2,6 +2,7 @@ import os import os.path as op import logging +from ianalyzer_readers.xml_tag import Tag from django.conf import settings @@ -46,8 +47,7 @@ def __init__(self): ) self._id.extractor = XML( - tag=['teiHeader', 'fileDesc', 'titleStmt', 'title'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('titleStmt'), Tag('title'), ) self.url.extractor = Constant( @@ -63,30 +63,27 @@ def __init__(self): # ) self.transcription.extractor = XML( - tag=['text', 'body', 'transcription'], - toplevel=False, + Tag('text'), Tag('body'), Tag('transcription'), flatten=True ) self.names.extractor = XML( - tag=['teiHeader', 'profileDesc', - 'particDesc', 'listPerson', 'person'], + Tag('teiHeader'), Tag('profileDesc'), Tag('particDesc'), Tag('listPerson'), + Tag('person'), flatten=True, multiple=True, - toplevel=False, + transform=lambda result: ' '.join(result).strip(), ) self.sex.extractor = XML( - tag=['teiHeader', 'profileDesc', - 'particDesc', 'listPerson', 'person'], + Tag('teiHeader'), Tag('profileDesc'), Tag('particDesc'), Tag('listPerson'), + Tag('person'), attribute='sex', multiple=True, - toplevel=False, ) self.age.extractor = XML( - tag=['text', 'body', 'age'], - toplevel=False, + Tag('text'), Tag('body'), Tag('age'), transform=lambda age: transform_age_integer(age) ) @@ -95,47 +92,41 @@ def __init__(self): ) self.settlement.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', - 'msDesc', 'history', 'origin', 'provenance'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('provenance'), ) self.material.extractor = XML( - tag=['text', 'body', 'material'], - toplevel=False, + Tag('text'), Tag('body'), Tag('material'), transform=lambda x: categorize_material(x) ) self.material_details = XML( - tag=['text', 'body', 'material'], - toplevel=False, + Tag('text'), Tag('body'), Tag('material'), ) self.language.extractor = XML( - tag=['teiHeader', 'profileDesc', 'langUsage', 'language'], - toplevel=False, + Tag('teiHeader'), Tag('profileDesc'), Tag('langUsage'), Tag('language'), multiple=True, transform=lambda x: normalize_language(x) ) self.comments.extractor = Combined( XML( - tag=['text', 'body', 'commentary'], - toplevel=False, + Tag('text'), Tag('body'), Tag('commentary'), + transform=str.strip, ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'history', 'origin', 'remarksOnDate'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('remarksOnDate'), transform=lambda x: 'DATE:\n{}\n'.format(x) if x else x ), XML( - tag=['text', 'body', 'ageComments'], - toplevel=False, + Tag('text'), Tag('body'), Tag('ageComments'), transform=lambda x: 'AGE:\n{}\n'.format(x) if x else x ), XML( - tag=['text', 'body', 'iconographyDescription'], - toplevel=False, + Tag('text'), Tag('body'), Tag('iconographyDescription'), transform=lambda x: 'ICONOGRAPHY:\n{}\n'.format(x) if x else x ), transform=lambda x: join_commentaries(x) @@ -143,19 +134,18 @@ def __init__(self): self.bibliography.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'msIdentifier', 'publications', 'publication'], - toplevel=False, - multiple=True + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('msIdentifier'), Tag('publications'), Tag('publication'), + multiple=True, ) self.location_details.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'msIdentifier', 'location'], - toplevel=False + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('msIdentifier'), Tag('location'), ) self.iconography.extractor = XML( - tag=['text', 'body', 'iconographyType'], - toplevel=False + Tag('text'), Tag('body'), Tag('iconographyType'), ) self.transcription_hebrew.extractor = Combined( diff --git a/backend/corpora/peaceportal/epidat.py b/backend/corpora/peaceportal/epidat.py index ef741b50c..394b45a88 100644 --- a/backend/corpora/peaceportal/epidat.py +++ b/backend/corpora/peaceportal/epidat.py @@ -1,11 +1,14 @@ import re from copy import copy +from ianalyzer_readers.xml_tag import Tag, TransformTag +from typing import Iterable, Optional +import bs4 from django.conf import settings from addcorpus.python_corpora.corpus import XMLCorpusDefinition from addcorpus.es_mappings import date_mapping -from addcorpus.python_corpora.extract import XML, Constant, Combined, FilterAttribute +from addcorpus.python_corpora.extract import XML, Constant, Combined, Pass from corpora.peaceportal.peaceportal import PeacePortal, categorize_material, \ clean_newline_characters, clean_commentary, join_commentaries, get_text_in_language, \ not_before_extractor @@ -28,28 +31,19 @@ def __init__(self): ) self._id.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', - 'msDesc', 'msIdentifier', 'idno'], - multiple=False, - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('msIdentifier'), Tag('idno'), flatten=True ) - self.url.extractor = FilterAttribute( - tag=['teiHeader', 'fileDesc', 'publicationStmt', 'idno'], - multiple=False, - toplevel=False, + self.url.extractor = XML( + Tag('teiHeader'), Tag('fileDesc'), Tag('publicationStmt'), Tag('idno', type='url'), flatten=True, - attribute_filter={ - 'attribute': 'type', - 'value': 'url' - } ) self.year.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'origDate', 'date'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('origDate'), Tag('date'), transform=lambda x: get_year(x), ) @@ -58,157 +52,147 @@ def __init__(self): # the dataset of the Steinheim institute is from the 19th/20th century and has accurate dates self.date.es_mapping = date_mapping() self.date.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'origDate', 'date'], - toplevel=False + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('origDate'), Tag('date'), ) - self.transcription.extractor = XML( - tag=['text', 'body', 'div'], - toplevel=False, - multiple=False, - flatten=True, + self.transcription.extractor = Pass( + XML( + Tag('text'), Tag('body'), Tag('div', type='edition'), Tag('ab'), + multiple=True, + flatten=True, + transform='\n'.join, + ), transform=lambda x: clean_newline_characters(x), - transform_soup_func=extract_transcript ) - self.transcription_german.extractor = XML( - tag=['text', 'body', ], - toplevel=False, - multiple=False, - flatten=True, + self.transcription_german.extractor = Pass( + XML( + Tag('text'), Tag('body'), Tag('div', type='translation'), Tag('ab'), + multiple=True, + flatten=True, + transform='\n'.join + ), transform=lambda x: clean_newline_characters(x), - transform_soup_func=extract_translation ) self.names.extractor = XML( - tag=['teiHeader', 'profileDesc', - 'particDesc', 'listPerson', 'person'], + Tag('teiHeader'), Tag('profileDesc'), Tag('particDesc'), + Tag('listPerson'), Tag('person'), flatten=True, multiple=True, - toplevel=False, + transform=' '.join, ) self.sex.extractor = XML( - tag=['teiHeader', 'profileDesc', - 'particDesc', 'listPerson', 'person'], + Tag('teiHeader'), Tag('profileDesc'), Tag('particDesc'), + Tag('listPerson'), Tag('person'), attribute='sex', multiple=True, - toplevel=False, transform=lambda x: convert_sex(x) ) self.dates_of_death.extractor = XML( - tag=['teiHeader', 'profileDesc', - 'particDesc', 'listPerson'], - transform_soup_func=extract_death, + Tag('teiHeader'), Tag('profileDesc'), Tag('particDesc'), + Tag('listPerson'), Tag('death'), attribute='when', - multiple=False, - toplevel=False, + multiple=True, ) self.country.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'origPlace', 'country'], - toplevel=False, - transform_soup_func=extract_country, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('origPlace'), Tag('country'), + TransformTag(_extract_country), transform=lambda x: clean_country(x), flatten=True, ) self.region.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'origPlace', 'country', 'region'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('origPlace'), Tag('country'), + Tag('region'), flatten=True ) self.settlement.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'origPlace', 'settlement'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('origPlace'), Tag('settlement'), + TransformTag(_extract_settlement), flatten=True, - transform_soup_func=extract_settlement, ) self.location_details.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'origPlace', 'settlement', 'geogName'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('origPlace'), Tag('settlement'), + Tag('geogName'), TransformTag(_extract_location_details), flatten=True, - transform_soup_func=extract_location_details, ) self.material.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'physDesc', - 'objectDesc', 'supportDesc', 'support', 'p', 'material'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('objectDesc'), Tag('supportDesc'), Tag('support'), + Tag('p'), Tag('material'), flatten=True, transform=lambda x: categorize_material(x) ) self.material_details.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'physDesc', - 'objectDesc', 'supportDesc', 'support', 'p', 'material'], - toplevel=False, - flatten=True + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('objectDesc'), Tag('supportDesc'), Tag('support'), + Tag('p'), Tag('material'), + flatten=True, ) self.language.extractor = XML( - tag=['teiHeader', 'profileDesc', 'langUsage', 'language'], - toplevel=False, + Tag('teiHeader'), Tag('profileDesc'), Tag('langUsage'), Tag('language'), multiple=True, transform=lambda x: get_language(x) ) self.comments.extractor = Combined( XML( - tag=['text', 'body'], - toplevel=False, - transform_soup_func=extract_commentary, + Tag('text'), Tag('body'), Tag('div', type='commentary'), + multiple=True, + extract_soup_func=_extract_commentary, + transform=lambda found: "\n".join(found) if len(found) > 1 else None ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'physDesc', - 'objectDesc', 'supportDesc', 'condition'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('objectDesc'), Tag('supportDesc'), Tag('condition'), flatten=True, - transform=lambda x: 'CONDITION:\n{}\n'.format(x) if x else x + transform=lambda x: f'CONDITION:\n{x}\n' if x else x ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'physDesc', - 'objectDesc', 'supportDesc', 'support', 'p'], - toplevel=False, - transform_soup_func=extract_support_comments, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('objectDesc'), Tag('supportDesc'), Tag('support'), + Tag('p'), + extract_soup_func=_extract_support_comments, ), transform=lambda x: join_commentaries(x) ) self.images.extractor = XML( - tag=['facsimile', 'graphic'], + Tag('facsimile'), Tag('graphic'), multiple=True, attribute='url', - toplevel=False ) self.coordinates.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'origPlace', 'settlement', 'geogName', 'geo'], - toplevel=False, - multiple=False, - flatten=True + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('origPlace'), Tag('settlement'), + Tag('geogName'), Tag('geo'), + flatten=True, ) self.iconography.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', - 'msDesc', 'physDesc', 'decoDesc', 'decoNote'], - toplevel=False, - multiple=False + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('decoDesc'), Tag('decoNote'), ) self.bibliography.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'msIdentifier', 'publications', 'publication'], - toplevel=False, - multiple=True + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('msIdentifier'), Tag('publications'), Tag('publication'), + multiple=True, + transform=lambda x: x if x else None ) self.transcription_hebrew.extractor = Combined( @@ -270,68 +254,27 @@ def get_language(values): return values -def extract_transcript(soup): - ''' - Helper function to ensure correct extraction of the transcripts. - Note that there are multiple formats in which these are stored, - but the text that we need is always in the `` children of - `['text', 'body', 'div']` (where div has `type=edition`, this is always the first one). - ''' - if not soup: - return - return soup.find_all('ab') - - -def extract_translation(soup): - ''' - Helper function to extract translation from the tag - ''' - if not soup: - return - translation = soup.find('div', {'type': 'translation'}) - if translation: - return translation.find_all('ab') - else: - return - - -def extract_commentary(soup): +def _extract_commentary(commentary: bs4.PageElement) -> Optional[str]: ''' Helper function to extract all commentaries from the tag. A single element will be returned with the commentaries found as text content. ''' - if not soup: - return - found = [] - commentaries = soup.find_all('div', {'type': 'commentary'}) - - for commentary in commentaries: - if commentary['subtype'] in ['Zitate', 'Zeilenkommentar', 'Prosopographie', 'Abkürzung', 'Endkommentar', 'Stilmittel']: - p = commentary.find('p') - if p: - text = p.get_text() - if text: - text = clean_commentary(text) - found.append('{}:\n{}\n'.format( - commentary['subtype'].strip().upper(), text)) - - if len(found) > 1: - cloned_soup = copy(soup) - cloned_soup.clear() - cloned_soup.string = "\n".join(found) - return cloned_soup - else: - return None - - -def extract_support_comments(soup): - if not soup: - return + if commentary['subtype'] in ['Zitate', 'Zeilenkommentar', 'Prosopographie', 'Abkürzung', 'Endkommentar', 'Stilmittel']: + p = commentary.find('p') + if p: + text = p.get_text() + if text: + text = clean_commentary(text) + return '{}:\n{}\n'.format( + commentary['subtype'].strip().upper(), text) + + +def _extract_support_comments(soup: bs4.PageElement) -> str: cloned_soup = copy(soup) cloned_soup.clear() - commentaries = add_support_comment(soup, '', 'dim', 'DIMENSIONS') - commentaries = add_support_comment( + commentaries = _add_support_comment(soup, '', 'dim', 'DIMENSIONS') + commentaries = _add_support_comment( soup, commentaries, 'objectType', 'OBJECTTYPE') # add any additional text from the

element, @@ -342,11 +285,10 @@ def extract_support_comments(soup): text = clean_commentary(text) commentaries = '{}{}:\n{}\n'.format(commentaries, 'SUPPORT', text) - cloned_soup.string = commentaries - return cloned_soup + return commentaries -def add_support_comment(soup, existing_commentaries, elem_name, commentary_name): +def _add_support_comment(soup: bs4.PageElement, existing_commentaries: str, elem_name, commentary_name) -> str: elem = soup.find(elem_name) if elem: text = elem.get_text() @@ -356,45 +298,35 @@ def add_support_comment(soup, existing_commentaries, elem_name, commentary_name) return existing_commentaries -def extract_death(soup): - ''' - Helper function to extract date of death from multiple person tags. - ''' - if not soup: - return - return soup.find_all('death') - -def extract_country(soup): +def _extract_country(soup) -> Iterable[bs4.PageElement]: ''' Helper function to extract country. This is needed because the output of `flatten` would otherwise include the text contents of the ``. ''' - return clone_soup_extract_child(soup, 'region') + return _clone_soup_extract_child(soup, 'region') -def extract_settlement(soup): - return clone_soup_extract_child(soup, 'geogName') +def _extract_settlement(soup) -> Iterable[bs4.PageElement]: + return _clone_soup_extract_child(soup, 'geogName') -def extract_location_details(soup): - return clone_soup_extract_child(soup, 'geo') +def _extract_location_details(soup) -> Iterable[bs4.PageElement]: + return _clone_soup_extract_child(soup, 'geo') -def clone_soup_extract_child(soup, to_extract): +def _clone_soup_extract_child(soup, to_extract) -> Iterable[bs4.PageElement]: ''' Helper function to clone the soup and extract a child element. This is useful when the output of `flatten` would otherwise include the text contents of the child. ''' - if not soup: - return cloned_soup = copy(soup) child = cloned_soup.find(to_extract) if child: child.extract() - return cloned_soup + return [cloned_soup] # TODO: add field diff --git a/backend/corpora/peaceportal/iis.py b/backend/corpora/peaceportal/iis.py index 8243fd7dc..18824db19 100644 --- a/backend/corpora/peaceportal/iis.py +++ b/backend/corpora/peaceportal/iis.py @@ -1,10 +1,11 @@ from copy import copy from os.path import join, split - +from ianalyzer_readers.xml_tag import Tag +from typing import Optional from django.conf import settings from addcorpus.python_corpora.corpus import XMLCorpusDefinition -from addcorpus.python_corpora.extract import Combined, Constant, ExternalFile, FilterAttribute, XML +from addcorpus.python_corpora.extract import Combined, Constant, ExternalFile, XML from addcorpus.serializers import LanguageField from corpora.peaceportal.peaceportal import PeacePortal, categorize_material, clean_newline_characters, \ clean_commentary, join_commentaries, get_text_in_language, \ @@ -29,19 +30,15 @@ def __init__(self): ) self._id.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', - 'msDesc', 'msIdentifier', 'idno'], - multiple=False, - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('msIdentifier'), Tag('idno'), flatten=True, transform=lambda x: ''.join(x.lower().split()) ) - self.url.extractor = FilterAttribute( - tag=['teiHeader', 'fileDesc', 'sourceDesc', - 'msDesc', 'msIdentifier', 'idno'], - multiple=False, - toplevel=False, + self.url.extractor = XML( + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('msIdentifier'), Tag('idno'), flatten=True, transform=lambda x: 'https://library.brown.edu/iip/viewinscr/{}'.format( ''.join(x.lower().split())) @@ -49,10 +46,9 @@ def __init__(self): # quick and dirty for now: extract value for 'notBefore' self.year.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'date'], - toplevel=False, - attribute='notBefore' + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('date'), + attribute='notBefore', ) self.not_before.extractor = not_before_extractor() @@ -66,19 +62,13 @@ def __init__(self): ) self.transcription.extractor = ExternalFile( - stream_handler=extract_transcript + stream_handler=_extract_transcript ) - self.transcription_english.extractor = FilterAttribute( - tag=['div'], + self.transcription_english.extractor = XML( + Tag('div', type='translation'), Tag('p', limit=1), toplevel=True, - multiple=False, flatten=True, - attribute_filter={ - 'attribute': 'type', - 'value': 'translation' - }, - transform_soup_func=extract_paragraph, transform=lambda x: ' '.join(x.split()) if x else None ) @@ -92,11 +82,11 @@ def __init__(self): # ) self.iconography.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', - 'msDesc', 'physDesc', 'decoDesc', 'decoNote'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('decoDesc'), Tag('decoNote'), multiple=True, - flatten=True + flatten=True, + transform='\n'.join, ) # is not present in IIS data @@ -109,147 +99,125 @@ def __init__(self): ) self.region.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'placeName', 'region'], - toplevel=False, - flatten=True + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('placeName'), Tag('region'), + flatten=True, ) self.settlement.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'placeName', 'settlement'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('placeName'), Tag('settlement'), flatten=True ) self.location_details.extractor = Combined( XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'placeName'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('placeName'), flatten=True ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'p'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('p'), flatten=True ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'provenance'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('provenance'), flatten=True ) ) self.material.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'physDesc', - 'objectDesc', 'supportDesc'], + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('objectDesc'), Tag('supportDesc'), attribute='ana', - toplevel=False, flatten=True, transform=lambda x: categorize_material(x) ) self.material_details.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'physDesc', - 'objectDesc', 'supportDesc'], + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('objectDesc'), Tag('supportDesc'), attribute='ana', - toplevel=False, flatten=True ) self.language.extractor = Combined( XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'msContents', - 'textLang'], + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('msContents'), Tag('textLang'), attribute='mainLang', - toplevel=False, transform=lambda x: normalize_language(x) ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'msContents', - 'textLang'], + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('msContents'), Tag('textLang'), attribute='otherLangs', - toplevel=False, transform=lambda x: normalize_language(x) ) ) self.language_code.extractor = Combined( XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'msContents', - 'textLang'], + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('msContents'), Tag('textLang'), attribute='mainLang', - toplevel=False ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'msContents', - 'textLang'], + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('msContents'), Tag('textLang'), attribute='otherLangs', - toplevel=False ) ) self.comments.extractor = Combined( XML( - tag=['text'], - toplevel=False, - multiple=False, + Tag('text'), Tag('div', type='commentary'), Tag('p', limit=1), flatten=True, - transform_soup_func=extract_comments, transform=lambda x: clean_commentary(x) if x else None ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'physDesc', - 'objectDesc', 'supportDesc', 'condition'], - toplevel=False, - transform_soup_func=extract_condition + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('objectDesc'), Tag('supportDesc'), Tag('condition'), + extract_soup_func=_extract_condition, ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'physDesc', - 'objectDesc', 'layoutDesc', 'layout', 'p'], + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('objectDesc'), Tag('layoutDesc'), Tag('layout'), Tag('p'), toplevel=False, - transform=lambda x: 'LAYOUT:\n{}\n\n'.format( - clean_commentary(x)) if x else None + transform=lambda x: f'LAYOUT:\n{clean_commentary(x)}\n\n' if x else None ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'physDesc', - 'objectDesc'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('objectDesc'), attribute='ana', - transform=lambda x: 'OBJECTTYPE:\n{}\n\n'.format( - x[1:]) if x else None + transform=lambda x: f'OBJECTTYPE:\n{x[1:]}\n\n' if x else None ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'physDesc', - 'objectDesc', 'supportDesc', 'support', 'dimensions'], - toplevel=False, - transform_soup_func=extract_dimensions, - transform=lambda x: 'DIMENSIONS:\n{}\n\n'.format( - x) if x else None + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('objectDesc'), Tag('supportDesc'), Tag('support'), + Tag('dimensions'), + extract_soup_func=_extract_dimensions, + transform=lambda x: f'DIMENSIONS:\n{x}\n\n' if x else None ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'physDesc', - 'objectDesc', 'supportDesc', 'support', 'p'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('objectDesc'), Tag('supportDesc'), Tag('support'), + Tag('p'), flatten=True, - transform=lambda x: 'SUPPORT:\n{}\n\n'.format( - clean_commentary(x)) if x else None + transform=lambda x: f'SUPPORT:\n{clean_commentary(x)}\n\n' if x else None ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', - 'msDesc', 'physDesc', 'handDesc', 'handNote'], - toplevel=False, - transform_soup_func=extract_handnotes + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('handDesc'), Tag('handNote'), + extract_soup_func=_extract_handnotes ), transform=lambda x: join_commentaries(x) ) self.bibliography.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'msIdentifier', 'publications', 'publication'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('msIdentifier'), Tag('publications'), Tag('publication'), multiple=True ) @@ -274,7 +242,7 @@ def __init__(self): self.fields = exclude_fields_without_extractor(self.fields) -def extract_transcript(filestream): +def _extract_transcript(filestream): text = filestream.read().strip() filestream.close() # remove the tabs and spaces inherited from xml @@ -284,31 +252,11 @@ def extract_transcript(filestream): return text -def extract_paragraph(soup): - ''' - Extract first

element from `soup`, ignore the rest. - Ideal for ignoring

headers in the HTML versions of the body. - ''' - if not soup: - return - return soup.find('p') - - -def extract_comments(soup): - ''' - Helper function to extract the commentary from either or (siblings under ) - ''' - if not soup: - return - commentary_div = soup.find('div', {'type': 'commentary'}) - return extract_paragraph(commentary_div) - - -def extract_attribute_and_child_p(soup, field_header): +def _extract_attribute_and_child_p(soup, field_header) -> Optional[str]: ''' Extract value for 'ana' attribute from soup, as well as the text from a

child. Will be returned - in a new soup, i.e. a single element with text content + in as a string in the following format `textcontent (attrivubtevalue)` ''' result = '' @@ -316,7 +264,7 @@ def extract_attribute_and_child_p(soup, field_header): ana = None if 'ana' in soup.attrs: ana = soup['ana'] - p = extract_paragraph(soup) + p = soup.find('p') if p: text = p.get_text() if text: @@ -327,21 +275,18 @@ def extract_attribute_and_child_p(soup, field_header): if result: cloned_soup = copy(soup) cloned_soup.clear() - cloned_soup.string = '{}:\n{}\n\n'.format(field_header, result) - return cloned_soup + return '{}:\n{}\n\n'.format(field_header, result) -def extract_condition(soup): - return extract_attribute_and_child_p(soup, 'CONDITION') +def _extract_condition(soup): + return _extract_attribute_and_child_p(soup, 'CONDITION') -def extract_handnotes(soup): - if not soup: - return - return extract_attribute_and_child_p(soup, 'HANDNOTES') +def _extract_handnotes(soup): + return _extract_attribute_and_child_p(soup, 'HANDNOTES') -def extract_dimensions(soup): +def _extract_dimensions(soup) -> str: result = '' height_elem = soup.find('height') if height_elem: @@ -361,10 +306,7 @@ def extract_dimensions(soup): if depth: result = "{} D: {}".format(result, depth) - cloned_soup = copy(soup) - cloned_soup.clear() - cloned_soup.string = result - return cloned_soup + return result def normalize_language(text): @@ -381,9 +323,8 @@ def normalize_language(text): def not_after_extractor(): ''' iis misses the enclosing tag ''' return XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'date'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('date'), attribute='notAfter', transform=lambda x: transform_to_date(x, 'upper') ) @@ -392,9 +333,8 @@ def not_after_extractor(): def not_before_extractor(): ''' iis misses the enclosing tag ''' return XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'date'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('date'), attribute='notBefore', transform=lambda x: transform_to_date(x, 'lower') ) diff --git a/backend/corpora/peaceportal/peaceportal.py b/backend/corpora/peaceportal/peaceportal.py index f278de6fd..72f9134bf 100644 --- a/backend/corpora/peaceportal/peaceportal.py +++ b/backend/corpora/peaceportal/peaceportal.py @@ -2,6 +2,7 @@ import datetime from langdetect import detect from langdetect.lang_detect_exception import LangDetectException +from ianalyzer_readers.xml_tag import Tag from django.conf import settings @@ -34,12 +35,8 @@ class PeacePortal(ParentCorpusDefinition): min_date = datetime.datetime(year=746, month=1, day=1) category = 'inscription' - # Data overrides from .common.XMLCorpus - tag_entry = 'TEI' + tag_entry = Tag('TEI') - # New data members - non_xml_msg = 'Skipping non-XML file {}' - non_match_msg = 'Skipping XML file with nonmatching name {}' # overwrite below in child class if you need to extract the (converted) transcription # from external files. See README. # el stands for modern Greek (1500-) @@ -158,15 +155,12 @@ def clean_commentary(commentary): return ' '.join(commentary.split()) -def join_commentaries(commentaries): +def join_commentaries(commentaries) -> str: ''' Helper function to join the result of a Combined extractor into one string, separating items by a newline ''' - results = [] - for comm in commentaries: - if comm: - results.append(comm) + results = filter(None, commentaries) return "\n".join(results) @@ -303,9 +297,8 @@ def transform_to_date_range(earliest, latest): def not_after_extractor(transform=True): ''' extractor for standard epidat format ''' return XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'origDate', 'date'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('origDate'), Tag('date'), attribute='notAfter', transform=lambda x: transform_to_date(x, 'upper') if transform else x ) @@ -314,9 +307,8 @@ def not_after_extractor(transform=True): def not_before_extractor(transform=True): ''' extractor for standard epidat format ''' return XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'origDate', 'date'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('origDate'), Tag('date'), attribute='notBefore', transform=lambda x: transform_to_date(x, 'lower') if transform else x ) diff --git a/backend/corpora/peaceportal/tests/test_peace.py b/backend/corpora/peaceportal/tests/test_peace.py index 968996702..26030962b 100644 --- a/backend/corpora/peaceportal/tests/test_peace.py +++ b/backend/corpora/peaceportal/tests/test_peace.py @@ -157,8 +157,12 @@ "bibliography": [ "Noy 1995, p. 69-70 (83)" ], - "comments": """DATE: + "comments": """Found on the 3rd of December 1904 in Cub.XL. The lower third of the plaque was left unused. There are poits between the syllables. Ferrua thought it might be pagan. +DATE: Uncertain + +AGE: +not mentioned """, "transcription_he": "", "transcription_la": "", @@ -265,6 +269,9 @@ def test_peace_imports(peace_test_settings, corpus_object): for key in target: tested_fields.add(key) assert key in doc + if doc[key] != target[key]: + compare = doc[key], target[key] + print(key) assert doc[key] == target[key] for key in doc: diff --git a/backend/corpora/peaceportal/tol.py b/backend/corpora/peaceportal/tol.py index 6393bc191..773c29808 100644 --- a/backend/corpora/peaceportal/tol.py +++ b/backend/corpora/peaceportal/tol.py @@ -1,10 +1,12 @@ import re from copy import copy - +from ianalyzer_readers.xml_tag import Tag, TransformTag +from typing import Optional +import bs4 from django.conf import settings from addcorpus.python_corpora.corpus import XMLCorpusDefinition -from addcorpus.python_corpora.extract import XML, Constant, Combined, FilterAttribute +from addcorpus.python_corpora.extract import XML, Constant, Combined from corpora.peaceportal.peaceportal import PeacePortal, categorize_material, \ clean_newline_characters, clean_commentary, join_commentaries, get_text_in_language, \ transform_to_date_range, not_before_extractor, not_after_extractor @@ -24,28 +26,19 @@ def __init__(self): ) self._id.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', - 'msDesc', 'msIdentifier', 'idno'], - multiple=False, - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), + Tag('msDesc'), Tag('msIdentifier'), Tag('idno'), flatten=True ) - self.url.extractor = FilterAttribute( - tag=['teiHeader', 'fileDesc', 'publicationStmt', 'idno'], - multiple=False, - toplevel=False, + self.url.extractor = XML( + Tag('teiHeader'), Tag('fileDesc'), Tag('publicationStmt'), Tag('idno', type='url'), flatten=True, - attribute_filter={ - 'attribute': 'type', - 'value': 'url' - } ) self.year.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'origDate', 'date'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('origDate'), Tag('date'), transform=lambda x: get_year(x), ) @@ -55,147 +48,136 @@ def __init__(self): self.date.extractor = Combined( not_before_extractor(), not_after_extractor(), - transform=transform_to_date_range + transform=lambda dates: transform_to_date_range(*dates), ) self.transcription.extractor = XML( - tag=['text', 'body', 'div'], - toplevel=False, - multiple=False, + Tag('text'), Tag('body'), Tag('div', type='edition'), + Tag('ab'), flatten=True, transform=lambda x: clean_newline_characters(x), - transform_soup_func=extract_transcript ) self.names.extractor = XML( - tag=['teiHeader', 'profileDesc', - 'particDesc', 'listPerson', 'person'], + Tag('teiHeader'), Tag('profileDesc'), + Tag('particDesc'), Tag('listPerson'), Tag('person'), flatten=True, multiple=True, - toplevel=False, + transform=lambda names: ' '.join if names else None, ) self.sex.extractor = XML( - tag=['teiHeader', 'profileDesc', - 'particDesc', 'listPerson', 'person'], + Tag('teiHeader'), Tag('profileDesc'), + Tag('particDesc'), Tag('listPerson'), Tag('person'), attribute='sex', multiple=True, - toplevel=False, transform=lambda x: convert_sex(x) ) self.dates_of_death.extractor = XML( - tag=['teiHeader', 'profileDesc', - 'particDesc', 'listPerson'], - transform_soup_func=extract_death, + Tag('teiHeader'), Tag('profileDesc'), + Tag('particDesc'), Tag('listPerson'), + Tag('death'), + multiple=True, attribute='when', - multiple=False, - toplevel=False, + transform=lambda x: x if x else None ) self.country.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'origPlace', 'country'], - toplevel=False, - transform_soup_func=extract_country, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('origPlace'), Tag('country'), + TransformTag(extract_country), transform=lambda x: clean_country(x), flatten=True, ) self.region.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'origPlace', 'country', 'region'], - toplevel=False, + Tag('teiHeader'), Tag('sourceDesc'), Tag('fileDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('origPlace'), Tag('country'), Tag('region'), flatten=True ) self.settlement.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'origPlace', 'settlement'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('origPlace'), Tag('settlement'), + TransformTag(extract_settlement), flatten=True, - transform_soup_func=extract_settlement, ) self.location_details.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'origPlace', 'settlement', 'geogName'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('origPlace'), Tag('settlement'), + Tag('geogName'), TransformTag(extract_location_details), flatten=True, - transform_soup_func=extract_location_details, ) self.material.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'physDesc', - 'objectDesc', 'supportDesc', 'support', 'p', 'material'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('objectDesc'), Tag('supportDesc'), Tag('support'), + Tag('p'), Tag('material'), flatten=True, transform=lambda x: categorize_material(x) ) self.material_details.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'physDesc', - 'objectDesc', 'supportDesc', 'support', 'p', 'material'], - toplevel=False, - flatten=True + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('objectDesc'), Tag('supportDesc'), Tag('support'), + Tag('p'), Tag('material'), + flatten=True, ) self.language.extractor = XML( - tag=['teiHeader', 'profileDesc', 'langUsage', 'language'], - toplevel=False, + Tag('teiHeader'), Tag('profileDesc'), Tag('langUsage'), Tag('language'), multiple=True, transform=lambda x: get_language(x) ) self.comments.extractor = Combined( XML( - tag=['text', 'body'], - toplevel=False, - transform_soup_func=extract_commentary, + Tag('text'), Tag('body'), Tag('div', type='commentary'), + multiple=True, + extract_soup_func=_extract_commentary, + transform=lambda x: '\n'.join(filter(None, x)) if x else None ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'physDesc', - 'objectDesc', 'supportDesc', 'condition'], - toplevel=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('objectDesc'), Tag('supportDesc'), Tag('condition'), flatten=True, - transform=lambda x: 'CONDITION:\n{}\n'.format(x) if x else x + transform=lambda x: f'CONDITION:\n{x}\n' if x else x ), XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', 'physDesc', - 'objectDesc', 'supportDesc', 'support', 'p'], - toplevel=False, - transform_soup_func=extract_support_comments, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('objectDesc'), Tag('supportDesc'), Tag('support'), + Tag('p'), + extract_soup_func=_extract_support_comments, ), transform=lambda x: join_commentaries(x) ) self.images.extractor = XML( - tag=['facsimile', 'graphic'], + Tag('facsimile'), Tag('graphic'), multiple=True, attribute='url', - toplevel=False + transform=lambda x: x if x else None ) self.coordinates.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'history', 'origin', 'origPlace', 'settlement', 'geogName', 'geo'], - toplevel=False, - multiple=False, + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('history'), Tag('origin'), Tag('origPlace'), Tag('settlement'), + Tag('geogName'), Tag('geo'), flatten=True ) self.iconography.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', - 'msDesc', 'physDesc', 'decoDesc', 'decoNote'], - toplevel=False, - multiple=False + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('physDesc'), Tag('decoDesc'), Tag('decoNote'), ) self.bibliography.extractor = XML( - tag=['teiHeader', 'fileDesc', 'sourceDesc', 'msDesc', - 'msIdentifier', 'publications', 'publication'], - toplevel=False, - multiple=True + Tag('teiHeader'), Tag('fileDesc'), Tag('sourceDesc'), Tag('msDesc'), + Tag('msIdentifier'), Tag('publications'), Tag('publication'), + multiple=True, + transform=lambda x: x if x else None ) self.transcription_hebrew.extractor = Combined( @@ -257,24 +239,10 @@ def get_language(values): return values -def extract_transcript(soup): - ''' - Helper function to ensure correct extraction of the transcripts. - Note that there are multiple formats in which these are stored, - but the text that we need is always in the `` children of - `['text', 'body', 'div']` (where div has `type=edition`, this is always the first one). - ''' - if not soup: - return - return soup.find_all('ab') - - def extract_translation(soup): ''' Helper function to extract translation from the tag ''' - if not soup: - return translation = soup.find('div', {'type': 'translation'}) if translation: return translation.find_all('ab') @@ -282,43 +250,25 @@ def extract_translation(soup): return -def extract_commentary(soup): +def _extract_commentary(commentary: bs4.PageElement) -> Optional[str]: ''' - Helper function to extract all commentaries from the tag. + Helper function to extract all commentaries from

1: - cloned_soup = copy(soup) - cloned_soup.clear() - cloned_soup.string = "\n".join(found) - return cloned_soup - else: - return None + if commentary['subtype'] in ['Zitate', 'Zeilenkommentar', 'Prosopographie', 'Abkürzung', 'Endkommentar', 'Stilmittel']: + p = commentary.find('p') + if p: + text = p.get_text() + if text: + subtype = commentary['subtype'] + text = clean_commentary(text) + return f'{subtype}:\n{text}\n' -def extract_support_comments(soup): - if not soup: - return - cloned_soup = copy(soup) - cloned_soup.clear() - commentaries = add_support_comment(soup, '', 'dim', 'DIMENSIONS') - commentaries = add_support_comment( +def _extract_support_comments(soup: bs4.PageElement) -> str: + commentaries = _add_support_comment(soup, '', 'dim', 'DIMENSIONS') + commentaries = _add_support_comment( soup, commentaries, 'objectType', 'OBJECTTYPE') # add any additional text from the

element, @@ -327,13 +277,12 @@ def extract_support_comments(soup): text = contents[len(contents) - 1].strip() if text: text = clean_commentary(text) - commentaries = '{}{}:\n{}\n'.format(commentaries, 'SUPPORT', text) + commentaries = f'{commentaries}SUPPORT:\n{text}\n' - cloned_soup.string = commentaries - return cloned_soup + return commentaries -def add_support_comment(soup, existing_commentaries, elem_name, commentary_name): +def _add_support_comment(soup: bs4.PageElement, existing_commentaries: str, elem_name, commentary_name) -> str: elem = soup.find(elem_name) if elem: text = elem.get_text() @@ -343,15 +292,6 @@ def add_support_comment(soup, existing_commentaries, elem_name, commentary_name) return existing_commentaries -def extract_death(soup): - ''' - Helper function to extract date of death from multiple person tags. - ''' - if not soup: - return - return soup.find_all('death') - - def extract_country(soup): ''' Helper function to extract country. @@ -375,13 +315,11 @@ def clone_soup_extract_child(soup, to_extract): This is useful when the output of `flatten` would otherwise include the text contents of the child. ''' - if not soup: - return cloned_soup = copy(soup) child = cloned_soup.find(to_extract) if child: - child.extract() - return cloned_soup + [child.extract()] + return [cloned_soup] # TODO: add field diff --git a/backend/corpora/periodicals/periodicals.py b/backend/corpora/periodicals/periodicals.py index 9b8b017de..24111c8a5 100644 --- a/backend/corpora/periodicals/periodicals.py +++ b/backend/corpora/periodicals/periodicals.py @@ -9,6 +9,7 @@ from datetime import datetime import re import openpyxl +from ianalyzer_readers.xml_tag import Tag, SiblingTag, ParentTag from django.conf import settings @@ -40,13 +41,9 @@ class Periodicals(XMLCorpusDefinition): def es_settings(self): return es_settings(self.languages[:1], stopword_analysis=True, stemming_analysis=True) - tag_toplevel = 'articles' - tag_entry = 'artInfo' - - # New data members - filename_pattern = re.compile('[a-zA-z]+_(\d+)_(\d+)') - non_xml_msg = 'Skipping non-XML file {}' - non_match_msg = 'Skipping XML file with nonmatching name {}' + tag_toplevel = Tag('articles') + tag_entry = Tag('artInfo') + external_file_tag_toplevel = Tag('issue') mimetype = 'image/jpeg' @@ -112,9 +109,7 @@ def sources(self, start=min_date, end=max_date): display_name='ID', description='Unique identifier of the entry.', es_mapping=keyword_mapping(), - extractor=extract.XML(tag=None, - toplevel=False, - attribute='id'), + extractor=extract.XML(attribute='id'), ), FieldDefinition( name='issue', @@ -147,7 +142,7 @@ def sources(self, start=min_date, end=max_date): description='Text content.', es_mapping=main_content_mapping(True, True, True, 'en'), results_overview=True, - extractor=extract.XML(tag='ocrText', flatten=True), + extractor=extract.XML(Tag('ocrText'), flatten=True), search_field_core=True, visualizations=["wordcloud"], language='en', @@ -163,15 +158,9 @@ def sources(self, start=min_date, end=max_date): 'indicator is in this range.' ) ), - extractor=extract.XML(tag='ocr', - external_file={ - 'xml_tag_toplevel': 'issue', - 'xml_tag_entry': 'article' - }, - secondary_tag = { - 'tag': 'id', - 'match': 'id' - } + extractor=extract.XML( + lambda metadata: Tag('id', string=metadata['id']), + SiblingTag('ocr'), ), sortable=True ), @@ -179,15 +168,10 @@ def sources(self, start=min_date, end=max_date): name='title', display_name='Article title', description='Title of the article.', - extractor=extract.XML(tag='ti', - external_file={ - 'xml_tag_toplevel': 'issue', - 'xml_tag_entry': 'article' - }, - secondary_tag = { - 'tag': 'id', - 'match': 'id' - } + extractor=extract.XML( + lambda metadata: Tag('id', string=metadata['id']), + SiblingTag('ti'), + external_file=True, ), visualizations=['wordcloud'] ), @@ -196,15 +180,10 @@ def sources(self, start=min_date, end=max_date): es_mapping={'type': 'keyword'}, display_name='Starting column', description='Which column the article starts in.', - extractor=extract.XML(tag='sc', - external_file={ - 'xml_tag_toplevel': 'issue', - 'xml_tag_entry': 'article' - }, - secondary_tag = { - 'tag': 'id', - 'match': 'id' - } + extractor=extract.XML( + lambda metadata: Tag('id', string=metadata['id']), + SiblingTag('sc'), + external_file=True, ) ), FieldDefinition( @@ -212,15 +191,10 @@ def sources(self, start=min_date, end=max_date): display_name='Page count', description='How many pages the article covers.', es_mapping={'type': 'integer'}, - extractor=extract.XML(tag='pc', - external_file={ - 'xml_tag_toplevel': 'issue', - 'xml_tag_entry': 'article' - }, - secondary_tag = { - 'tag': 'id', - 'match': 'id' - } + extractor=extract.XML( + lambda metadata: Tag('id', string=metadata['id']), + SiblingTag('pc'), + external_file=True, ) ), FieldDefinition( @@ -228,15 +202,10 @@ def sources(self, start=min_date, end=max_date): display_name='Word count', description='Number of words in the article.', es_mapping={'type': 'integer'}, - extractor=extract.XML(tag='wordCount', - external_file={ - 'xml_tag_toplevel': 'issue', - 'xml_tag_entry': 'article' - }, - secondary_tag = { - 'tag': 'id', - 'match': 'id' - } + extractor=extract.XML( + lambda metadata: Tag('id', string=metadata['id']), + SiblingTag('wordCount'), + external_file=True, ) ), FieldDefinition( @@ -245,15 +214,10 @@ def sources(self, start=min_date, end=max_date): display_name='Category', description='Article category.', es_mapping={'type': 'keyword'}, - extractor=extract.XML(tag='ct', - external_file={ - 'xml_tag_toplevel': 'issue', - 'xml_tag_entry': 'article' - }, - secondary_tag = { - 'tag': 'id', - 'match': 'id' - } + extractor=extract.XML( + lambda metadata: Tag('id', string=metadata['id']), + SiblingTag('ct'), + external_file=True, ), search_filter=filters.MultipleChoiceFilter( description='Accept only articles in these categories.', @@ -266,16 +230,11 @@ def sources(self, start=min_date, end=max_date): display_name='Page number', description='At which page the article starts.', es_mapping={'type': 'integer'}, - extractor=extract.XML(tag='pa', - parent_level=1, - external_file={ - 'xml_tag_toplevel': 'issue', - 'xml_tag_entry': 'article' - }, - secondary_tag = { - 'tag': 'id', - 'match': 'id' - }, + extractor=extract.XML( + lambda metadata: Tag('id', string=metadata['id']), + ParentTag(2), + Tag('pa'), + external_file=True, transform=lambda x: re.sub('[\[\]]', '', x) ) ), diff --git a/backend/corpora/rechtspraak/rechtspraak.py b/backend/corpora/rechtspraak/rechtspraak.py index 3f07a3488..fc46c2d39 100644 --- a/backend/corpora/rechtspraak/rechtspraak.py +++ b/backend/corpora/rechtspraak/rechtspraak.py @@ -5,6 +5,7 @@ from os import makedirs, remove from typing import Optional from zipfile import ZipFile, BadZipFile +from ianalyzer_readers.xml_tag import Tag, ParentTag from django.conf import settings @@ -17,20 +18,25 @@ logger = logging.getLogger('indexing') -def rdf_description_extractor(tag, section='xml', **kwargs): - '''rdf:Description extractor +def _rdf_description_extractor(tag: Tag, section='xml', **kwargs) -> extract.XML: + ''' + Extracts a child of the rdf:Description tag + There are two rdf:Description tags available in the data: - description about the open data enrichment - description about the source There is only deterministic way to select the right one: - - check the dcterms:format sibling tag''' + - check the dcterms:format sibling tag + ''' return extract.XML( - tag=tag, - secondary_tag={'tag': 'dcterms:format', 'exact': f'text/{section}'}, + Tag('dcterms:format', string=f'text/{section}'), + ParentTag(1), + tag, **kwargs ) + class Rechtspraak(XMLCorpusDefinition): title = "Judicial system Netherlands" description = "Open data of (anonymised) court rulings of the Dutch judicial system" @@ -49,7 +55,7 @@ class Rechtspraak(XMLCorpusDefinition): def es_settings(self): return es_settings(self.languages[:1], stopword_analysis=True, stemming_analysis=True) - tag_toplevel = 'open-rechtspraak' + tag_toplevel = Tag('open-rechtspraak') def unpack(self, min_year: Optional[int] = None, @@ -144,7 +150,7 @@ def sources(self, min_date: Optional[int] = None, max_date: Optional[int] = None display_name='ID', description='', es_mapping=keyword_mapping(), - extractor=rdf_description_extractor('dcterms:identifier'), + extractor=_rdf_description_extractor(Tag('dcterms:identifier')), csv_core=True, ), FieldDefinition( @@ -153,8 +159,8 @@ def sources(self, min_date: Optional[int] = None, max_date: Optional[int] = None description='Document has available text content.', es_mapping={'type': 'boolean'}, extractor=extract.Backup( - extract.XML('uitspraak', flatten=True), - extract.XML('conclusie', flatten=True), + extract.XML(Tag('uitspraak'), flatten=True), + extract.XML(Tag('conclusie'), flatten=True), extract.Constant(False), transform=bool ), @@ -176,7 +182,7 @@ def sources(self, min_date: Optional[int] = None, max_date: Optional[int] = None FieldDefinition( name='date', display_name='Date', - extractor=rdf_description_extractor('dcterms:date'), + extractor=_rdf_description_extractor(Tag('dcterms:date')), es_mapping={'type': 'date', 'format': 'yyyy-MM-dd'}, results_overview=True, csv_core=True, @@ -192,7 +198,7 @@ def sources(self, min_date: Optional[int] = None, max_date: Optional[int] = None FieldDefinition( name='issued', display_name='Publication Date', - extractor=rdf_description_extractor('dcterms:issued'), + extractor=_rdf_description_extractor(Tag('dcterms:issued')), es_mapping={'type': 'date', 'format': 'yyyy-MM-dd'}, search_filter=filters.DateFilter( min_date, @@ -205,14 +211,14 @@ def sources(self, min_date: Optional[int] = None, max_date: Optional[int] = None FieldDefinition( name='publisher', display_name='Publisher', - extractor=rdf_description_extractor('dcterms:publisher'), + extractor=_rdf_description_extractor(Tag('dcterms:publisher')), es_mapping={'type': 'keyword'}, language='nl', ), FieldDefinition( name='creator', display_name='Court', - extractor=rdf_description_extractor('dcterms:creator'), + extractor=_rdf_description_extractor(Tag('dcterms:creator')), es_mapping={'type': 'keyword'}, csv_core=True, results_overview=True, @@ -227,12 +233,12 @@ def sources(self, min_date: Optional[int] = None, max_date: Optional[int] = None name='zaaknr', display_name='Case Number', es_mapping=keyword_mapping(), - extractor=rdf_description_extractor('psi:zaaknummer') + extractor=_rdf_description_extractor(Tag('psi:zaaknummer')), ), FieldDefinition( name='type', display_name='Type', - extractor=rdf_description_extractor('dcterms:type'), + extractor=_rdf_description_extractor(Tag('dcterms:type')), es_mapping={'type': 'keyword'}, csv_core=True, results_overview=True, @@ -246,7 +252,7 @@ def sources(self, min_date: Optional[int] = None, max_date: Optional[int] = None FieldDefinition( name='procedure', display_name='(type of) Procedure', - extractor=rdf_description_extractor('psi:procedure'), + extractor=_rdf_description_extractor(Tag('psi:procedure')), csv_core=True, es_mapping={'type': 'keyword'}, search_filter=filters.MultipleChoiceFilter( @@ -260,13 +266,13 @@ def sources(self, min_date: Optional[int] = None, max_date: Optional[int] = None name='spatial', display_name='Location', es_mapping=keyword_mapping(), - extractor=rdf_description_extractor('dcterms:spatial'), + extractor=_rdf_description_extractor(Tag('dcterms:spatial')), language='nl', ), FieldDefinition( name='subject', display_name='Area of law', - extractor=rdf_description_extractor('dcterms:subject'), + extractor=_rdf_description_extractor(Tag('dcterms:subject')), csv_core=True, es_mapping={'type': 'keyword'}, search_filter=filters.MultipleChoiceFilter( @@ -279,8 +285,8 @@ def sources(self, min_date: Optional[int] = None, max_date: Optional[int] = None FieldDefinition( name='title', display_name='Title', - extractor=rdf_description_extractor( - 'dcterms:title', section='html'), + extractor=_rdf_description_extractor( + Tag('dcterms:title'), section='html'), results_overview=True, search_field_core=True, language='nl', @@ -288,7 +294,7 @@ def sources(self, min_date: Optional[int] = None, max_date: Optional[int] = None FieldDefinition( name='abstract', display_name='Abstract', - extractor=extract.XML(tag='inhoudsindicatie', flatten=True), + extractor=extract.XML(Tag('inhoudsindicatie'), flatten=True), results_overview=True, language='nl', ), @@ -298,8 +304,8 @@ def sources(self, min_date: Optional[int] = None, max_date: Optional[int] = None display_type='text_content', es_mapping=main_content_mapping(True, True, True, 'nl'), extractor=extract.Backup( - extract.XML('uitspraak', flatten=True), - extract.XML('conclusie', flatten=True), + extract.XML(Tag('uitspraak'), flatten=True), + extract.XML(Tag('conclusie'), flatten=True), extract.Constant('Content not available') ), csv_core=True, @@ -312,7 +318,7 @@ def sources(self, min_date: Optional[int] = None, max_date: Optional[int] = None display_type='url', description='URL of the case on rechtspraak.nl', es_mapping=keyword_mapping(), - extractor=rdf_description_extractor( - 'dcterms:identifier', section='html') + extractor=_rdf_description_extractor( + Tag('dcterms:identifier'), section='html') ) ] diff --git a/backend/corpora/times/images/times.jpg b/backend/corpora/times/images/times.jpg index 6acbd5524..4bc3d3608 100644 Binary files a/backend/corpora/times/images/times.jpg and b/backend/corpora/times/images/times.jpg differ diff --git a/backend/corpora/times/times.py b/backend/corpora/times/times.py index c05ee90a4..5a0fbb954 100644 --- a/backend/corpora/times/times.py +++ b/backend/corpora/times/times.py @@ -22,6 +22,8 @@ from django.conf import settings from media.media_url import media_url +from ianalyzer_readers.xml_tag import Tag, ParentTag + logger = logging.getLogger(__name__) @@ -43,8 +45,8 @@ class Times(XMLCorpusDefinition): def es_settings(self): return es_settings(self.languages[:1], stopword_analysis=True, stemming_analysis=True) - tag_toplevel = 'issue' - tag_entry = 'article' + tag_toplevel = Tag('issue') + tag_entry = Tag('article') def sources(self, start=datetime.min, end=datetime.max): ''' @@ -117,7 +119,9 @@ def sources(self, start=datetime.min, end=datetime.max): description='Library where the microfilm is sourced', es_mapping=keyword_mapping(), extractor=extract.XML( - tag=['metadatainfo', 'sourceLibrary'], toplevel=True, + Tag('metadatainfo'), + Tag('sourceLibrary'), + toplevel=True, applicable=after(1985) ) ), @@ -127,11 +131,13 @@ def sources(self, start=datetime.min, end=datetime.max): es_mapping=keyword_mapping(), extractor=extract.Choice( extract.XML( - tag='ed', toplevel=True, + Tag('ed'), + toplevel=True, applicable=until(1985) ), extract.XML( - tag='ed', toplevel=True, multiple=True, + Tag('ed'), + toplevel=True, multiple=True, applicable=after(1985) ) ), @@ -143,7 +149,8 @@ def sources(self, start=datetime.min, end=datetime.max): es_mapping={'type': 'integer'}, description='Source issue number.', extractor=extract.XML( - tag='is', toplevel=True, + Tag('is'), + toplevel=True, # Hardcoded to ignore one particular issue with source data transform=lambda x: (62226 if x == "6222662226" else int(x)) ), @@ -156,7 +163,8 @@ def sources(self, start=datetime.min, end=datetime.max): description='Volume number.', es_mapping=keyword_mapping(), extractor=extract.XML( - tag='volNum', toplevel=True, + Tag('volNum'), + toplevel=True, applicable=after(1985) ), csv_core=True @@ -170,7 +178,8 @@ def sources(self, start=datetime.min, end=datetime.max): sortable=True, description='Publication date as full string, as found in source file', extractor=extract.XML( - tag='da', toplevel=True + Tag('da'), + toplevel=True ) ), FieldDefinition( @@ -184,7 +193,7 @@ def sources(self, start=datetime.min, end=datetime.max): 'indicator is in this range.' ) ), - extractor=extract.XML(tag='ocr', transform=float), + extractor=extract.XML(Tag('ocr'), transform=float), sortable=True ), FieldDefinition( @@ -196,7 +205,7 @@ def sources(self, start=datetime.min, end=datetime.max): 'For issues that span more than 1 day.' ), extractor=extract.XML( - tag='tdate', toplevel=True, + Tag('tdate'), toplevel=True, applicable=after(1985) ) ), @@ -206,7 +215,7 @@ def sources(self, start=datetime.min, end=datetime.max): description='Page count: number of images present in the issue.', es_mapping={'type': 'integer'}, extractor=extract.XML( - tag='ip', toplevel=True, transform=int + Tag('ip'), toplevel=True, transform=int ), sortable=True ), @@ -223,7 +232,9 @@ def sources(self, start=datetime.min, end=datetime.max): option_count=2 ), extractor=extract.XML( - tag=['..', 'pageid'], attribute='isPartOf', + ParentTag(), + Tag('pageid'), + attribute='isPartOf', applicable=after(1985) ) ), @@ -232,7 +243,10 @@ def sources(self, start=datetime.min, end=datetime.max): display_name='Supplement title', description='Supplement title.', extractor=extract.XML( - tag=['..', 'pageid', 'supptitle'], multiple=True, + ParentTag(), + Tag('pageid'), + Tag('supptitle'), + multiple=True, applicable=after(1985) ), ), @@ -241,7 +255,10 @@ def sources(self, start=datetime.min, end=datetime.max): display_name='Supplement subtitle', description='Supplement subtitle.', extractor=extract.XML( - tag=['..', 'pageid', 'suppsubtitle'], multiple=True, + ParentTag(), + Tag('pageid'), + Tag('suppsubtitle'), + multiple=True, applicable=after(1985) ) ), @@ -270,7 +287,7 @@ def sources(self, start=datetime.min, end=datetime.max): display_name='ID', description='Article identifier.', es_mapping=keyword_mapping(), - extractor=extract.XML(tag='id') + extractor=extract.XML(Tag('id')) ), FieldDefinition( name='ocr-relevant', @@ -278,7 +295,7 @@ def sources(self, start=datetime.min, end=datetime.max): description='Whether OCR confidence level is relevant.', es_mapping={'type': 'boolean'}, extractor=extract.XML( - tag='ocr', attribute='relevant', + Tag('ocr'), attribute='relevant', transform=string_contains("yes"), ) ), @@ -290,7 +307,7 @@ def sources(self, start=datetime.min, end=datetime.max): 'where article starts.' ), es_mapping=keyword_mapping(), - extractor=extract.XML(tag='sc') + extractor=extract.XML(Tag('sc')) ), FieldDefinition( name='page', @@ -298,8 +315,8 @@ def sources(self, start=datetime.min, end=datetime.max): description='Start page label, from source (1, 2, 17A, ...).', es_mapping=keyword_mapping(), extractor=extract.Choice( - extract.XML(tag='pa', applicable=until(1985)), - extract.XML(tag=['..', 'pa'], applicable=after(1985)) + extract.XML(Tag('pa'), applicable=until(1985)), + extract.XML(ParentTag(), Tag('pa'), applicable=after(1985)) ) ), FieldDefinition( @@ -311,7 +328,7 @@ def sources(self, start=datetime.min, end=datetime.max): 'of the article.' ), extractor=extract.XML( - tag='pc', transform=int + Tag('pc'), transform=int ), sortable=True ), @@ -322,13 +339,13 @@ def sources(self, start=datetime.min, end=datetime.max): search_field_core=True, visualizations=['wordcloud'], description='Article title.', - extractor=extract.XML(tag='ti') + extractor=extract.XML(Tag('ti')) ), FieldDefinition( name='subtitle', display_name='Subtitle', description='Article subtitle.', - extractor=extract.XML(tag='ta', multiple=True), + extractor=extract.XML(Tag('ta'), multiple=True), search_field_core=True ), FieldDefinition( @@ -336,7 +353,7 @@ def sources(self, start=datetime.min, end=datetime.max): display_name='Subheader', description='Article subheader (product dependent field).', extractor=extract.XML( - tag='subheader', multiple=True, + Tag('subheader'), multiple=True, applicable=after(1985) ) ), @@ -347,11 +364,11 @@ def sources(self, start=datetime.min, end=datetime.max): es_mapping=keyword_mapping(True), extractor=extract.Choice( extract.XML( - tag='au', multiple=True, + Tag('au'), multiple=True, applicable=until(1985) ), extract.XML( - tag='au_composed', multiple=True, + Tag('au_composed'), multiple=True, applicable=after(1985) ) ), @@ -364,7 +381,7 @@ def sources(self, start=datetime.min, end=datetime.max): description='Credited as source.', es_mapping=keyword_mapping(True), extractor=extract.XML( - tag='altSource', multiple=True + Tag('altSource'), multiple=True ) ), FieldDefinition( @@ -377,7 +394,7 @@ def sources(self, start=datetime.min, end=datetime.max): description='Accept only articles in these categories.', option_count=25 ), - extractor=extract.XML(tag='ct', multiple=True), + extractor=extract.XML(Tag('ct'), multiple=True), csv_core=True ), FieldDefinition( @@ -396,11 +413,11 @@ def sources(self, start=datetime.min, end=datetime.max): ), extractor=extract.Choice( extract.XML( - tag='il', multiple=True, + Tag('il'), multiple=True, applicable=until(1985) ), extract.XML( - tag='il', attribute='type', multiple=True, + Tag('il'), attribute='type', multiple=True, applicable=after(1985) ) ), @@ -411,7 +428,8 @@ def sources(self, start=datetime.min, end=datetime.max): display_name='Content preamble', description='Raw OCR\'ed text (preamble).', extractor=extract.XML( - tag=['text', 'text.preamble'], + Tag('text'), + Tag('text.preamble'), flatten=True ) ), @@ -420,7 +438,8 @@ def sources(self, start=datetime.min, end=datetime.max): display_name='Content heading', description='Raw OCR\'ed text (header).', extractor=extract.XML( - tag=['text', 'text.title'], + Tag('text'), + Tag('text.title'), flatten=True ) ), @@ -434,8 +453,11 @@ def sources(self, start=datetime.min, end=datetime.max): results_overview=True, search_field_core=True, extractor=extract.XML( - tag=['text', 'text.cr'], multiple=True, - flatten=True + Tag('text'), + Tag('text.cr'), + multiple=True, + flatten=True, + transform='\n'.join, ), language='en', ), diff --git a/backend/corpora/troonredes/troonredes.py b/backend/corpora/troonredes/troonredes.py index 0fa5cdc01..02ad45ab0 100644 --- a/backend/corpora/troonredes/troonredes.py +++ b/backend/corpora/troonredes/troonredes.py @@ -8,6 +8,7 @@ import os from os.path import join, splitext from datetime import datetime +from ianalyzer_readers.xml_tag import Tag from django.conf import settings @@ -45,11 +46,10 @@ class Troonredes(XMLCorpusDefinition): def es_settings(self): return es_settings(self.languages[:1], stopword_analysis=True, stemming_analysis=True) - tag_toplevel = 'doc' - tag_entry = 'entry' + tag_toplevel = Tag('doc') + tag_entry = Tag('entry') non_xml_msg = 'Skipping non-XML file {}' - non_match_msg = 'Skipping XML file with nonmatching name {}' def sources(self, start=min_date, end=max_date): logger = logging.getLogger(__name__) @@ -68,7 +68,7 @@ def sources(self, start=min_date, end=max_date): name='date', display_name='Date', description='Date of the speech', - extractor=extract.XML(tag='date'), + extractor=extract.XML(Tag('date')), es_mapping={'type': 'date', 'format': 'yyyy-MM-dd'}, results_overview=True, csv_core=True, @@ -90,7 +90,7 @@ def sources(self, start=min_date, end=max_date): name='title', display_name='Title', description='title.', - extractor=extract.XML(tag='title'), + extractor=extract.XML(Tag('title')), results_overview=True, search_field_core=True, language='nl', @@ -99,7 +99,7 @@ def sources(self, start=min_date, end=max_date): name='monarch', display_name='Monarch', description='Monarch that gave the speech.', - extractor=extract.XML(tag='monarch'), + extractor=extract.XML(Tag('monarch')), es_mapping={'type': 'keyword'}, results_overview=True, csv_core=True, @@ -116,7 +116,7 @@ def sources(self, start=min_date, end=max_date): name='speech_type', display_name='Speech type', description='Type of speech.', - extractor=extract.XML(tag='speech_type'), + extractor=extract.XML(Tag('speech_type')), es_mapping={'type': 'keyword'}, results_overview=True, csv_core=True, @@ -138,7 +138,7 @@ def sources(self, start=min_date, end=max_date): results_overview=True, search_field_core=True, visualizations=['wordcloud', 'ngram'], - extractor=extract.XML(tag='content'), + extractor=extract.XML(Tag('content')), language='nl', ), ] diff --git a/backend/corpora/ublad/ublad.py b/backend/corpora/ublad/ublad.py index ec340eb9c..5b0c8949e 100644 --- a/backend/corpora/ublad/ublad.py +++ b/backend/corpora/ublad/ublad.py @@ -1,23 +1,17 @@ from datetime import datetime import os -from os.path import join, splitext -import locale +from os.path import join import logging from django.conf import settings from addcorpus.python_corpora.corpus import HTMLCorpusDefinition, FieldDefinition -from addcorpus.python_corpora.extract import FilterAttribute +from ianalyzer_readers.extract import XML +from ianalyzer_readers.xml_tag import Tag from addcorpus.es_mappings import * from addcorpus.python_corpora.filters import DateFilter from addcorpus.es_settings import es_settings -from ianalyzer_readers.readers.html import HTMLReader -from ianalyzer_readers.readers.core import Field -from ianalyzer_readers.extract import html, Constant - -from bs4 import BeautifulSoup, Tag - def transform_content(soup): """ Transforms the text contents of a page node (soup) into a string consisting @@ -39,13 +33,18 @@ def transform_content(soup): page_text += paragraph_text + '\n\n' return page_text +months = ['januari', 'februari', 'maart', 'april', 'mei', 'juni', 'juli', 'augustus', + 'september', 'oktober', 'november', 'december'] + def transform_date(date_string): + day_string, month_string, year_string = date_string.split() try: - locale.setlocale(locale.LC_ALL, 'nl_NL.UTF-8') - date = datetime.strptime(date_string, '%d %B %Y').strftime('%Y-%m-%d') - locale.setlocale(locale.LC_ALL, '') - return date - except ValueError: + day = int(day_string) + month = next(i + 1 for i, month in enumerate(months) if month == month_string) + year = int(year_string) + date = datetime(year=year, month=month, day=day) + return date.strftime('%Y-%m-%d') + except: logger.error("Unable to get date from {}".format(date_string)) return None @@ -79,7 +78,7 @@ class UBlad(HTMLCorpusDefinition): def es_settings(self): return es_settings(self.languages[:1], stopword_analysis=True, stemming_analysis=True) - def sources(self, start=min_date, end=max_date): + def sources(self, **kwargs): for directory, _, filenames in os.walk(self.data_directory): _body, tail = os.path.split(directory) if '.snapshot' in _: @@ -102,15 +101,10 @@ def sources(self, start=min_date, end=max_date): search_field_core=True, visualizations=['ngram', 'wordcloud'], es_mapping = main_content_mapping(True, True, True, 'nl'), - extractor= FilterAttribute(tag='div', - recursive=True, - multiple=False, - flatten=False, - extract_soup_func=transform_content, - attribute_filter={ - 'attribute': 'class', - 'value': 'ocr_page' - }) + extractor=XML( + Tag('div', attrs={'class': 'ocr_page'}), + extract_soup_func=transform_content, + ) ), FieldDefinition( name='pagenum', @@ -118,21 +112,19 @@ def sources(self, start=min_date, end=max_date): description='Page number', csv_core=True, es_mapping = int_mapping(), - extractor = FilterAttribute(tag='meta', attribute='content', attribute_filter={ - 'attribute': 'name', - 'value': 'pagenum' - } + extractor = XML( + Tag('meta', attrs={'name': 'pagenum'}), + attribute='content' ) ), FieldDefinition( name='journal_title', display_name='Publication Title', description='Title of the publication', - extractor = FilterAttribute(tag='meta', attribute='content', attribute_filter={ - 'attribute': 'name', - 'value': 'journal_title' - } - ) + extractor=XML( + Tag('meta', attrs={'name': 'journal_title'}), + attribute='content', + ), ), FieldDefinition( name='volume_id', @@ -140,21 +132,19 @@ def sources(self, start=min_date, end=max_date): description='Unique identifier for this volume', hidden=True, es_mapping=keyword_mapping(), - extractor = FilterAttribute(tag='meta', attribute='content', attribute_filter={ - 'attribute': 'name', - 'value': 'identifier_ocn' - } - ) + extractor=XML( + Tag('meta', attrs={'name': 'identifier_ocn'}), + attribute='content', + ), ), FieldDefinition( name='id', display_name='Page ID', description='Unique identifier for this page', hidden=True, - extractor = FilterAttribute(tag='meta', attribute='content', attribute_filter={ - 'attribute': 'name', - 'value': 'identifier_indexid' - } + extractor=XML( + Tag('meta', attrs={'name': 'identifier_indexid'}), + attribute='content', ) ), FieldDefinition( @@ -163,10 +153,9 @@ def sources(self, start=min_date, end=max_date): description='The number of the edition in this volume. Every year starts at 1.', sortable=True, es_mapping = keyword_mapping(), - extractor = FilterAttribute(tag='meta', attribute='content', attribute_filter={ - 'attribute': 'name', - 'value': 'aflevering' - } + extractor=XML( + Tag('meta', attrs={'name': 'aflevering'}), + attribute='content', ) ), FieldDefinition( @@ -177,10 +166,9 @@ def sources(self, start=min_date, end=max_date): csv_core=True, description='The volume number of this publication. There is one volume per year.', es_mapping=keyword_mapping(), - extractor = FilterAttribute(tag='meta', attribute='content', attribute_filter={ - 'attribute': 'name', - 'value': 'yearstring' - } + extractor=XML( + Tag('meta', attrs={'name': 'yearstring'}), + attribute='content', ), ), FieldDefinition( @@ -198,12 +186,12 @@ def sources(self, start=min_date, end=max_date): 'Accept only articles with publication date in this range.' ) ), - extractor = FilterAttribute(tag='meta', attribute='content', attribute_filter={ - 'attribute': 'name', - 'value': 'datestring', - }, - transform=transform_date - ) + extractor=XML( + Tag('meta', attrs={'name': 'datestring'}), + attribute='content', + transform=transform_date, + + ), ), FieldDefinition( name='repo_url', @@ -212,11 +200,10 @@ def sources(self, start=min_date, end=max_date): es_mapping=keyword_mapping(), display_type='url', searchable=False, - extractor=FilterAttribute(tag='meta', attribute='content', attribute_filter={ - 'attribute': 'name', - 'value': 'link_repository' - } - ) + extractor=XML( + Tag('meta', attrs={'name': 'link_repository'}), + attribute='content', + ), ), FieldDefinition( name='reader_url', @@ -225,11 +212,10 @@ def sources(self, start=min_date, end=max_date): es_mapping=keyword_mapping(), display_type='url', searchable=False, - extractor=FilterAttribute(tag='meta', attribute='content', attribute_filter={ - 'attribute': 'name', - 'value': 'link_objects_image' - } - ) + extractor=XML( + Tag('meta', attrs={'name': 'link_objects_image'}), + attribute='content', + ), ), FieldDefinition( name='jpg_url', @@ -238,11 +224,10 @@ def sources(self, start=min_date, end=max_date): es_mapping=keyword_mapping(), display_type='url', searchable=False, - extractor=FilterAttribute(tag='meta', attribute='content', attribute_filter={ - 'attribute': 'name', - 'value': 'link_objects_jpg' - } - ) + extractor=XML( + Tag('meta', attrs={'name': 'link_objects_jpg'}), + attribute='content', + ), ), FieldDefinition( name='worldcat_url', @@ -251,11 +236,10 @@ def sources(self, start=min_date, end=max_date): es_mapping=keyword_mapping(), display_type='url', searchable=False, - extractor=FilterAttribute(tag='meta', attribute='content', attribute_filter={ - 'attribute': 'name', - 'value': 'link_worldcat' - } - ) + extractor=XML( + Tag('meta', attrs={'name': 'link_worldcat'}), + attribute='content', + ), ) ] diff --git a/backend/download/tests/test_download_views.py b/backend/download/tests/test_download_views.py index 7cbbd981c..0aec685ad 100644 --- a/backend/download/tests/test_download_views.py +++ b/backend/download/tests/test_download_views.py @@ -265,3 +265,22 @@ def test_query_text_in_csv(db, client, basic_mock_corpus, basic_corpus_public, i reader = csv.DictReader(stream, delimiter=';') row = next(reader) assert row['query'] == 'ghost' + +@pytest.mark.xfail(reason='query in context download does not work') +def test_download_with_query_in_context( + db, admin_client, small_mock_corpus, index_small_mock_corpus +): + es_query = query.set_query_text(query.MATCH_ALL, 'the') + es_query['highlight'] = { 'fragment_size': 200, 'fields': { 'content': {} } } + es_query['size'] = 3 + request_json = { + 'corpus': small_mock_corpus, + 'es_query': es_query, + 'fields': ['date', 'content', 'context'], + 'route': f"/search/{small_mock_corpus}?query=the&highlight=200", + 'encoding': 'utf-8' + } + response = admin_client.post( + '/api/download/search_results', request_json, content_type='application/json' + ) + assert status.is_success(response.status_code) diff --git a/backend/download/views.py b/backend/download/views.py index e6d96a819..a8bb4db38 100644 --- a/backend/download/views.py +++ b/backend/download/views.py @@ -2,14 +2,13 @@ import os from addcorpus.models import Corpus -from addcorpus.permissions import (CorpusAccessPermission, +from addcorpus.permissions import (CanSearchCorpus, corpus_name_from_request) from django.conf import settings from django.http.response import FileResponse from download import convert_csv, tasks from download.models import Download from download.serializers import DownloadSerializer -from es import download as es_download from rest_framework.exceptions import (APIException, NotFound, ParseError, PermissionDenied) from rest_framework.permissions import IsAuthenticated @@ -17,7 +16,6 @@ from rest_framework.views import APIView from rest_framework.viewsets import ModelViewSet from api.utils import check_json_keys -from api.api_query import api_query_to_es_query logger = logging.getLogger() @@ -36,7 +34,7 @@ class ResultsDownloadView(APIView): Download search results up to 1.000 documents ''' - permission_classes = [CorpusAccessPermission] + permission_classes = [CanSearchCorpus] def post(self, request, *args, **kwargs): check_json_keys(request, ['es_query', 'corpus', 'fields', 'route', 'encoding']) @@ -64,7 +62,7 @@ class ResultsDownloadTaskView(APIView): over 10.000 documents ''' - permission_classes = [IsAuthenticated, CorpusAccessPermission] + permission_classes = [IsAuthenticated, CanSearchCorpus] def post(self, request, *args, **kwargs): check_json_keys(request, ['es_query', 'corpus', 'fields', 'route']) @@ -88,7 +86,7 @@ class FullDataDownloadTaskView(APIView): for a visualisation. ''' - permission_classes = [IsAuthenticated, CorpusAccessPermission] + permission_classes = [IsAuthenticated, CanSearchCorpus] def post(self, request, *args, **kwargs): check_json_keys(request, ['visualization', 'parameters', 'corpus_name']) diff --git a/backend/es/conftest.py b/backend/es/conftest.py index 40d462e05..8f60cd588 100644 --- a/backend/es/conftest.py +++ b/backend/es/conftest.py @@ -2,6 +2,7 @@ from time import sleep from django.contrib.auth.models import Group +import elasticsearch from addcorpus.python_corpora.load_corpus import load_corpus_definition from addcorpus.models import Corpus @@ -27,8 +28,11 @@ def es_ner_search_client(es_client, basic_mock_corpus, basic_corpus_public, inde """ # add data from mock corpus corpus = Corpus.objects.get(name=basic_mock_corpus) - es_client.indices.put_mapping(index=corpus.configuration.es_index, properties={ - "content:ner": {"type": "annotated_text"}}) + try: + es_client.indices.put_mapping(index=corpus.configuration.es_index, properties={ + "content:ner": {"type": "annotated_text"}}) + except elasticsearch.BadRequestError: + pytest.skip('Annotated text plugin not installed') es_client.index(index=corpus.configuration.es_index, document={ 'id': 'my_identifier', diff --git a/backend/es/views.py b/backend/es/views.py index 55e36ed7c..dde609fec 100644 --- a/backend/es/views.py +++ b/backend/es/views.py @@ -5,8 +5,7 @@ from rest_framework.views import APIView from rest_framework.response import Response from rest_framework.exceptions import APIException - -from addcorpus.permissions import CorpusAccessPermission +from addcorpus.permissions import CanSearchCorpus from api.save_query import should_save_query from addcorpus.models import Corpus from api.models import Query @@ -46,7 +45,7 @@ class ForwardSearchView(APIView): the query parameter will be used. ''' - permission_classes = [CorpusAccessPermission, CanSearchTags] + permission_classes = [CanSearchCorpus, CanSearchTags] def post(self, request, *args, **kwargs): corpus_name = kwargs.get('corpus') @@ -113,7 +112,7 @@ class NamedEntitySearchView(APIView): 'MISC': 'miscellaneous' } - permission_classes = [CorpusAccessPermission] + permission_classes = [CanSearchCorpus] def get(self, request, *args, **kwargs): corpus_name = kwargs.get('corpus') diff --git a/backend/ianalyzer/urls.py b/backend/ianalyzer/urls.py index 67b181ea8..b64cd5fcb 100644 --- a/backend/ianalyzer/urls.py +++ b/backend/ianalyzer/urls.py @@ -33,12 +33,13 @@ from media import urls as media_urls from tag import urls as tag_urls from tag.views import TagViewSet -from addcorpus.views import CorpusDefinitionViewset +from addcorpus.views import CorpusDefinitionViewset, CorpusDocumentationPageViewset api_router = routers.DefaultRouter() # register viewsets with this router api_router.register('search_history', QueryViewset, basename='query') api_router.register('tag/tags', TagViewSet) api_router.register('corpus/definitions', CorpusDefinitionViewset, basename='corpus') +api_router.register('corpus/documentation', CorpusDocumentationPageViewset, basename='corpus-documentation') if settings.PROXY_FRONTEND: spa_url = re_path(r'^(?P.*)$', proxy_frontend) diff --git a/backend/media/views.py b/backend/media/views.py index 2ee55d162..bc80a70d1 100644 --- a/backend/media/views.py +++ b/backend/media/views.py @@ -2,7 +2,7 @@ import os from addcorpus.python_corpora.load_corpus import load_corpus_definition -from addcorpus.permissions import (CorpusAccessPermission, +from addcorpus.permissions import (CanSearchCorpus, corpus_name_from_request) from api.utils import check_json_keys from django.http.response import FileResponse @@ -19,7 +19,7 @@ class GetMediaView(APIView): Return the image/pdf of a document ''' - permission_classes = (CorpusAccessPermission,) + permission_classes = [CanSearchCorpus] def get(self, request, *args, **kwargs): corpus_name = corpus_name_from_request(request) @@ -60,7 +60,7 @@ class MediaMetadataView(APIView): Return metadata on the media for a document ''' - permission_classes = (CorpusAccessPermission,) + permission_classes = [CanSearchCorpus] def post(self, request, *args, **kwargs): corpus_name = corpus_name_from_request(request) diff --git a/backend/requirements.txt b/backend/requirements.txt index d850969c1..de7dcd3ce 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile @@ -32,7 +32,7 @@ celery==5.3.1 # -r requirements.in # flower # pytest-celery -certifi==2023.7.22 +certifi==2024.7.4 # via # elastic-transport # requests @@ -73,9 +73,9 @@ defusedxml==0.7.1 # djangosaml2 # pysaml2 # python3-openid -dj-rest-auth[with-social,with_social]==4.0.1 +dj-rest-auth[with_social]==4.0.1 # via -r requirements.in -django==4.2.11 +django==4.2.15 # via # -r requirements.in # dj-rest-auth @@ -90,7 +90,7 @@ django-livereload-server==0.4 # via -r requirements.in django-revproxy @ git+https://github.com/jazzband/django-revproxy.git@1defbb2dad5c0632391d54bcd3dbdaeabf46266a # via -r requirements.in -djangorestframework==3.14.0 +djangorestframework==3.15.2 # via # -r requirements.in # dj-rest-auth @@ -132,7 +132,7 @@ h11==0.14.0 # wsproto humanize==4.9.0 # via flower -ianalyzer-readers==0.1.0 +ianalyzer-readers==0.2.0 # via -r requirements.in idna==3.4 # via @@ -143,6 +143,8 @@ iniconfig==2.0.0 # via # pytest # seleniumbase +isodate==0.6.1 + # via rdflib joblib==1.3.2 # via # nltk @@ -175,7 +177,7 @@ mdurl==0.1.2 # seleniumbase miniful==0.0.6 # via fst-pso -nltk==3.8.1 +nltk==3.9.1 # via -r requirements.in numpy==1.24.4 # via @@ -254,6 +256,8 @@ pyopenssl==23.2.0 # via pysaml2 pyotp==2.9.0 # via seleniumbase +pyparsing==3.1.2 + # via rdflib pypdf2==3.0.1 # via -r requirements.in pysaml2==7.3.1 @@ -297,10 +301,11 @@ python3-openid==3.2.0 # via django-allauth pytz==2023.3 # via - # djangorestframework # flower # pandas # pysaml2 +rdflib==7.0.0 + # via ianalyzer-readers redis==5.0.0 # via -r requirements.in referencing==0.33.0 @@ -325,7 +330,7 @@ rpds-py==0.18.0 # referencing sbvirtualdisplay==1.2.0 # via seleniumbase -scikit-learn==1.3.0 +scikit-learn==1.5.0 # via -r requirements.in scipy==1.10.1 # via @@ -347,6 +352,7 @@ six==1.16.0 # via # behave # django-livereload-server + # isodate # langdetect # parse-type # python-dateutil @@ -395,7 +401,10 @@ trio-websocket==0.10.3 # selenium # seleniumbase typing-extensions==4.7.1 - # via asgiref + # via + # asgiref + # kombu + # pypdf2 tzdata==2023.3 # via # celery diff --git a/backend/tag/views.py b/backend/tag/views.py index 760b584ca..b29d765b0 100644 --- a/backend/tag/views.py +++ b/backend/tag/views.py @@ -3,13 +3,13 @@ from rest_framework.viewsets import ModelViewSet from rest_framework.views import APIView from django.http import HttpRequest -from rest_framework.exceptions import NotFound, PermissionDenied, ParseError +from rest_framework.exceptions import NotFound, PermissionDenied from .models import Tag, TaggedDocument from .permissions import IsTagOwner from .serializers import TagSerializer from addcorpus.models import Corpus -from addcorpus.permissions import CorpusAccessPermission +from addcorpus.permissions import CanSearchCorpus def check_corpus_name(request: HttpRequest): ''' @@ -58,7 +58,7 @@ def list(self, *args, **kwargs): return Response(serializer.data) class DocumentTagsView(APIView): - permission_classes = [IsAuthenticated, CorpusAccessPermission] + permission_classes = [IsAuthenticated, CanSearchCorpus] def get(self, request, *args, **kwargs): ''' diff --git a/backend/users/conftest.py b/backend/users/conftest.py index 3f07990aa..06b8682ae 100644 --- a/backend/users/conftest.py +++ b/backend/users/conftest.py @@ -19,7 +19,7 @@ def group_without_access(): @pytest.fixture def test_corpus(group_with_access): - corpus = Corpus.objects.create(name='test-corpus') + corpus = Corpus.objects.create(name='test-corpus', active=True) corpus.groups.add(group_with_access) corpus.save() yield corpus diff --git a/backend/users/models.py b/backend/users/models.py index 3ddcb992f..96b880fe5 100644 --- a/backend/users/models.py +++ b/backend/users/models.py @@ -1,8 +1,9 @@ import django.contrib.auth.models as django_auth_models from django.db import models +from addcorpus.models import Corpus DEFAULT_DOWNLOAD_LIMIT = 10000 - +PUBLIC_GROUP_NAME = 'basic' class CustomUser(django_auth_models.AbstractUser): saml = models.BooleanField(blank=True, null=True, default=False) @@ -10,14 +11,29 @@ class CustomUser(django_auth_models.AbstractUser): help_text='Maximum documents that this user can download per query', default=DEFAULT_DOWNLOAD_LIMIT) - def has_access(self, corpus_name): - # superusers automatically have access to all corpora + def can_search(self, corpus: Corpus) -> bool: + ''' + Whether the user is allowed to search the corpus + ''' + + if not corpus.active: + return False + + # superusers do not need explicit group membership if self.is_superuser: return True - # check if any corpus added to the user's group(s) match the corpus name - return any(corpus for group in self.groups.all() - for corpus in group.corpora.filter(name=corpus_name)) + return self.groups.filter(corpora=corpus).exists() + + def searchable_corpora(self): + ''' + Queryset of corpora that the user is allowed to search + ''' + + if self.is_superuser: + return Corpus.objects.filter(active=True) + + return Corpus.objects.filter(active=True, groups__user=self) class AnoymousProfile(object): @@ -30,10 +46,15 @@ class CustomAnonymousUser(django_auth_models.AnonymousUser): ''' profile = AnoymousProfile() - def has_access(self, corpus_name): - basic_group, _ = django_auth_models.Group.objects.get_or_create( - name='basic') - return basic_group.corpora.filter(name=corpus_name) + def can_search(self, corpus: Corpus): + if not corpus.active: + return False + + Group = django_auth_models.Group + return Group.objects.filter(name=PUBLIC_GROUP_NAME, corpora=corpus).exists() + + def searchable_corpora(self): + return Corpus.objects.filter(active=True, groups__name=PUBLIC_GROUP_NAME) django_auth_models.AnonymousUser = CustomAnonymousUser diff --git a/backend/users/tests/test_corpus_access.py b/backend/users/tests/test_corpus_access.py index 62bbe455d..e34b5e9a3 100644 --- a/backend/users/tests/test_corpus_access.py +++ b/backend/users/tests/test_corpus_access.py @@ -1,16 +1,14 @@ from users.models import CustomUser -from addcorpus.models import Corpus -from django.contrib.auth.models import Group def test_corpus_access(db, group_with_access, group_without_access, test_corpus): user = CustomUser.objects.create(username='test-user') - assert not user.has_access('test-corpus') + assert not user.can_search(test_corpus) user.groups.add(group_without_access) user.save() - assert not user.has_access('test-corpus') + assert not user.can_search(test_corpus) user.groups.add(group_with_access) user.save() - assert user.has_access('test-corpus') + assert user.can_search(test_corpus) diff --git a/backend/visualization/tasks.py b/backend/visualization/tasks.py index 2050e107c..c8c2e8a2d 100644 --- a/backend/visualization/tasks.py +++ b/backend/visualization/tasks.py @@ -1,7 +1,7 @@ from celery import chord, group, shared_task from django.conf import settings from visualization import wordcloud, ngram, term_frequency -from es import download as es_download +from es import download as es_download, search as es_search from api.api_query import api_query_to_es_query @shared_task() @@ -23,7 +23,42 @@ def get_geo_data(request_json): es_query = api_query_to_es_query(request_json, corpus_name) list_of_documents, _ = es_download.scroll( corpus_name, es_query, source_includes=['id', geo_field]) - return list_of_documents + + # Convert documents to GeoJSON features + geojson_features = [] + for doc in list_of_documents: + if doc['_source'][geo_field] is not None: + feature = { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": doc['_source'][geo_field]['coordinates'] + }, + "properties": { + "id": doc['_source']['id'] + } + } + geojson_features.append(feature) + + return geojson_features + + +@shared_task() +def get_geo_centroid(request_json): + corpus_name = request_json['corpus'] + geo_field = request_json['field'] + query_model = { + "aggs": { + "center": { + "geo_centroid": { + "field": geo_field + } + } + } + } + result = es_search.search(corpus_name, query_model, size=0) + return es_search.aggregation_results(result)['center'] + @shared_task def get_ngram_data_bin(**kwargs): diff --git a/backend/visualization/urls.py b/backend/visualization/urls.py index ed26f6a97..62a0ada49 100644 --- a/backend/visualization/urls.py +++ b/backend/visualization/urls.py @@ -4,6 +4,7 @@ urlpatterns = [ path('wordcloud', WordcloudView.as_view()), path('geo', MapView.as_view()), + path('geo_centroid', MapCentroidView.as_view()), path('ngram', NgramView.as_view()), path('date_term_frequency', DateTermFrequencyView.as_view()), path('aggregate_term_frequency', AggregateTermFrequencyView.as_view()), diff --git a/backend/visualization/views.py b/backend/visualization/views.py index dd6c93763..8657761ce 100644 --- a/backend/visualization/views.py +++ b/backend/visualization/views.py @@ -6,7 +6,7 @@ import logging from django.conf import settings from rest_framework.permissions import IsAuthenticated -from addcorpus.permissions import CorpusAccessPermission +from addcorpus.permissions import CanSearchCorpus from tag.permissions import CanSearchTags from visualization.field_stats import report_coverage from addcorpus.permissions import corpus_name_from_request @@ -20,7 +20,7 @@ class WordcloudView(APIView): Most frequent terms for a small batch of results ''' - permission_classes = [CorpusAccessPermission, CanSearchTags] + permission_classes = [CanSearchCorpus, CanSearchTags] def post(self, request, *args, **kwargs): check_json_keys(request, ['corpus', 'es_query', 'field', 'size']) @@ -40,11 +40,10 @@ def post(self, request, *args, **kwargs): class MapView(APIView): ''' - Most frequent terms for a small batch of results + Retrieve documents with geo_field coordinates. ''' - permission_classes = [IsAuthenticated, - CorpusAccessPermission, CanSearchTags] + permission_classes = [CanSearchCorpus] def post(self, request, *args, **kwargs): check_json_keys(request, ['corpus', 'es_query', 'field']) @@ -57,12 +56,28 @@ def post(self, request, *args, **kwargs): raise APIException(detail='could not generate geo data') +class MapCentroidView(APIView): + ''' + Retrieve the centroid of documents with a geo_field for a corpus. + ''' + + permission_classes = [CanSearchCorpus] + + def post(self, request, *args, **kwargs): + check_json_keys(request, ['corpus', 'field']) + try: + center = tasks.get_geo_centroid(request.data) + return Response(center) + except Exception as e: + logger.error(e) + raise APIException(detail='Could not retrieve geo centroid') + class NgramView(APIView): ''' Schedule a task to retrieve ngrams containing the search term ''' - permission_classes = [CorpusAccessPermission, CanSearchTags] + permission_classes = [CanSearchCorpus, CanSearchTags] def post(self, request, *args, **kwargs): check_json_keys(request, [ @@ -86,7 +101,7 @@ class DateTermFrequencyView(APIView): compared by a date field ''' - permission_classes = [CorpusAccessPermission, CanSearchTags] + permission_classes = [CanSearchCorpus, CanSearchTags] def post(self, request, *args, **kwargs): check_json_keys( @@ -114,7 +129,7 @@ class AggregateTermFrequencyView(APIView): compared by a keyword field ''' - permission_classes = [CorpusAccessPermission, CanSearchTags] + permission_classes = [CanSearchCorpus, CanSearchTags] def post(self, request, *args, **kwargs): check_json_keys( @@ -141,7 +156,7 @@ class FieldCoverageView(APIView): Get the coverage of each field in a corpus ''' - permission_classes = [CorpusAccessPermission] + permission_classes = [CanSearchCorpus] def get(self, request, *args, **kwargs): corpus = corpus_name_from_request(request) diff --git a/backend/wordmodels/tests/test_wordmodels_views.py b/backend/wordmodels/tests/test_wordmodels_views.py index b82b020a4..cdd5ac689 100644 --- a/backend/wordmodels/tests/test_wordmodels_views.py +++ b/backend/wordmodels/tests/test_wordmodels_views.py @@ -1,8 +1,8 @@ import pytest def test_wm_documentation_view(admin_client, mock_corpus): - response = admin_client.get(f'/api/corpus/documentation/{mock_corpus}/') - assert any(page['type'] == 'Word models' for page in response.data) + response = admin_client.get(f'/api/corpus/documentation/?corpus={mock_corpus}') + assert any(page['type'] == 'Word models'for page in response.data) def test_related_words_view(admin_client, mock_corpus): diff --git a/backend/wordmodels/views.py b/backend/wordmodels/views.py index 1c4d28ac3..4992e6207 100644 --- a/backend/wordmodels/views.py +++ b/backend/wordmodels/views.py @@ -1,6 +1,6 @@ from rest_framework.views import APIView from rest_framework.response import Response -from addcorpus.permissions import CorpusAccessPermission, corpus_name_from_request +from addcorpus.permissions import CanSearchCorpus, corpus_name_from_request from wordmodels import utils, visualisations from rest_framework.exceptions import APIException @@ -9,7 +9,7 @@ class RelatedWordsView(APIView): Get words with the highest similarity to the query term ''' - permission_classes = [CorpusAccessPermission] + permission_classes = [CanSearchCorpus] def post(self, request, *args, **kwargs): corpus = corpus_name_from_request(request) @@ -33,7 +33,7 @@ class SimilarityView(APIView): Get similarity between two query terms ''' - permission_classes = [CorpusAccessPermission] + permission_classes = [CanSearchCorpus] def get(self, request, *args, **kwargs): corpus = corpus_name_from_request(request) @@ -53,7 +53,7 @@ class WordInModelView(APIView): Check if a word has a vector in the model for a corpus ''' - permission_classes = [CorpusAccessPermission] + permission_classes = [CanSearchCorpus] def get(self, request, *args, **kwargs): corpus = corpus_name_from_request(request) diff --git a/docker-compose.yaml b/docker-compose.yaml index f3f4d63b3..90f5481c7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -13,6 +13,7 @@ services: volumes: - ianalyzer-db:/var/lib/postgresql/data/ backend: + image: ghcr.io/uudigitalhumanitieslab/ianalyzer-backend:latest build: context: ./backend depends_on: @@ -39,6 +40,7 @@ services: target: /corpora command: bash -c "python manage.py migrate && python manage.py loadcorpora && python manage.py runserver 0.0.0.0:8000" frontend: + image: ghcr.io/uudigitalhumanitieslab/ianalyzer-frontend:latest build: context: ./frontend ports: @@ -52,6 +54,7 @@ services: target: /frontend/build command: sh -c "yarn prebuild && yarn start-docker" elasticsearch: + image: ghcr.io/uudigitalhumanitieslab/ianalyzer-elastic:latest build: context: . dockerfile: DockerfileElastic @@ -79,8 +82,7 @@ services: image: redis:latest restart: unless-stopped celery: - build: - context: ./backend + image: ghcr.io/uudigitalhumanitieslab/ianalyzer-backend:latest environment: CELERY_BROKER: $CELERY_BROKER SQL_DATABASE: $SQL_DATABASE diff --git a/documentation/Making-a-release.md b/documentation/Making-a-release.md index 33be0298e..e06d0a1dc 100644 --- a/documentation/Making-a-release.md +++ b/documentation/Making-a-release.md @@ -23,13 +23,14 @@ Determine if your release is a major, minor, or patch release to figure out the Start a new branch for your releases. Use `git flow release start x.x.x` or `git flow hotfix start x.x.x`. -Update the version number in `package.json`. + +Use the `yarn [major|minor|patch]` command to update the version number in `package.json`. This also updates the `CITATION.cff` file with the new version number and release date. ## Check if everything works In your local environment, start up elasticsearch and run backend tests with `yarn test-back`. Run frontend tests with `yarn test-front`. -Publish the release branch with `git flow release publish x.x.x`. The push will trigger the [release workflow](https://github.com/UUDigitalHumanitieslab/I-analyzer/blob/develop/.github/workflows/release.yaml) to update the version number and release date in `CITATION.cff`. Deploy on the test or acc server. Check that everything works as intended. +Publish the release branch with `git flow release publish x.x.x`. Deploy on the test or acc server. Check that everything works as intended. ## Publish the release diff --git a/frontend/package.json b/frontend/package.json index 4e09b6657..058d17c2a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -59,6 +59,9 @@ "terser-webpack-plugin": "^5.3.0", "tslib": "^2.0.0", "typescript": "5.4.3", + "vega": "^5.30.0", + "vega-embed": "^6.26.0", + "vega-lite": "^5.19.0", "zone.js": "~0.14.4" }, "devDependencies": { diff --git a/frontend/src/app/about/about.component.ts b/frontend/src/app/about/about.component.ts index 3699bfcf5..580cfbf2f 100644 --- a/frontend/src/app/about/about.component.ts +++ b/frontend/src/app/about/about.component.ts @@ -1,7 +1,7 @@ import { Component, OnInit } from '@angular/core'; import { SafeHtml, Title } from '@angular/platform-browser'; -import { environment } from '../../environments/environment'; -import { DialogService } from '../services'; +import { environment } from '@environments/environment'; +import { DialogService } from '@services'; @Component({ selector: 'ia-about', diff --git a/frontend/src/app/app.component.ts b/frontend/src/app/app.component.ts index 8b38bd766..c8a70f381 100644 --- a/frontend/src/app/app.component.ts +++ b/frontend/src/app/app.component.ts @@ -1,7 +1,7 @@ import { Component } from '@angular/core'; import { AuthService } from './services/auth.service'; -import { environment } from '../environments/environment'; +import { environment } from '@environments/environment'; @Component({ selector: 'ia-root', diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index 20c8c3e7c..a2482d60f 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -53,17 +53,18 @@ import { SharedModule } from './shared/shared.module'; import { TagOverviewComponent } from './tag/tag-overview/tag-overview.component'; import { WordModelsComponent } from './word-models/word-models.component'; import { WordModelsModule } from './word-models/word-models.module'; +import { forwardLegacyParamsGuard } from './forward-legacy-params.guard'; export const appRoutes: Routes = [ { path: 'search/:corpus', component: SearchComponent, - canActivate: [CorpusGuard], + canActivate: [CorpusGuard, forwardLegacyParamsGuard], }, { path: 'word-models/:corpus', component: WordModelsComponent, - canActivate: [CorpusGuard], + canActivate: [CorpusGuard, forwardLegacyParamsGuard], }, { path: 'info/:corpus', diff --git a/frontend/src/app/corpus-definitions/corpus-definitions.module.ts b/frontend/src/app/corpus-definitions/corpus-definitions.module.ts index 9bfd7a334..19ec64bc2 100644 --- a/frontend/src/app/corpus-definitions/corpus-definitions.module.ts +++ b/frontend/src/app/corpus-definitions/corpus-definitions.module.ts @@ -1,6 +1,6 @@ import { NgModule } from '@angular/core'; import { DefinitionsOverviewComponent } from './definitions-overview/definitions-overview.component'; -import { SharedModule } from '../shared/shared.module'; +import { SharedModule } from '@shared/shared.module'; import { CreateDefinitionComponent } from './create-definition/create-definition.component'; import { EditDefinitionComponent } from './edit-definition/edit-definition.component'; import { DefinitionJsonUploadComponent } from './definition-json-upload/definition-json-upload.component'; diff --git a/frontend/src/app/corpus-definitions/create-definition/create-definition.component.ts b/frontend/src/app/corpus-definitions/create-definition/create-definition.component.ts index 9d7e16469..676501c68 100644 --- a/frontend/src/app/corpus-definitions/create-definition/create-definition.component.ts +++ b/frontend/src/app/corpus-definitions/create-definition/create-definition.component.ts @@ -1,11 +1,11 @@ import { Component } from '@angular/core'; -import { actionIcons, formIcons } from '../../shared/icons'; -import { ApiService } from '../../services'; -import { APIEditableCorpus, CorpusDefinition } from '../../models/corpus-definition'; +import { actionIcons, formIcons } from '@shared/icons'; +import { APIEditableCorpus, CorpusDefinition } from '@models/corpus-definition'; import * as _ from 'lodash'; import { Router } from '@angular/router'; import { HttpErrorResponse } from '@angular/common/http'; import { Subject } from 'rxjs'; +import { ApiService } from '@services'; @Component({ selector: 'ia-create-definition', diff --git a/frontend/src/app/corpus-definitions/definition-json-upload/definition-json-upload.component.ts b/frontend/src/app/corpus-definitions/definition-json-upload/definition-json-upload.component.ts index 4e91bf606..1c23bb7ab 100644 --- a/frontend/src/app/corpus-definitions/definition-json-upload/definition-json-upload.component.ts +++ b/frontend/src/app/corpus-definitions/definition-json-upload/definition-json-upload.component.ts @@ -2,7 +2,7 @@ import { Component, Input, OnChanges, OnDestroy, Output } from '@angular/core'; import * as _ from 'lodash'; import { BehaviorSubject, Observable, Subject, from, of } from 'rxjs'; import { catchError, filter, switchMap, takeUntil, tap } from 'rxjs/operators'; -import { actionIcons } from '../../shared/icons'; +import { actionIcons } from '@shared/icons'; @Component({ selector: 'ia-definition-json-upload', diff --git a/frontend/src/app/corpus-definitions/definitions-overview/definitions-overview.component.ts b/frontend/src/app/corpus-definitions/definitions-overview/definitions-overview.component.ts index 399950560..dee403457 100644 --- a/frontend/src/app/corpus-definitions/definitions-overview/definitions-overview.component.ts +++ b/frontend/src/app/corpus-definitions/definitions-overview/definitions-overview.component.ts @@ -1,8 +1,8 @@ import { Component } from '@angular/core'; -import { actionIcons } from '../../shared/icons'; -import { ApiService} from '../../services'; +import { actionIcons } from '@shared/icons'; +import { ApiService } from '@services'; import { Observable } from 'rxjs'; -import { APIEditableCorpus } from '../../models/corpus-definition'; +import { APIEditableCorpus } from '@models/corpus-definition'; import * as _ from 'lodash'; @Component({ diff --git a/frontend/src/app/corpus-definitions/edit-definition/edit-definition.component.ts b/frontend/src/app/corpus-definitions/edit-definition/edit-definition.component.ts index 262b26286..fde7c1957 100644 --- a/frontend/src/app/corpus-definitions/edit-definition/edit-definition.component.ts +++ b/frontend/src/app/corpus-definitions/edit-definition/edit-definition.component.ts @@ -1,8 +1,8 @@ import { Component, } from '@angular/core'; -import { actionIcons, formIcons } from '../../shared/icons'; +import { actionIcons, formIcons } from '@shared/icons'; import { Subject } from 'rxjs'; -import { CorpusDefinition } from '../../models/corpus-definition'; -import { ApiService } from '../../services'; +import { CorpusDefinition } from '@models/corpus-definition'; +import { ApiService } from '@services'; import { ActivatedRoute } from '@angular/router'; import * as _ from 'lodash'; import { HttpErrorResponse } from '@angular/common/http'; diff --git a/frontend/src/app/corpus-header/corpus-header.component.ts b/frontend/src/app/corpus-header/corpus-header.component.ts index 9abffd5fc..cf9bf67a4 100644 --- a/frontend/src/app/corpus-header/corpus-header.component.ts +++ b/frontend/src/app/corpus-header/corpus-header.component.ts @@ -1,6 +1,6 @@ import { Component, Input, OnChanges, OnInit, SimpleChanges } from '@angular/core'; -import { Corpus } from '../models'; -import { corpusIcons } from '../shared/icons'; +import { Corpus } from '@models'; +import { corpusIcons } from '@shared/icons'; @Component({ selector: 'ia-corpus-header', diff --git a/frontend/src/app/corpus-header/corpus.module.ts b/frontend/src/app/corpus-header/corpus.module.ts index d5d658fae..a4da975c6 100644 --- a/frontend/src/app/corpus-header/corpus.module.ts +++ b/frontend/src/app/corpus-header/corpus.module.ts @@ -1,7 +1,7 @@ import { NgModule } from '@angular/core'; -import { SharedModule } from '../shared/shared.module'; +import { SharedModule } from '@shared/shared.module'; import { CorpusHeaderComponent } from './corpus-header.component'; -import { CorpusService } from '../services'; +import { CorpusService } from '@services'; import { RouterModule } from '@angular/router'; import { CorpusInfoComponent } from '../corpus-info/corpus-info.component'; import { FieldInfoComponent } from '../corpus-info/field-info/field-info.component'; diff --git a/frontend/src/app/corpus-info/corpus-info.component.ts b/frontend/src/app/corpus-info/corpus-info.component.ts index dd12cc57b..5880f1c73 100644 --- a/frontend/src/app/corpus-info/corpus-info.component.ts +++ b/frontend/src/app/corpus-info/corpus-info.component.ts @@ -1,10 +1,12 @@ import { Component, OnInit } from '@angular/core'; -import { ApiService, CorpusService } from '../services'; -import { Corpus, CorpusDocumentationPage, FieldCoverage } from '../models'; +import { ApiService, CorpusService } from '@services'; +import { Corpus, CorpusDocumentationPage, FieldCoverage } from '@models'; import { marked } from 'marked'; import { Observable } from 'rxjs'; import { Title } from '@angular/platform-browser'; -import { pageTitle } from '../utils/app'; +import { pageTitle } from '@utils/app'; +import { map } from 'rxjs/operators'; +import * as _ from 'lodash'; @Component({ selector: 'ia-corpus-info', @@ -30,7 +32,10 @@ export class CorpusInfoComponent implements OnInit { setCorpus(corpus: Corpus) { this.corpus = corpus; - this.documentation$ = this.apiService.corpusDocumentation(corpus.name); + this.documentation$ = this.apiService.corpusDocumentationPages(corpus).pipe( + map(pages => pages.filter(page => this.includePage(corpus, page))), + map(pages => _.sortBy(pages, 'index')) + ); this.apiService.fieldCoverage(corpus.name).then( result => this.fieldCoverage = result ); @@ -41,4 +46,8 @@ export class CorpusInfoComponent implements OnInit { return marked.parse(content); } + private includePage(corpus: Corpus, page: CorpusDocumentationPage): boolean { + return (page.type !== 'Word models') || corpus.wordModelsPresent; + } + } diff --git a/frontend/src/app/corpus-info/field-info/field-info.component.ts b/frontend/src/app/corpus-info/field-info/field-info.component.ts index d7add40e7..033edf0c7 100644 --- a/frontend/src/app/corpus-info/field-info/field-info.component.ts +++ b/frontend/src/app/corpus-info/field-info/field-info.component.ts @@ -1,5 +1,5 @@ import { Component, Input, OnInit } from '@angular/core'; -import { CorpusField } from '../../models'; +import { CorpusField } from '@models'; import * as _ from 'lodash'; @Component({ diff --git a/frontend/src/app/corpus-selection/corpus-filter/corpus-filter.component.spec.ts b/frontend/src/app/corpus-selection/corpus-filter/corpus-filter.component.spec.ts index 43added3b..3e6b72a34 100644 --- a/frontend/src/app/corpus-selection/corpus-filter/corpus-filter.component.spec.ts +++ b/frontend/src/app/corpus-selection/corpus-filter/corpus-filter.component.spec.ts @@ -3,7 +3,7 @@ import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing'; import { CorpusFilterComponent } from './corpus-filter.component'; import { commonTestBed } from '../../common-test-bed'; import { mockCorpus, mockCorpus2 } from '../../../mock-data/corpus'; -import { Corpus } from '../../models'; +import { Corpus } from '@models'; describe('CorpusFilterComponent', () => { let component: CorpusFilterComponent; diff --git a/frontend/src/app/corpus-selection/corpus-filter/corpus-filter.component.ts b/frontend/src/app/corpus-selection/corpus-filter/corpus-filter.component.ts index 2cb7f3617..db6d55fa1 100644 --- a/frontend/src/app/corpus-selection/corpus-filter/corpus-filter.component.ts +++ b/frontend/src/app/corpus-selection/corpus-filter/corpus-filter.component.ts @@ -1,9 +1,9 @@ import { Component, Input, OnInit, Output } from '@angular/core'; -import { Corpus } from '../../models'; +import { Corpus } from '@models'; import { BehaviorSubject, combineLatest, Observable, Subject } from 'rxjs'; import * as _ from 'lodash'; import { map } from 'rxjs/operators'; -import { formIcons } from '../../shared/icons'; +import { formIcons } from '@shared/icons'; @Component({ selector: 'ia-corpus-filter', diff --git a/frontend/src/app/corpus-selection/corpus-selection.component.ts b/frontend/src/app/corpus-selection/corpus-selection.component.ts index f22a3dc87..44872be72 100644 --- a/frontend/src/app/corpus-selection/corpus-selection.component.ts +++ b/frontend/src/app/corpus-selection/corpus-selection.component.ts @@ -1,10 +1,10 @@ import { Component, Input, OnInit } from '@angular/core'; -import { Corpus } from '../models/corpus'; +import { Corpus } from '@models/corpus'; import * as _ from 'lodash'; -import { AuthService } from '../services'; +import { AuthService } from '@services'; import { map } from 'rxjs/operators'; import { Observable } from 'rxjs'; -import { actionIcons } from '../shared/icons'; +import { actionIcons } from '@shared/icons'; @Component({ diff --git a/frontend/src/app/corpus-selection/corpus-selection.module.ts b/frontend/src/app/corpus-selection/corpus-selection.module.ts index e91c368e7..608d0a5d8 100644 --- a/frontend/src/app/corpus-selection/corpus-selection.module.ts +++ b/frontend/src/app/corpus-selection/corpus-selection.module.ts @@ -1,5 +1,5 @@ import { NgModule } from '@angular/core'; -import { SharedModule } from '../shared/shared.module'; +import { SharedModule } from '@shared/shared.module'; import { CorpusSelectionComponent } from './corpus-selection.component'; import { CorpusSelectorComponent } from './corpus-selector/corpus-selector.component'; import { CorpusFilterComponent } from './corpus-filter/corpus-filter.component'; diff --git a/frontend/src/app/corpus-selection/corpus-selector/corpus-selector.component.ts b/frontend/src/app/corpus-selection/corpus-selector/corpus-selector.component.ts index 8ceaf3dbe..60c712728 100644 --- a/frontend/src/app/corpus-selection/corpus-selector/corpus-selector.component.ts +++ b/frontend/src/app/corpus-selection/corpus-selector/corpus-selector.component.ts @@ -1,8 +1,8 @@ import { Component, Input, OnInit } from '@angular/core'; -import { Corpus } from '../../models'; +import { Corpus } from '@models'; import { Router } from '@angular/router'; import * as _ from 'lodash'; -import { corpusIcons } from '../../shared/icons'; +import { corpusIcons } from '@shared/icons'; @Component({ selector: 'ia-corpus-selector', diff --git a/frontend/src/app/dialog/dialog.component.ts b/frontend/src/app/dialog/dialog.component.ts index b4e8255d5..4413eed99 100644 --- a/frontend/src/app/dialog/dialog.component.ts +++ b/frontend/src/app/dialog/dialog.component.ts @@ -3,8 +3,8 @@ import {Router} from '@angular/router'; import { SafeHtml } from '@angular/platform-browser'; import { Subscription } from 'rxjs'; -import { DialogService } from './../services/index'; -import { navIcons } from '../shared/icons'; +import { navIcons } from '@shared/icons'; +import { DialogService } from '@services'; @Component({ selector: 'ia-dialog', diff --git a/frontend/src/app/document-page/document-page.component.ts b/frontend/src/app/document-page/document-page.component.ts index 488accb81..9540043e4 100644 --- a/frontend/src/app/document-page/document-page.component.ts +++ b/frontend/src/app/document-page/document-page.component.ts @@ -2,12 +2,12 @@ import { Component, OnInit } from '@angular/core'; import { ActivatedRoute } from '@angular/router'; import * as _ from 'lodash'; import { combineLatest } from 'rxjs'; -import { Corpus, FoundDocument } from '../models'; -import { CorpusService, ElasticSearchService } from '../services'; -import { makeContextParams } from '../utils/document-context'; -import { documentIcons } from '../shared/icons'; +import { Corpus, FoundDocument } from '@models'; +import { CorpusService, ElasticSearchService } from '@services'; +import { makeContextParams } from '@utils/document-context'; +import { documentIcons } from '@shared/icons'; import { Title } from '@angular/platform-browser'; -import { pageTitle } from '../utils/app'; +import { pageTitle } from '@utils/app'; @Component({ selector: 'ia-document-page', diff --git a/frontend/src/app/document-view/document-view.component.html b/frontend/src/app/document-view/document-view.component.html index 26c464270..16772e50f 100644 --- a/frontend/src/app/document-view/document-view.component.html +++ b/frontend/src/app/document-view/document-view.component.html @@ -17,7 +17,12 @@ - {{field.displayName}} + + + {{field.displayName}} + + { }); it('should create tabs', () => { - const debug = fixture.debugElement.queryAll(By.css('a[role=tab]')); + const debug = fixture.debugElement.queryAll(By.css('[role=tab]')); expect(debug.length).toBe(2); expect(debug[0].attributes['id']).toBe('tab-speech'); expect(debug[1].attributes['id']).toBe('tab-scan'); diff --git a/frontend/src/app/document-view/document-view.component.ts b/frontend/src/app/document-view/document-view.component.ts index 203c3d38b..58e639858 100644 --- a/frontend/src/app/document-view/document-view.component.ts +++ b/frontend/src/app/document-view/document-view.component.ts @@ -1,9 +1,9 @@ import { Component, Input, OnChanges, SimpleChanges } from '@angular/core'; -import { CorpusField, FoundDocument, Corpus, QueryModel } from '../models/index'; -import { DocumentView } from '../models/document-page'; +import { CorpusField, FoundDocument, Corpus, QueryModel } from '@models/index'; +import { DocumentView } from '@models/document-page'; import * as _ from 'lodash'; -import { documentIcons, entityIcons } from '../shared/icons'; +import { documentIcons, entityIcons } from '@shared/icons'; @Component({ selector: 'ia-document-view', diff --git a/frontend/src/app/document/document-popup/document-popup.component.spec.ts b/frontend/src/app/document/document-popup/document-popup.component.spec.ts index b497ceecf..d8040c8fd 100644 --- a/frontend/src/app/document/document-popup/document-popup.component.spec.ts +++ b/frontend/src/app/document/document-popup/document-popup.component.spec.ts @@ -5,8 +5,8 @@ import { DocumentPopupComponent } from './document-popup.component'; import { commonTestBed } from '../../common-test-bed'; import { makeDocument } from '../../../mock-data/constructor-helpers'; import { mockCorpus, mockCorpus2, mockField } from '../../../mock-data/corpus'; -import { DocumentPage } from '../../models/document-page'; -import { QueryModel } from '../../models'; +import { DocumentPage } from '@models/document-page'; +import { QueryModel } from '@models'; import { query } from '@angular/animations'; diff --git a/frontend/src/app/document/document-popup/document-popup.component.ts b/frontend/src/app/document/document-popup/document-popup.component.ts index b41997c14..8f3ef12c6 100644 --- a/frontend/src/app/document/document-popup/document-popup.component.ts +++ b/frontend/src/app/document/document-popup/document-popup.component.ts @@ -1,10 +1,14 @@ import { Component, Input, OnChanges, OnDestroy, SimpleChanges } from '@angular/core'; -import { DocumentFocus, DocumentPage, DocumentView } from '../../models/document-page'; +import { + DocumentFocus, + DocumentPage, + DocumentView, +} from '@models/document-page'; import { takeUntil } from 'rxjs/operators'; import * as _ from 'lodash'; -import { FoundDocument, QueryModel } from '../../models'; +import { FoundDocument, QueryModel } from '@models'; import { Subject } from 'rxjs'; -import { documentIcons, actionIcons, corpusIcons } from '../../shared/icons'; +import { documentIcons, actionIcons, corpusIcons } from '@shared/icons'; @Component({ selector: 'ia-document-popup', diff --git a/frontend/src/app/document/document-preview/document-preview.component.html b/frontend/src/app/document/document-preview/document-preview.component.html index 248e7a632..6e4a02f44 100644 --- a/frontend/src/app/document/document-preview/document-preview.component.html +++ b/frontend/src/app/document/document-preview/document-preview.component.html @@ -2,8 +2,10 @@ -
- {{field.displayName}}: + + + {{field.displayName}}: + diff --git a/frontend/src/app/document/document-preview/document-preview.component.spec.ts b/frontend/src/app/document/document-preview/document-preview.component.spec.ts index 7bc64ec43..9d10b6980 100644 --- a/frontend/src/app/document/document-preview/document-preview.component.spec.ts +++ b/frontend/src/app/document/document-preview/document-preview.component.spec.ts @@ -4,7 +4,7 @@ import { DocumentPreviewComponent } from './document-preview.component'; import { commonTestBed } from '../../common-test-bed'; import { mockField } from '../../../mock-data/corpus'; import { makeDocument } from '../../../mock-data/constructor-helpers'; -import { DocumentPage } from '../../models/document-page'; +import { DocumentPage } from '@models/document-page'; describe('DocumentPreviewComponent', () => { let component: DocumentPreviewComponent; diff --git a/frontend/src/app/document/document-preview/document-preview.component.ts b/frontend/src/app/document/document-preview/document-preview.component.ts index ec7bd639d..4518020da 100644 --- a/frontend/src/app/document/document-preview/document-preview.component.ts +++ b/frontend/src/app/document/document-preview/document-preview.component.ts @@ -1,7 +1,7 @@ import { Component, Input } from '@angular/core'; -import { FoundDocument } from '../../models'; -import { DocumentPage } from '../../models/document-page'; -import { actionIcons, documentIcons } from '../../shared/icons'; +import { FoundDocument } from '@models'; +import { DocumentPage } from '@models/document-page'; +import { actionIcons, documentIcons } from '@shared/icons'; @Component({ selector: 'ia-document-preview', diff --git a/frontend/src/app/document/document.module.ts b/frontend/src/app/document/document.module.ts index 67822bcc8..6bc0a9ef8 100644 --- a/frontend/src/app/document/document.module.ts +++ b/frontend/src/app/document/document.module.ts @@ -1,5 +1,5 @@ import { NgModule } from '@angular/core'; -import { SharedModule } from '../shared/shared.module'; +import { SharedModule } from '@shared/shared.module'; import { DocumentViewComponent } from '../document-view/document-view.component'; import { DocumentPageComponent } from '../document-page/document-page.component'; import { ImageViewModule } from '../image-view/image-view.module'; @@ -10,7 +10,12 @@ import { DocumentPopupComponent } from './document-popup/document-popup.componen import { DialogModule } from 'primeng/dialog'; import { DocumentPreviewComponent } from './document-preview/document-preview.component'; import { EntityLegendComponent } from './entity-legend/entity-legend.component'; -import { ElasticsearchHighlightPipe, GeoDataPipe, ParagraphPipe, SnippetPipe } from '../shared/pipes'; +import { + ElasticsearchHighlightPipe, + GeoDataPipe, + ParagraphPipe, + SnippetPipe, +} from '@shared/pipes'; @NgModule({ declarations: [ diff --git a/frontend/src/app/document/entity-legend/entity-legend.component.ts b/frontend/src/app/document/entity-legend/entity-legend.component.ts index 882fcd59c..fc830d45f 100644 --- a/frontend/src/app/document/entity-legend/entity-legend.component.ts +++ b/frontend/src/app/document/entity-legend/entity-legend.component.ts @@ -1,8 +1,8 @@ import { Component, Input, OnChanges } from '@angular/core'; import * as _ from 'lodash'; -import { entityIcons } from '../../shared/icons'; -import { FieldEntities } from '../../models'; +import { entityIcons } from '@shared/icons'; +import { FieldEntities } from '@models'; @Component({ selector: 'ia-entity-legend', diff --git a/frontend/src/app/download/download-options/download-options.component.ts b/frontend/src/app/download/download-options/download-options.component.ts index c62c4d302..be256be0a 100644 --- a/frontend/src/app/download/download-options/download-options.component.ts +++ b/frontend/src/app/download/download-options/download-options.component.ts @@ -1,5 +1,11 @@ import { Component, Input, OnInit, Output, EventEmitter, OnChanges, SimpleChanges } from '@angular/core'; -import { Download, PendingDownload, DownloadOptions, TermFrequencyParameters, TermFrequencyDownloadParameters } from '../../models'; +import { + Download, + PendingDownload, + DownloadOptions, + TermFrequencyParameters, + TermFrequencyDownloadParameters, +} from '@models'; @Component({ selector: 'ia-download-options', diff --git a/frontend/src/app/download/download.component.html b/frontend/src/app/download/download.component.html index 485160d46..5dd0fc22b 100644 --- a/frontend/src/app/download/download.component.html +++ b/frontend/src/app/download/download.component.html @@ -1,22 +1,99 @@ -
-
-

- -

-

- - + +

{{total}} results.

+ +

+ You can download your search results as a CSV file. View the + manual + for more information. +

+ +
+

+ Only the first {{downloadLimit}} results will be included in the file.

-
+
+
+ +
+ + +
+

+ Select which fields should be included as columns in the CSV file. +

+
+ +
+

+ Sort results +

+ +
+ + + - + +
+
+ File encoding + +
+

+ We recommend using utf-8 encoding for most applications, including Python and R. + For importing files in Microsoft Excel, we recommend utf-16. +

+
+
+ +
+ + +
+

+ Your download contains too many documents to be immediately available. + You can request the download now, and receive an email when it's + ready. +

+
+
+ +
+ +
+ diff --git a/frontend/src/app/download/download.component.spec.ts b/frontend/src/app/download/download.component.spec.ts index 6d802b880..7feb28315 100644 --- a/frontend/src/app/download/download.component.spec.ts +++ b/frontend/src/app/download/download.component.spec.ts @@ -2,9 +2,10 @@ import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing'; import { mockCorpus, mockField, mockField2 } from '../../mock-data/corpus'; import { commonTestBed } from '../common-test-bed'; -import { QueryModel } from '../models'; +import { QueryModel } from '@models'; import { DownloadComponent } from './download.component'; +import { SimpleChange } from '@angular/core'; describe('DownloadComponent', () => { let component: DownloadComponent; @@ -19,7 +20,9 @@ describe('DownloadComponent', () => { component = fixture.componentInstance; component.corpus = mockCorpus; component.queryModel = new QueryModel(mockCorpus); - component.ngOnChanges(); + component.ngOnChanges({ + queryModel: new SimpleChange(undefined, component.queryModel, true) + }); fixture.detectChanges(); }); @@ -29,16 +32,15 @@ describe('DownloadComponent', () => { it('should respond to field selection', () => { // Start with a single field - expect(component['getCsvFields']()).toEqual(mockCorpus.fields); + expect(component['getColumnNames']()).toEqual(['great_field', 'speech']); // Deselect all - component.selectCsvFields([]); - expect(component['getCsvFields']()).toEqual([]); + component.selectedCsvFields = []; + expect(component['getColumnNames']()).toEqual([]); // Select two - component.selectCsvFields([mockField, mockField2]); - const expected_fields = [mockField, mockField2]; - expect(component['getCsvFields']()).toEqual(expected_fields); - expect(component.selectedCsvFields).toEqual(expected_fields); + component.selectedCsvFields = [mockField, mockField2]; + const expected_fields = ['great_field', 'speech']; + expect(component['getColumnNames']()).toEqual(expected_fields); }); }); diff --git a/frontend/src/app/download/download.component.ts b/frontend/src/app/download/download.component.ts index 62c1db767..b8d3dbfd2 100644 --- a/frontend/src/app/download/download.component.ts +++ b/frontend/src/app/download/download.component.ts @@ -1,10 +1,32 @@ -import { Component, Input, OnChanges } from '@angular/core'; +import { Component, Input, OnChanges, SimpleChanges } from '@angular/core'; import * as _ from 'lodash'; -import { environment } from '../../environments/environment'; -import { DownloadService, NotificationService } from '../services/index'; -import { Corpus, CorpusField, DownloadOptions, PendingDownload, QueryModel, ResultOverview } from '../models/index'; -import { actionIcons } from '../shared/icons'; +import { environment } from '@environments/environment'; +import { + AuthService, + DownloadService, + NotificationService, + SearchService, +} from '@services/index'; +import { + Corpus, + CorpusField, + PendingDownload, + QueryModel, + SortState, +} from '@models/index'; +import { actionIcons } from '@shared/icons'; +import { TotalResults } from '@models/total-results'; +import { SimpleStore } from '../store/simple-store'; +import { Observable, map } from 'rxjs'; +import { Router } from '@angular/router'; +import { pageResultsParametersToParams } from '@utils/params'; +import { + DEFAULT_HIGHLIGHT_SIZE, + PageResults, + PageResultsParameters, +} from '@models/page-results'; + @Component({ selector: 'ia-download', @@ -14,11 +36,6 @@ import { actionIcons } from '../shared/icons'; export class DownloadComponent implements OnChanges { @Input() public corpus: Corpus; @Input() public queryModel: QueryModel; - @Input() public resultOverview: ResultOverview; - @Input() public hasLimitedResults: boolean; - // download limit is either the user's download limit, or (for unauthenticated users) the corpus' direct download limit - @Input() public downloadLimit: number; - @Input() public route: string; public selectedCsvFields: CorpusField[]; public availableCsvFields: CorpusField[]; @@ -29,9 +46,22 @@ export class DownloadComponent implements OnChanges { public pendingDownload: PendingDownload; + resultsConfig: PageResults; + actionIcons = actionIcons; - directDownloadLimit = environment.directDownloadLimit; + downloadLimit: number; + + canDownloadDirectly$: Observable; + + encodingOptions = ['utf-8', 'utf-16']; + encoding: 'utf-8' | 'utf-16' = 'utf-8'; + + totalResults: TotalResults; + downloadDisabled$: Observable; + + private directDownloadLimit: number = environment.directDownloadLimit; + private userDownloadLimit: number; private downloadsPageLink = { text: 'view downloads', @@ -40,66 +70,61 @@ export class DownloadComponent implements OnChanges { constructor( private downloadService: DownloadService, - private notificationService: NotificationService - ) {} - - get downloadDisabled(): boolean { - return !this.resultOverview || this.resultOverview.resultsCount === 0; + private notificationService: NotificationService, + private searchService: SearchService, + private authService: AuthService, + private router: Router, + ) { + this.userDownloadLimit = this.authService.getCurrentUser()?.downloadLimit; + this.downloadLimit = this.userDownloadLimit || this.directDownloadLimit; } - ngOnChanges() { - this.availableCsvFields = _.filter(this.corpus?.fields, 'downloadable'); - const highlight = this.resultOverview?.highlight; - // 'Query in context' becomes an extra option if any field in the corpus has been marked as highlightable - if (highlight !== undefined) { - this.availableCsvFields.push({ - name: 'context', - description: `Query surrounded by ${highlight} characters`, - displayName: 'Query in context', - displayType: 'text_content', - csvCore: false, - hidden: false, - sortable: false, - searchable: false, - downloadable: true, - filterOptions: null, - mappingType: null, - } as unknown as CorpusField); + ngOnChanges(changes: SimpleChanges): void { + if (changes.corpus) { + this.availableCsvFields = _.filter(this.corpus?.fields, 'downloadable'); + this.selectedCsvFields = _.filter(this.corpus?.fields, 'csvCore'); + } + if (changes.queryModel) { + this.totalResults?.complete(); + this.resultsConfig?.complete(); + this.totalResults = new TotalResults( + new SimpleStore(), this.searchService, this.queryModel + ); + this.downloadDisabled$ = this.totalResults.result$.pipe( + map(result => result > 0) + ); + this.canDownloadDirectly$ = this.totalResults.result$.pipe( + map(this.enableDirectDownload.bind(this)) + ); + this.resultsConfig = new PageResults( + new SimpleStore(), this.searchService, this.queryModel + ); } } - /** - * called by download csv button. Large files are rendered in backend via Celery async task, - * and an email is sent with download link from backend - */ - public chooseDownloadMethod() { - if ( - this.resultOverview.resultsCount < this.directDownloadLimit || - this.downloadLimit === undefined - ) { - this.directDownload(); + onHighlightChange(event): void { + if (event.target.checked) { + this.resultsConfig.setParams({ highlight: DEFAULT_HIGHLIGHT_SIZE }); } else { - this.longDownload(); + this.resultsConfig.setParams({ highlight: null }); } } /** download short file directly */ - public confirmDirectDownload(options: DownloadOptions) { - const nDocuments = Math.min( - this.resultOverview.resultsCount, - this.directDownloadLimit - ); + public confirmDirectDownload(): void { + const sort = this.resultsConfig.state$.value.sort; + const highlight = this.resultsConfig.state$.value.highlight; this.isDownloading = true; this.downloadService .download( this.corpus, this.queryModel, - this.getCsvFields(), - nDocuments, - this.route, - this.resultOverview.sort, - this.resultOverview.highlight, - options + this.getColumnNames(), + this.directDownloadLimit, + this.resultsRoute(this.queryModel, sort, highlight), + sort, + highlight, + { encoding: this.encoding } ) .catch((error) => { this.notificationService.showMessage(error); @@ -110,25 +135,19 @@ export class DownloadComponent implements OnChanges { }); } - public selectCsvFields(selection: CorpusField[]) { - this.selectedCsvFields = selection; - } - - /** results can be downloaded directly: show menu to pick file options */ - private directDownload() { - this.pendingDownload = { download_type: 'search_results' }; - } /** start backend task to create csv file */ - private longDownload() { + longDownload(): void { + const sort = this.resultsConfig.state$.value.sort; + const highlight = this.resultsConfig.state$.value.highlight; this.downloadService .downloadTask( this.corpus, this.queryModel, - this.getCsvFields(), - this.route, - this.resultOverview.sort, - this.resultOverview.highlight + this.getColumnNames(), + this.resultsRoute(this.queryModel, sort, highlight), + sort, + highlight, ) .then((results) => { this.notificationService.showMessage( @@ -142,11 +161,40 @@ export class DownloadComponent implements OnChanges { }); } - private getCsvFields(): CorpusField[] { + private enableDirectDownload(totalResults: number): boolean { + const totalToDownload = _.min([totalResults, this.downloadLimit]); + return totalToDownload <= this.directDownloadLimit; + } + + private getColumnNames(): string[] { + let selectedFields: CorpusField[]; if (this.selectedCsvFields === undefined) { - return this.corpus.fields.filter((field) => field.csvCore); + selectedFields = this.corpus.fields.filter((field) => field.csvCore); } else { - return this.selectedCsvFields; + selectedFields = this.selectedCsvFields; + } + const selected = _.map(selectedFields, 'name'); + if (this.resultsConfig.state$.value.highlight) { + selected.push('context'); } + return selected; + } + + /** + * Generate URL to view these results in the web interface + */ + private resultsRoute( + queryModel: QueryModel, sort: SortState, highlight?: number + ): string { + const resultsParameters: PageResultsParameters = {sort, from: 0, size: 20, highlight }; + const queryParams = { + ...queryModel.toQueryParams(), + ...pageResultsParametersToParams(resultsParameters, queryModel.corpus) + }; + const tree = this.router.createUrlTree( + ['/search', queryModel.corpus.name], + { queryParams } + ); + return tree.toString(); } } diff --git a/frontend/src/app/download/download.module.ts b/frontend/src/app/download/download.module.ts index fe3de9d65..75b122f8a 100644 --- a/frontend/src/app/download/download.module.ts +++ b/frontend/src/app/download/download.module.ts @@ -1,10 +1,10 @@ import { NgModule } from '@angular/core'; import { DownloadComponent } from './download.component'; import { DownloadOptionsComponent } from './download-options/download-options.component'; -import { DownloadService } from '../services'; -import { SelectFieldComponent } from '../select-field/select-field.component'; +import { DownloadService } from '@services'; import { MultiSelectModule } from 'primeng/multiselect'; -import { SharedModule } from '../shared/shared.module'; +import { SharedModule } from '@shared/shared.module'; +import { ResultsSortModule } from '../search/results-sort/results-sort.module'; @@ -15,16 +15,15 @@ import { SharedModule } from '../shared/shared.module'; declarations: [ DownloadComponent, DownloadOptionsComponent, - SelectFieldComponent, ], imports: [ SharedModule, MultiSelectModule, + ResultsSortModule, ], exports: [ DownloadComponent, DownloadOptionsComponent, - SelectFieldComponent, ] }) export class DownloadModule { } diff --git a/frontend/src/app/filter/ad-hoc-filter/ad-hoc-filter.component.ts b/frontend/src/app/filter/ad-hoc-filter/ad-hoc-filter.component.ts index b79fccdb6..f09e09a71 100644 --- a/frontend/src/app/filter/ad-hoc-filter/ad-hoc-filter.component.ts +++ b/frontend/src/app/filter/ad-hoc-filter/ad-hoc-filter.component.ts @@ -1,5 +1,5 @@ import { Component } from '@angular/core'; -import { AdHocFilter, } from '../../models'; +import { AdHocFilter } from '@models'; import { BaseFilterComponent } from './../base-filter.component'; @Component({ diff --git a/frontend/src/app/filter/base-filter.component.ts b/frontend/src/app/filter/base-filter.component.ts index e7c0f7039..c114a75e1 100644 --- a/frontend/src/app/filter/base-filter.component.ts +++ b/frontend/src/app/filter/base-filter.component.ts @@ -1,9 +1,9 @@ import { Component, Input, OnChanges, OnDestroy, SimpleChanges } from '@angular/core'; import * as _ from 'lodash'; -import { QueryModel } from '../models/index'; +import { QueryModel } from '@models/index'; import { Subscription } from 'rxjs'; -import { FilterInterface } from '../models/base-filter'; +import { FilterInterface } from '@models/base-filter'; /** * Filter component receives the corpus fields containing search filters as input diff --git a/frontend/src/app/filter/boolean-filter/boolean-filter.component.spec.ts b/frontend/src/app/filter/boolean-filter/boolean-filter.component.spec.ts index 08ddbd437..674acab3d 100644 --- a/frontend/src/app/filter/boolean-filter/boolean-filter.component.spec.ts +++ b/frontend/src/app/filter/boolean-filter/boolean-filter.component.spec.ts @@ -2,7 +2,7 @@ import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing'; import { mockCorpus3, mockField } from '../../../mock-data/corpus'; import { commonTestBed } from '../../common-test-bed'; -import { BooleanFilter, QueryModel } from '../../models'; +import { BooleanFilter, QueryModel } from '@models'; import { BooleanFilterComponent } from './boolean-filter.component'; import { By } from '@angular/platform-browser'; diff --git a/frontend/src/app/filter/boolean-filter/boolean-filter.component.ts b/frontend/src/app/filter/boolean-filter/boolean-filter.component.ts index d67899f86..85271f3e6 100644 --- a/frontend/src/app/filter/boolean-filter/boolean-filter.component.ts +++ b/frontend/src/app/filter/boolean-filter/boolean-filter.component.ts @@ -1,7 +1,7 @@ import { Component } from '@angular/core'; import { BaseFilterComponent } from '../base-filter.component'; -import { BooleanFilter } from '../../models'; +import { BooleanFilter } from '@models'; @Component({ selector: 'ia-boolean-filter', diff --git a/frontend/src/app/filter/date-filter/date-filter.component.spec.ts b/frontend/src/app/filter/date-filter/date-filter.component.spec.ts index eb722b79c..c382f1316 100644 --- a/frontend/src/app/filter/date-filter/date-filter.component.spec.ts +++ b/frontend/src/app/filter/date-filter/date-filter.component.spec.ts @@ -2,11 +2,11 @@ import { ComponentFixture, TestBed, fakeAsync, flushMicrotasks, waitForAsync } f import { mockCorpus3, mockFieldDate } from '../../../mock-data/corpus'; import { commonTestBed } from '../../common-test-bed'; -import { DateFilter, DateFilterData, QueryModel } from '../../models'; +import { DateFilter, DateFilterData, QueryModel } from '@models'; import { DateFilterComponent } from './date-filter.component'; import { SimpleStore } from '../../store/simple-store'; -import { SearchService } from '../../services'; +import { SearchService } from '@services'; import * as _ from 'lodash'; describe('DateFilterComponent', () => { diff --git a/frontend/src/app/filter/date-filter/date-filter.component.ts b/frontend/src/app/filter/date-filter/date-filter.component.ts index 2476978d2..7acefa202 100644 --- a/frontend/src/app/filter/date-filter/date-filter.component.ts +++ b/frontend/src/app/filter/date-filter/date-filter.component.ts @@ -1,11 +1,15 @@ import { Component } from '@angular/core'; import * as _ from 'lodash'; -import { DateFilter, DateFilterData, QueryModel } from '../../models'; +import { DateFilter, DateFilterData, QueryModel } from '@models'; import { BaseFilterComponent } from '../base-filter.component'; import { BehaviorSubject, combineLatest } from 'rxjs'; -import { Aggregator, MaxDateAggregator, MinDateAggregator } from '../../models/aggregation'; -import { SearchService } from '../../services'; +import { + Aggregator, + MaxDateAggregator, + MinDateAggregator, +} from '@models/aggregation'; +import { SearchService } from '@services'; @Component({ selector: 'ia-date-filter', diff --git a/frontend/src/app/filter/filter-box/filter-box.component.ts b/frontend/src/app/filter/filter-box/filter-box.component.ts index 6bc5f12a0..75285863e 100644 --- a/frontend/src/app/filter/filter-box/filter-box.component.ts +++ b/frontend/src/app/filter/filter-box/filter-box.component.ts @@ -1,6 +1,6 @@ import { Component, Input } from '@angular/core'; -import { QueryModel, SearchFilter } from '../../models'; -import { filterIcons } from '../../shared/icons'; +import { QueryModel, SearchFilter } from '@models'; +import { filterIcons } from '@shared/icons'; @Component({ selector: 'ia-filter-box', diff --git a/frontend/src/app/filter/filter-manager.component.html b/frontend/src/app/filter/filter-manager.component.html index a12392cd1..41057bd2d 100644 --- a/frontend/src/app/filter/filter-manager.component.html +++ b/frontend/src/app/filter/filter-manager.component.html @@ -1,7 +1,7 @@
-

Filters

+

Filters

{ diff --git a/frontend/src/app/filter/filter-manager.component.ts b/frontend/src/app/filter/filter-manager.component.ts index 6a27ff61e..13bd13fcb 100644 --- a/frontend/src/app/filter/filter-manager.component.ts +++ b/frontend/src/app/filter/filter-manager.component.ts @@ -5,10 +5,10 @@ import * as _ from 'lodash'; import { combineLatest, Observable } from 'rxjs'; import { map } from 'rxjs/operators'; -import { FilterInterface, QueryModel } from '../models/index'; -import { filterIcons } from '../shared/icons'; -import { AuthService } from '../services/auth.service'; -import { isTagFilter } from '../models/tag-filter'; +import { FilterInterface, QueryModel } from '@models/index'; +import { filterIcons } from '@shared/icons'; +import { AuthService } from '@services/auth.service'; +import { isTagFilter } from '@models/tag-filter'; @Component({ selector: 'ia-filter-manager', diff --git a/frontend/src/app/filter/filter.module.ts b/frontend/src/app/filter/filter.module.ts index 3ef0a5517..7b8cf16ca 100644 --- a/frontend/src/app/filter/filter.module.ts +++ b/frontend/src/app/filter/filter.module.ts @@ -11,8 +11,8 @@ import { RangeFilterComponent, TagFilterComponent, } from '.'; -import { SearchService } from '../services'; -import { SharedModule } from '../shared/shared.module'; +import { SearchService } from '@services'; +import { SharedModule } from '@shared/shared.module'; import { FilterManagerComponent } from './filter-manager.component'; diff --git a/frontend/src/app/filter/multiple-choice-filter/multiple-choice-filter.component.spec.ts b/frontend/src/app/filter/multiple-choice-filter/multiple-choice-filter.component.spec.ts index 154cedf51..238915427 100644 --- a/frontend/src/app/filter/multiple-choice-filter/multiple-choice-filter.component.spec.ts +++ b/frontend/src/app/filter/multiple-choice-filter/multiple-choice-filter.component.spec.ts @@ -2,7 +2,7 @@ import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing'; import { mockCorpus, mockFieldMultipleChoice } from '../../../mock-data/corpus'; import { commonTestBed } from '../../common-test-bed'; -import { MultipleChoiceFilter, QueryModel } from '../../models'; +import { MultipleChoiceFilter, QueryModel } from '@models'; import { MultipleChoiceFilterComponent } from './multiple-choice-filter.component'; import * as _ from 'lodash'; diff --git a/frontend/src/app/filter/multiple-choice-filter/multiple-choice-filter.component.ts b/frontend/src/app/filter/multiple-choice-filter/multiple-choice-filter.component.ts index 43ebbb97e..1192643f8 100644 --- a/frontend/src/app/filter/multiple-choice-filter/multiple-choice-filter.component.ts +++ b/frontend/src/app/filter/multiple-choice-filter/multiple-choice-filter.component.ts @@ -2,10 +2,10 @@ import { Component } from '@angular/core'; import * as _ from 'lodash'; +import { TermsAggregator, TermsResult } from '@models/aggregation'; +import { SearchService } from '@services'; +import { MultipleChoiceFilter, MultipleChoiceFilterOptions } from '@models'; import { BaseFilterComponent } from '../base-filter.component'; -import { MultipleChoiceFilter, MultipleChoiceFilterOptions } from '../../models'; -import { SearchService } from '../../services'; -import { TermsAggregator, TermsResult } from '../../models/aggregation'; @Component({ selector: 'ia-multiple-choice-filter', diff --git a/frontend/src/app/filter/range-filter/range-filter.component.spec.ts b/frontend/src/app/filter/range-filter/range-filter.component.spec.ts index 33daeaec2..0a775e86f 100644 --- a/frontend/src/app/filter/range-filter/range-filter.component.spec.ts +++ b/frontend/src/app/filter/range-filter/range-filter.component.spec.ts @@ -2,12 +2,12 @@ import { ComponentFixture, TestBed, fakeAsync, flushMicrotasks, waitForAsync } f import { commonTestBed } from '../../common-test-bed'; -import { RangeFilterComponent } from './range-filter.component'; -import { QueryModel, RangeFilter } from '../../models'; -import { mockCorpus3, mockField3 } from '../../../mock-data/corpus'; -import { SearchService } from '../../services'; +import { SearchService } from '@services'; import * as _ from 'lodash'; +import { mockCorpus3, mockField3 } from '../../../mock-data/corpus'; +import { QueryModel, RangeFilter } from '@models'; import { SimpleStore } from '../../store/simple-store'; +import { RangeFilterComponent } from './range-filter.component'; describe('RangeFilterComponent', () => { let component: RangeFilterComponent; diff --git a/frontend/src/app/filter/range-filter/range-filter.component.ts b/frontend/src/app/filter/range-filter/range-filter.component.ts index 5e278ab1b..05dca4ed3 100644 --- a/frontend/src/app/filter/range-filter/range-filter.component.ts +++ b/frontend/src/app/filter/range-filter/range-filter.component.ts @@ -1,11 +1,11 @@ import { Component, OnDestroy, OnInit } from '@angular/core'; -import { RangeFilterData, RangeFilter, QueryModel } from '../../models'; +import { RangeFilterData, RangeFilter, QueryModel } from '@models'; import { BaseFilterComponent } from '../base-filter.component'; import { Subject, interval } from 'rxjs'; import { debounce, takeUntil } from 'rxjs/operators'; -import { Aggregator, MaxAggregator, MinAggregator } from '../../models/aggregation'; -import { SearchService } from '../../services'; +import { Aggregator, MaxAggregator, MinAggregator } from '@models/aggregation'; +import { SearchService } from '@services'; import * as _ from 'lodash'; @Component({ diff --git a/frontend/src/app/filter/tag-filter/tag-filter.component.ts b/frontend/src/app/filter/tag-filter/tag-filter.component.ts index 04fd961c7..7e59cb115 100644 --- a/frontend/src/app/filter/tag-filter/tag-filter.component.ts +++ b/frontend/src/app/filter/tag-filter/tag-filter.component.ts @@ -1,9 +1,9 @@ import { Component } from '@angular/core'; import { BaseFilterComponent } from '../base-filter.component'; -import { TagFilter } from '../../models/tag-filter'; -import { TagService } from '../../services/tag.service'; +import { TagFilter } from '@models/tag-filter'; +import { TagService } from '@services/tag.service'; import { Observable } from 'rxjs'; -import { Tag } from '../../models'; +import { Tag } from '@models'; @Component({ selector: 'ia-tag-filter', diff --git a/frontend/src/app/footer/footer.component.html b/frontend/src/app/footer/footer.component.html index 08fb8f0f9..a25951b2c 100644 --- a/frontend/src/app/footer/footer.component.html +++ b/frontend/src/app/footer/footer.component.html @@ -1,4 +1,4 @@ -

+
@@ -29,4 +29,4 @@

-
+
diff --git a/frontend/src/app/footer/footer.component.ts b/frontend/src/app/footer/footer.component.ts index 448bc3dce..66598884d 100644 --- a/frontend/src/app/footer/footer.component.ts +++ b/frontend/src/app/footer/footer.component.ts @@ -1,6 +1,6 @@ import { Component } from '@angular/core'; -import { environment } from '../../environments/environment'; +import { environment } from '@environments/environment'; @Component({ selector: 'ia-footer', diff --git a/frontend/src/app/forward-legacy-params.guard.spec.ts b/frontend/src/app/forward-legacy-params.guard.spec.ts new file mode 100644 index 000000000..9925b15c9 --- /dev/null +++ b/frontend/src/app/forward-legacy-params.guard.spec.ts @@ -0,0 +1,42 @@ +import { TestBed } from '@angular/core/testing'; +import { Router, RouterModule} from '@angular/router'; + +import { forwardLegacyParamsGuard } from './forward-legacy-params.guard'; +import { Component } from '@angular/core'; + +@Component({ + template: '

Hello world!

', +}) +class TestComponent {} + +describe('forwardLegacyParamsGuard', () => { + let router: Router; + const legacyUrl = '/search/my-corpus?query=test&compareTerm=test&compareTerm=testing&compareTerm=tester'; + const targetUrl = '/search/my-corpus?query=test&compareTerms=testing,tester'; + + beforeEach(() => { + TestBed.configureTestingModule({ + imports: [ + RouterModule.forRoot([ + { + path: 'search/:corpus', + component: TestComponent, + canActivate: [forwardLegacyParamsGuard], + } + ]) + ], + declarations: [TestComponent], + }); + router = TestBed.inject(Router); + }); + + it('should activate valid routes', async () => { + await router.navigateByUrl(targetUrl); + expect(router.url).toBe(targetUrl) + }); + + it('should forward legacy parameters', async () => { + await router.navigateByUrl(legacyUrl); + expect(router.url).toBe(targetUrl); + }); +}); diff --git a/frontend/src/app/forward-legacy-params.guard.ts b/frontend/src/app/forward-legacy-params.guard.ts new file mode 100644 index 000000000..e6bb7efcc --- /dev/null +++ b/frontend/src/app/forward-legacy-params.guard.ts @@ -0,0 +1,63 @@ +import { + ActivatedRouteSnapshot, CanActivateFn, createUrlTreeFromSnapshot, Params, + RouterStateSnapshot +} from '@angular/router'; +import * as _ from 'lodash'; +import { COMPARE_TERMS_KEY, DELIMITER } from './models/compared-queries'; + +const LEGACY_COMPARE_TERMS_PARAM = 'compareTerm'; + +/** + * Forwards URLs that include outdated query parameters. + * + * This forwards URLs with the outdated `compareTerm` parameter. (May be expanded with + * other parameters in the future.) Allows old links/bookmarks to keep functioning. + * + * For example: + * `?query=a&compareTerm=a&compareTerm=b&compareTerm=c` + * will be redirected to + * `?query=a&compareTerms=b,c` + */ +export const forwardLegacyParamsGuard: CanActivateFn = ( + route: ActivatedRouteSnapshot, + state: RouterStateSnapshot +) => { + if (hasLegacyParams(route)) { + const url = createUrlTreeFromSnapshot(route, ['.']); + // include updated query parameters + url.queryParams = updatedCompareTermsParams(route); + // preserve fragment + url.fragment = route.fragment; + return url; + } + return true; +}; + +const hasLegacyParams = (route: ActivatedRouteSnapshot): boolean => { + return route.queryParamMap.has(LEGACY_COMPARE_TERMS_PARAM); +} + +/** + * Returns updated version of the query parameters in a URL. + * + * Replaces legacy `compareTerm` with `compareTerms` parameter. Returns the Params object + * with updated values. + */ +const updatedCompareTermsParams = (route: ActivatedRouteSnapshot): Params => { + const comparedTerms = legacyComparedTerms(route); + const params = _.omit(route.queryParams, [LEGACY_COMPARE_TERMS_PARAM]); + params[COMPARE_TERMS_KEY] = comparedTerms.join(DELIMITER); + return params; +} + +/** The compared terms in a URL in legacy format + * + * Example of legacy parameters: `?query=a&compareTerm=a&compareTerm=b&compareTerm=c` + * + * This would return `['b', 'c']` as the compared terms. + */ +const legacyComparedTerms = (route: ActivatedRouteSnapshot): string[] => { + const query = route.queryParamMap.get('query'); + const allTerms = route.queryParamMap.getAll(LEGACY_COMPARE_TERMS_PARAM); + return _.without(allTerms, query); +} diff --git a/frontend/src/app/history/download-history/download-history.component.ts b/frontend/src/app/history/download-history/download-history.component.ts index eadc44b8b..3d1e704b6 100644 --- a/frontend/src/app/history/download-history/download-history.component.ts +++ b/frontend/src/app/history/download-history/download-history.component.ts @@ -1,13 +1,27 @@ import { Component, OnInit } from '@angular/core'; import * as _ from 'lodash'; -import { Download, DownloadOptions, DownloadParameters, DownloadType, QueryModel } from '../../models'; -import { ApiService, CorpusService, DownloadService, NotificationService } from '../../services'; +import { + Download, + DownloadOptions, + DownloadParameters, + DownloadType, + QueryModel, +} from '@models'; +import { + ApiService, + CorpusService, + DownloadService, + NotificationService, +} from '@services'; import { HistoryDirective } from '../history.directive'; -import { findByName } from '../../utils/utils'; -import { actionIcons } from '../../shared/icons'; -import { downloadQueryModel, downloadQueryModels } from '../../utils/download-history'; +import { findByName } from '@utils/utils'; +import { actionIcons } from '@shared/icons'; +import { + downloadQueryModel, + downloadQueryModels, +} from '@utils/download-history'; import { Title } from '@angular/platform-browser'; -import { pageTitle } from '../../utils/app'; +import { pageTitle } from '@utils/app'; @Component({ selector: 'ia-download-history', @@ -72,8 +86,8 @@ export class DownloadHistoryComponent extends HistoryDirective implements OnInit parameters.fields : [parameters[0].field_name]; const corpus = findByName(this.corpora, download.corpus); const fields = fieldNames.map(fieldName => - findByName(corpus.fields, fieldName).displayName - ); + findByName(corpus.fields, fieldName)?.displayName + ).filter(_.negate(_.isUndefined)); return _.join(fields, ', '); } diff --git a/frontend/src/app/history/history.directive.ts b/frontend/src/app/history/history.directive.ts index deae409f2..541894ccc 100644 --- a/frontend/src/app/history/history.directive.ts +++ b/frontend/src/app/history/history.directive.ts @@ -1,8 +1,8 @@ import { Directive } from '@angular/core'; import { MenuItem } from 'primeng/api'; -import { Corpus, Download, QueryDb } from '../models'; -import { CorpusService } from '../services'; -import { findByName } from '../utils/utils'; +import { Corpus, Download, QueryDb } from '@models'; +import { CorpusService } from '@services'; +import { findByName } from '@utils/utils'; @Directive({ selector: '[iaHistory]' diff --git a/frontend/src/app/history/history.module.ts b/frontend/src/app/history/history.module.ts index 9111b1c78..6c9cd274e 100644 --- a/frontend/src/app/history/history.module.ts +++ b/frontend/src/app/history/history.module.ts @@ -1,8 +1,8 @@ import { NgModule } from '@angular/core'; import { DropdownModule } from 'primeng/dropdown'; import { DownloadModule } from '../download/download.module'; -import { DownloadService, QueryService } from '../services'; -import { SharedModule } from '../shared/shared.module'; +import { DownloadService, QueryService } from '@services'; +import { SharedModule } from '@shared/shared.module'; import { DownloadHistoryComponent } from './download-history/download-history.component'; import { QueryFiltersComponent, diff --git a/frontend/src/app/history/search-history/query-filters.component.spec.ts b/frontend/src/app/history/search-history/query-filters.component.spec.ts index 5acccd124..d01261a7a 100644 --- a/frontend/src/app/history/search-history/query-filters.component.spec.ts +++ b/frontend/src/app/history/search-history/query-filters.component.spec.ts @@ -1,5 +1,5 @@ import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing'; -import { QueryModel } from '../../models'; +import { QueryModel } from '@models'; import { mockCorpus } from '../../../mock-data/corpus'; import { commonTestBed } from '../../common-test-bed'; diff --git a/frontend/src/app/history/search-history/query-filters.component.ts b/frontend/src/app/history/search-history/query-filters.component.ts index a60bd415c..a350a10b6 100644 --- a/frontend/src/app/history/search-history/query-filters.component.ts +++ b/frontend/src/app/history/search-history/query-filters.component.ts @@ -1,6 +1,6 @@ import { Component, Input, OnInit } from '@angular/core'; -import { QueryModel } from '../../models/index'; +import { QueryModel } from '@models/index'; @Component({ selector: '[ia-query-filters]', diff --git a/frontend/src/app/history/search-history/query-text.pipe.ts b/frontend/src/app/history/search-history/query-text.pipe.ts index cb741cd8e..57f18af25 100644 --- a/frontend/src/app/history/search-history/query-text.pipe.ts +++ b/frontend/src/app/history/search-history/query-text.pipe.ts @@ -1,5 +1,5 @@ import { Pipe, PipeTransform } from '@angular/core'; -import { QueryModel } from '../../models'; +import { QueryModel } from '@models'; @Pipe({name: 'formatQueryText'}) export class QueryTextPipe implements PipeTransform { diff --git a/frontend/src/app/history/search-history/search-history.component.ts b/frontend/src/app/history/search-history/search-history.component.ts index 62d1a8817..523a88a05 100644 --- a/frontend/src/app/history/search-history/search-history.component.ts +++ b/frontend/src/app/history/search-history/search-history.component.ts @@ -1,14 +1,14 @@ import { Component, OnInit } from '@angular/core'; import { Params } from '@angular/router'; import * as _ from 'lodash'; -import { apiQueryToQueryModel } from '../../utils/es-query'; -import { QueryDb } from '../../models/index'; -import { CorpusService, QueryService } from '../../services/index'; +import { apiQueryToQueryModel } from '@utils/es-query'; +import { QueryDb } from '@models/index'; +import { CorpusService, QueryService } from '@services/index'; import { HistoryDirective } from '../history.directive'; -import { findByName } from '../../utils/utils'; -import { actionIcons } from '../../shared/icons'; +import { findByName } from '@utils/utils'; +import { actionIcons } from '@shared/icons'; import { Title } from '@angular/platform-browser'; -import { pageTitle } from '../../utils/app'; +import { pageTitle } from '@utils/app'; @Component({ selector: 'ia-search-history', diff --git a/frontend/src/app/home/home.component.ts b/frontend/src/app/home/home.component.ts index b52d53e7e..00d432322 100644 --- a/frontend/src/app/home/home.component.ts +++ b/frontend/src/app/home/home.component.ts @@ -1,10 +1,10 @@ import { Component, OnInit } from '@angular/core'; import { Title } from '@angular/platform-browser'; import { BehaviorSubject } from 'rxjs'; -import { Corpus } from '../models/corpus'; -import { CorpusService } from '../services/index'; -import { showLoading } from '../utils/utils'; -import { environment } from '../../environments/environment'; +import { Corpus } from '@models/corpus'; +import { CorpusService } from '@services/index'; +import { showLoading } from '@utils/utils'; +import { environment } from '@environments/environment'; @Component({ selector: 'ia-home', diff --git a/frontend/src/app/image-view/image-navigation.component.html b/frontend/src/app/image-view/image-navigation.component.html index be484e946..e84dfafe2 100644 --- a/frontend/src/app/image-view/image-navigation.component.html +++ b/frontend/src/app/image-view/image-navigation.component.html @@ -4,9 +4,10 @@ - + - -
  • - -
    • -
  • - +
  • +
  • … diff --git a/frontend/src/app/search/pagination/pagination.component.ts b/frontend/src/app/search/pagination/pagination.component.ts index 791a3ac1e..390b9daab 100644 --- a/frontend/src/app/search/pagination/pagination.component.ts +++ b/frontend/src/app/search/pagination/pagination.component.ts @@ -1,6 +1,6 @@ import { Component, EventEmitter, Input, OnChanges, Output } from '@angular/core'; -import { PageResultsParameters } from '../../models/page-results'; +import { PageResultsParameters } from '@models/page-results'; @Component({ selector: 'ia-pagination', diff --git a/frontend/src/app/search/results-sort/results-sort.module.ts b/frontend/src/app/search/results-sort/results-sort.module.ts new file mode 100644 index 000000000..2e22e6409 --- /dev/null +++ b/frontend/src/app/search/results-sort/results-sort.module.ts @@ -0,0 +1,18 @@ +import { NgModule } from '@angular/core'; +import { SearchSortingComponent } from './search-sorting.component'; +import { SharedModule } from '@shared/shared.module'; + + + +@NgModule({ + declarations: [ + SearchSortingComponent, + ], + imports: [ + SharedModule + ], + exports: [ + SearchSortingComponent, + ] +}) +export class ResultsSortModule { } diff --git a/frontend/src/app/search/search-sorting.component.html b/frontend/src/app/search/results-sort/search-sorting.component.html similarity index 87% rename from frontend/src/app/search/search-sorting.component.html rename to frontend/src/app/search/results-sort/search-sorting.component.html index 7fce81798..d1f2045ac 100644 --- a/frontend/src/app/search/search-sorting.component.html +++ b/frontend/src/app/search/results-sort/search-sorting.component.html @@ -11,7 +11,7 @@
    • - -
    -
    - -
    -
    - -
    -
    - - -
    -
    - - - - - +
    +
    +
    + +
    +
    + +
    +
    + +
    +
    + + +
    - +
    @@ -53,14 +42,19 @@
    - + - + - + + + + + +
    diff --git a/frontend/src/app/search/search.component.ts b/frontend/src/app/search/search.component.ts index 6f813421b..a4cae006b 100644 --- a/frontend/src/app/search/search.component.ts +++ b/frontend/src/app/search/search.component.ts @@ -1,23 +1,23 @@ -import { Component, ElementRef, HostListener, ViewChild } from '@angular/core'; -import { ActivatedRoute, ParamMap, Router } from '@angular/router'; +import { Component, ElementRef, HostListener, OnDestroy, OnInit, ViewChild } from '@angular/core'; import * as _ from 'lodash'; import { Subscription } from 'rxjs'; -import { Corpus, CorpusField, ResultOverview, QueryModel, User } from '../models/index'; -import { CorpusService, DialogService, ParamService, } from '../services/index'; -import { ParamDirective } from '../param/param-directive'; -import { AuthService } from '../services/auth.service'; +import { Corpus, CorpusField, QueryModel, User } from '@models/index'; +import { CorpusService, DialogService } from '@services/index'; + +import { AuthService } from '@services/auth.service'; import { distinct, filter } from 'rxjs/operators'; -import { actionIcons, searchIcons } from '../shared/icons'; +import { actionIcons, searchIcons } from '@shared/icons'; import { RouterStoreService } from '../store/router-store.service'; import { Title } from '@angular/platform-browser'; +import { SearchTab, SearchTabs } from './search-tabs'; @Component({ selector: 'ia-search', templateUrl: './search.component.html', styleUrls: ['./search.component.scss'], }) -export class SearchComponent extends ParamDirective { +export class SearchComponent implements OnInit, OnDestroy { @ViewChild('searchSection', { static: false }) public searchSection: ElementRef; @@ -25,37 +25,23 @@ export class SearchComponent extends ParamDirective { public corpus: Corpus; - /** - * The filters have been modified. - */ - public isSearching: boolean; - public hasSearched: boolean; - /** - * Whether the total number of hits exceeds the download limit. - */ - public hasLimitedResults = false; - public user: User; searchIcons = searchIcons; actionIcons = actionIcons; - activeTab: string; - public queryModel: QueryModel; /** * This is the query text currently entered in the interface. */ public queryText: string; - resultOverview: ResultOverview; - public filterFields: CorpusField[] = []; - public showVisualization: boolean; - public nullableParameters = []; + tabs: SearchTabs; + protected corpusSubscription: Subscription; /** @@ -68,14 +54,10 @@ export class SearchComponent extends ParamDirective { private authService: AuthService, private corpusService: CorpusService, private dialogService: DialogService, - paramService: ParamService, - route: ActivatedRoute, - router: Router, private routerStoreService: RouterStoreService, private title: Title, ) { - super(route, router, paramService); - + this.tabs = new SearchTabs(this.routerStoreService); } @HostListener('window:scroll', []) @@ -85,8 +67,8 @@ export class SearchComponent extends ParamDirective { this.searchSection.nativeElement.getBoundingClientRect().y === 0; } - async initialize(): Promise { - this.user = await this.authService.getCurrentUserPromise(); + ngOnInit() { + this.authService.getCurrentUserPromise().then(user => this.user = user); this.corpusSubscription = this.corpusService.currentCorpus .pipe( filter((corpus) => !!corpus), @@ -102,29 +84,12 @@ export class SearchComponent extends ParamDirective { } } - teardown() { + ngOnDestroy() { this.user = undefined; this.corpusSubscription.unsubscribe(); this.queryModel.complete(); } - setStateFromParams(params: ParamMap) { - this.showVisualization = params.has('visualize') ? true : false; - } - - /** - * Event triggered from search-results.component - * - * @param input - */ - public onSearched(input: ResultOverview) { - this.isSearching = false; - this.hasSearched = true; - this.resultOverview = input; - this.hasLimitedResults = - this.user? input.resultsCount > this.user.downloadLimit : true; - } - public showQueryDocumentation() { this.dialogService.showManualPage('query'); } @@ -133,6 +98,10 @@ export class SearchComponent extends ParamDirective { this.queryModel.setQueryText(this.queryText); } + onTabChange(tab: SearchTab) { + this.tabs.setParams({tab}); + } + private setCorpus(corpus: Corpus) { this.corpus = corpus; this.setQueryModel(); diff --git a/frontend/src/app/search/search.module.ts b/frontend/src/app/search/search.module.ts index b1f72d613..3620f63ee 100644 --- a/frontend/src/app/search/search.module.ts +++ b/frontend/src/app/search/search.module.ts @@ -1,16 +1,18 @@ import { NgModule } from '@angular/core'; -import { SharedModule } from '../shared/shared.module'; +import { SharedModule } from '@shared/shared.module'; import { PaginationComponent } from './pagination/pagination.component'; import { HighlightSelectorComponent } from './highlight-selector.component'; import { SearchResultsComponent } from './search-results.component'; import { SearchComponent } from './search.component'; import { DocumentModule } from '../document/document.module'; import { CorpusModule } from '../corpus-header/corpus.module'; -import { SearchSortingComponent } from './search-sorting.component'; import { FilterModule } from '../filter/filter.module'; import { DownloadModule } from '../download/download.module'; -import { QueryService, SearchService } from '../services'; +import { QueryService, SearchService } from '@services'; import { VisualizationModule } from '../visualization/visualization.module'; +import { ResultsSortModule } from './results-sort/results-sort.module'; +import { SelectFieldComponent } from '../select-field/select-field.component'; +import { MultiSelectModule } from 'primeng/multiselect'; @@ -24,7 +26,7 @@ import { VisualizationModule } from '../visualization/visualization.module'; PaginationComponent, SearchComponent, SearchResultsComponent, - SearchSortingComponent, + SelectFieldComponent, ], imports: [ CorpusModule, @@ -33,6 +35,8 @@ import { VisualizationModule } from '../visualization/visualization.module'; FilterModule, SharedModule, VisualizationModule, + ResultsSortModule, + MultiSelectModule, ], exports: [ SearchComponent, diff --git a/frontend/src/app/select-field/select-field.component.ts b/frontend/src/app/select-field/select-field.component.ts index 67b21bbde..a42610243 100644 --- a/frontend/src/app/select-field/select-field.component.ts +++ b/frontend/src/app/select-field/select-field.component.ts @@ -1,8 +1,8 @@ /* eslint-disable @typescript-eslint/member-ordering */ import * as _ from 'lodash'; import { Component, EventEmitter, Input, OnChanges, Output } from '@angular/core'; -import { CorpusField, QueryModel } from '../models/index'; -import { actionIcons } from '../shared/icons'; +import { CorpusField, QueryModel } from '@models/index'; +import { actionIcons } from '@shared/icons'; @Component({ selector: 'ia-select-field', diff --git a/frontend/src/app/services/api.service.spec.ts b/frontend/src/app/services/api.service.spec.ts index b242bae34..09b58fba1 100644 --- a/frontend/src/app/services/api.service.spec.ts +++ b/frontend/src/app/services/api.service.spec.ts @@ -5,7 +5,7 @@ import { ApiService } from './api.service'; import { HttpClient, HttpErrorResponse } from '@angular/common/http'; import { fakeNgramResult } from '../../mock-data/api'; import { Subject, from, throwError } from 'rxjs'; -import { TaskResult, TasksOutcome } from '../models'; +import { TaskResult, TasksOutcome } from '@models'; describe('ApiService', () => { let service: ApiService; diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index b05bae0e7..28a2e33d0 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -16,6 +16,7 @@ import { FieldCoverage, FoundDocument, GeoDocument, + GeoLocation, LimitedResultsDownloadParameters, MostFrequentWordsResult, NGramRequestParameters, @@ -28,10 +29,13 @@ import { UserResponse, UserRole, WordcloudParameters, -} from '../models/index'; -import { environment } from '../../environments/environment'; +} from '@models/index'; +import { environment } from '@environments/environment'; import * as _ from 'lodash'; -import { APICorpusDefinition, APIEditableCorpus } from '../models/corpus-definition'; +import { + APICorpusDefinition, + APIEditableCorpus, +} from '@models/corpus-definition'; interface SolisLoginResponse { success: boolean; @@ -148,6 +152,11 @@ export class ApiService { return this.http.post(url, data).toPromise(); } + public geoCentroid(data: {corpus: string, field: string}): Promise { + const url = this.apiRoute(this.visApiURL, 'geo_centroid'); + return this.http.post(url, data).toPromise(); + } + public ngramTasks(data: NGramRequestParameters): Promise { const url = this.apiRoute(this.visApiURL, 'ngram'); return this.http.post(url, data).toPromise(); @@ -223,12 +232,15 @@ export class ApiService { } // Corpus - public corpusDocumentation(corpusName: string): Observable { - const url = this.apiRoute( - this.corpusApiUrl, - `documentation/${corpusName}/` - ); - return this.http.get(url); + public corpusDocumentationPages(corpus?: Corpus): Observable { + const params = new URLSearchParams({corpus: corpus.name}).toString(); + const url = this.apiRoute(this.corpusApiUrl, `documentation/?${params}`); + return this.http.get(url.toString()); + } + + public corpusDocumentationPage(pageID: number): Observable { + const url = this.apiRoute(this.corpusApiUrl, `documentation/${pageID}/`); + return this.http.get(url); } public corpus() { diff --git a/frontend/src/app/services/auth.service.ts b/frontend/src/app/services/auth.service.ts index 7675fed2e..ef70c3df6 100644 --- a/frontend/src/app/services/auth.service.ts +++ b/frontend/src/app/services/auth.service.ts @@ -13,12 +13,12 @@ import { distinctUntilChanged, mergeMap, takeUntil, tap } from 'rxjs/operators'; -import { environment } from '../../environments/environment'; -import { User, UserResponse } from '../models'; +import { environment } from '@environments/environment'; +import { User, UserResponse } from '@models'; import { ApiService } from './api.service'; import { SessionService } from './session.service'; import * as _ from 'lodash'; -import { encodeUserData, parseUserData } from '../utils/user'; +import { encodeUserData, parseUserData } from '@utils/user'; @Injectable({ providedIn: 'root', diff --git a/frontend/src/app/services/corpus.service.ts b/frontend/src/app/services/corpus.service.ts index 6c4e7ff6d..96296a223 100644 --- a/frontend/src/app/services/corpus.service.ts +++ b/frontend/src/app/services/corpus.service.ts @@ -3,10 +3,15 @@ import { Injectable } from '@angular/core'; import { BehaviorSubject } from 'rxjs'; -import { Corpus, CorpusField, DocumentContext, SortDirection, SortState } from '../models/index'; +import { + Corpus, + CorpusField, + DocumentContext, + SortDirection, + SortState, +} from '@models/index'; import { ApiRetryService } from './api-retry.service'; -import { AuthService } from './auth.service'; -import { findByName } from '../utils/utils'; +import { findByName } from '@utils/utils'; import * as _ from 'lodash'; @Injectable({ @@ -23,7 +28,6 @@ export class CorpusService { constructor( private apiRetryService: ApiRetryService, - private authService: AuthService ) { this.parseField = this.parseField.bind(this); } diff --git a/frontend/src/app/services/download.service.spec.ts b/frontend/src/app/services/download.service.spec.ts index 308c65640..4f726ffc6 100644 --- a/frontend/src/app/services/download.service.spec.ts +++ b/frontend/src/app/services/download.service.spec.ts @@ -4,7 +4,12 @@ import { ApiService } from './api.service'; import { ApiServiceMock } from '../../mock-data/api'; import { DownloadService } from './download.service'; import { mockCorpus, mockField } from '../../mock-data/corpus'; -import { DownloadOptions, LimitedResultsDownloadParameters, QueryModel, SortState } from '../models'; +import { + DownloadOptions, + LimitedResultsDownloadParameters, + QueryModel, + SortState, +} from '@models'; describe('DownloadService', () => { let apiService: ApiService; @@ -41,7 +46,8 @@ describe('DownloadService', () => { }; spyOn(apiService, 'download').and.returnValue(Promise.resolve({})); - service.download(query.corpus, query, query.corpus.fields, size, route, sort, highlight, options); + const fieldNames = query.corpus.fields.map(field => field.name) + service.download(query.corpus, query, fieldNames, size, route, sort, highlight, options); const expectedBody: LimitedResultsDownloadParameters = { corpus: mockCorpus.name, fields: ['great_field', 'speech'], @@ -62,8 +68,8 @@ describe('DownloadService', () => { sort: [{ great_field: 'desc' }], highlight: { fragment_size: highlight, - pre_tags: [''], - post_tags: [''], + pre_tags: [''], + post_tags: [''], order: 'score', fields: [{ speech: {} }], }, diff --git a/frontend/src/app/services/download.service.ts b/frontend/src/app/services/download.service.ts index 59ece2cb2..3b0565c20 100644 --- a/frontend/src/app/services/download.service.ts +++ b/frontend/src/app/services/download.service.ts @@ -2,10 +2,17 @@ import { Injectable } from '@angular/core'; import { saveAs } from 'file-saver'; import { ApiService } from './api.service'; -import { Corpus, CorpusField, DownloadOptions, LimitedResultsDownloadParameters, QueryModel, SortState } from '../models/index'; +import { + Corpus, + CorpusField, + DownloadOptions, + LimitedResultsDownloadParameters, + QueryModel, + SortState, +} from '@models/index'; import * as _ from 'lodash'; -import { resultsParamsToAPIQuery } from '../utils/es-query'; -import { PageResultsParameters } from '../models/page-results'; +import { resultsParamsToAPIQuery } from '@utils/es-query'; +import { PageResultsParameters } from '@models/page-results'; @Injectable() export class DownloadService { @@ -19,7 +26,7 @@ export class DownloadService { public async download( corpus: Corpus, queryModel: QueryModel, - fields: CorpusField[], + fieldNames: string[], requestedResults: number, route: string, sort: SortState, @@ -38,7 +45,7 @@ export class DownloadService { { ...query, corpus: corpus.name, - fields: fields.map((field) => field.name), + fields: fieldNames, route, }, fileOptions @@ -68,14 +75,13 @@ export class DownloadService { public async downloadTask( corpus: Corpus, queryModel: QueryModel, - fields: - CorpusField[], + fields: string[], route: string, sort: SortState, highlightFragmentSize: number ) { const query = queryModel.toAPIQuery(); - return this.apiService.downloadTask({ corpus: corpus.name, ...query, fields: fields.map(field => field.name), route }) + return this.apiService.downloadTask({ corpus: corpus.name, ...query, fields, route }) .then(result => result) .catch(error => { throw new Error(error.headers.message[0]); diff --git a/frontend/src/app/services/elastic-search.service.spec.ts b/frontend/src/app/services/elastic-search.service.spec.ts index e1ace87e8..4a378ef49 100644 --- a/frontend/src/app/services/elastic-search.service.spec.ts +++ b/frontend/src/app/services/elastic-search.service.spec.ts @@ -1,13 +1,13 @@ import { TestBed } from '@angular/core/testing'; import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing'; import { ElasticSearchService, SearchResponse } from './elastic-search.service'; -import { QueryModel } from '../models'; +import { QueryModel } from '@models'; import { mockCorpus, mockField, mockField2 } from '../../mock-data/corpus'; import { EntityService } from './entity.service'; import { EntityServiceMock } from '../../mock-data/entity'; import { TagServiceMock } from '../../mock-data/tag'; import { TagService } from './tag.service'; -import { TermsAggregator } from '../models/aggregation'; +import { TermsAggregator } from '@models/aggregation'; const mockResponse: SearchResponse = { diff --git a/frontend/src/app/services/elastic-search.service.ts b/frontend/src/app/services/elastic-search.service.ts index defd2d261..0f6348b92 100644 --- a/frontend/src/app/services/elastic-search.service.ts +++ b/frontend/src/app/services/elastic-search.service.ts @@ -3,15 +3,18 @@ import { Injectable } from '@angular/core'; import { HttpClient } from '@angular/common/http'; import { - FoundDocument, Corpus, QueryModel, SearchResults, - SearchHit -} from '../models/index'; -import { Aggregator } from '../models/aggregation'; + FoundDocument, + Corpus, + QueryModel, + SearchResults, + SearchHit, +} from '@models/index'; +import { Aggregator } from '@models/aggregation'; import * as _ from 'lodash'; import { TagService } from './tag.service'; -import { APIQuery } from '../models/search-requests'; -import { PageResultsParameters } from '../models/page-results'; -import { resultsParamsToAPIQuery } from '../utils/es-query'; +import { APIQuery } from '@models/search-requests'; +import { PageResultsParameters } from '@models/page-results'; +import { resultsParamsToAPIQuery } from '@utils/es-query'; import { EntityService } from './entity.service'; diff --git a/frontend/src/app/services/entity.service.ts b/frontend/src/app/services/entity.service.ts index 056e1e543..892925961 100644 --- a/frontend/src/app/services/entity.service.ts +++ b/frontend/src/app/services/entity.service.ts @@ -2,7 +2,7 @@ import { HttpClient } from '@angular/common/http'; import { Injectable } from '@angular/core'; import { Observable } from 'rxjs'; -import { Corpus, NamedEntitiesResult } from '../models'; +import { Corpus, NamedEntitiesResult } from '@models'; @Injectable({ providedIn: 'root', diff --git a/frontend/src/app/services/query.service.ts b/frontend/src/app/services/query.service.ts index 624e53236..5ab3737f8 100644 --- a/frontend/src/app/services/query.service.ts +++ b/frontend/src/app/services/query.service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@angular/core'; import { ApiRetryService } from './api-retry.service'; -import { QueryDb } from '../models/query'; +import { QueryDb } from '@models/query'; import * as _ from 'lodash'; @Injectable() diff --git a/frontend/src/app/services/search.service.spec.ts b/frontend/src/app/services/search.service.spec.ts index 1facbb5f3..662dbfd98 100644 --- a/frontend/src/app/services/search.service.spec.ts +++ b/frontend/src/app/services/search.service.spec.ts @@ -10,7 +10,7 @@ import { QueryService } from './query.service'; import { SearchService } from './search.service'; import { SessionService } from './session.service'; import { HttpClientTestingModule } from '@angular/common/http/testing'; -import { QueryModel } from '../models'; +import { QueryModel } from '@models'; import { mockCorpus } from '../../mock-data/corpus'; import { AuthService } from './auth.service'; import { AuthServiceMock } from '../../mock-data/auth'; diff --git a/frontend/src/app/services/search.service.ts b/frontend/src/app/services/search.service.ts index e45eaffba..591440bfd 100644 --- a/frontend/src/app/services/search.service.ts +++ b/frontend/src/app/services/search.service.ts @@ -2,11 +2,9 @@ import { Injectable } from '@angular/core'; import { ApiService } from './api.service'; import { ElasticSearchService } from './elastic-search.service'; -import { - Corpus, QueryModel, SearchResults, -} from '../models/index'; -import { PageResultsParameters } from '../models/page-results'; -import { Aggregator } from '../models/aggregation'; +import { Corpus, QueryModel, SearchResults } from '@models/index'; +import { PageResultsParameters } from '@models/page-results'; +import { Aggregator } from '@models/aggregation'; @Injectable() diff --git a/frontend/src/app/services/tag.service.ts b/frontend/src/app/services/tag.service.ts index 48222ee43..b6de1469d 100644 --- a/frontend/src/app/services/tag.service.ts +++ b/frontend/src/app/services/tag.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@angular/core'; import { pick } from 'lodash'; import { BehaviorSubject, Observable, of } from 'rxjs'; import { map, tap } from 'rxjs/operators'; -import { FoundDocument, Tag } from '../models'; +import { FoundDocument, Tag } from '@models'; import { ApiService } from './api.service'; import { AuthService } from './auth.service'; diff --git a/frontend/src/app/services/visualization.service.ts b/frontend/src/app/services/visualization.service.ts index 2d7119910..51f4b042c 100644 --- a/frontend/src/app/services/visualization.service.ts +++ b/frontend/src/app/services/visualization.service.ts @@ -4,13 +4,14 @@ import { Corpus, DateTermFrequencyParameters, GeoDocument, + GeoLocation, MostFrequentWordsResult, NGramRequestParameters, NgramParameters, QueryModel, TaskResult, TimeCategory, -} from '../models'; +} from '@models'; import { ApiService } from './api.service'; import { Observable } from 'rxjs'; @@ -46,6 +47,14 @@ export class VisualizationService { }); } + public async getGeoCentroid(fieldName: string, corpus: Corpus): + Promise { + return this.apiService.geoCentroid({ + corpus: corpus.name, + field: fieldName, + }); +} + public makeAggregateTermFrequencyParameters( corpus: Corpus, queryModel: QueryModel, fieldName: string, bins: {fieldValue: string|number; size: number}[], ): AggregateTermFrequencyParameters { diff --git a/frontend/src/app/services/wordmodels.service.ts b/frontend/src/app/services/wordmodels.service.ts index 3fa3b03ac..f5ac4178c 100644 --- a/frontend/src/app/services/wordmodels.service.ts +++ b/frontend/src/app/services/wordmodels.service.ts @@ -5,7 +5,7 @@ import { RelatedWordsResults, WordInModelResult, WordSimilarity, -} from '../models'; +} from '@models'; @Injectable() export class WordmodelsService { diff --git a/frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.ts b/frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.ts index d332b8d9c..f3a8760fb 100644 --- a/frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.ts +++ b/frontend/src/app/settings/search-history-settings/delete-search-history/delete-search-history.component.ts @@ -1,7 +1,7 @@ import { Component, OnInit } from '@angular/core'; -import { ApiService, NotificationService } from '../../../services'; +import { ApiService, NotificationService } from '@services'; import { tap } from 'rxjs/operators'; -import { actionIcons } from '../../../shared/icons'; +import { actionIcons } from '@shared/icons'; @Component({ selector: 'ia-delete-search-history', diff --git a/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.ts b/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.ts index c6e6d1f7e..73ec5825b 100644 --- a/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.ts +++ b/frontend/src/app/settings/search-history-settings/toggle-search-history/toggle-search-history.component.ts @@ -1,5 +1,5 @@ import { Component } from '@angular/core'; -import { AuthService, NotificationService } from '../../../services'; +import { AuthService, NotificationService } from '@services'; import { map } from 'rxjs/operators'; import { Observable } from 'rxjs'; diff --git a/frontend/src/app/settings/settings.component.ts b/frontend/src/app/settings/settings.component.ts index 48f1cf33f..8ceaf1476 100644 --- a/frontend/src/app/settings/settings.component.ts +++ b/frontend/src/app/settings/settings.component.ts @@ -1,6 +1,6 @@ import { Component, OnInit } from '@angular/core'; import { Title } from '@angular/platform-browser'; -import { pageTitle } from '../utils/app'; +import { pageTitle } from '@utils/app'; @Component({ selector: 'ia-settings', diff --git a/frontend/src/app/settings/settings.module.ts b/frontend/src/app/settings/settings.module.ts index 6f1f833e3..0ba0c05b8 100644 --- a/frontend/src/app/settings/settings.module.ts +++ b/frontend/src/app/settings/settings.module.ts @@ -1,5 +1,5 @@ import { NgModule } from '@angular/core'; -import { SharedModule } from '../shared/shared.module'; +import { SharedModule } from '@shared/shared.module'; import { SettingsComponent } from './settings.component'; import { SearchHistorySettingsComponent } from './search-history-settings/search-history-settings.component'; import { DeleteSearchHistoryComponent } from './search-history-settings/delete-search-history/delete-search-history.component'; diff --git a/frontend/src/app/shared/dropdown/dropdown-menu.directive.ts b/frontend/src/app/shared/dropdown/dropdown-menu.directive.ts index 8c15f9c53..7ad61086d 100644 --- a/frontend/src/app/shared/dropdown/dropdown-menu.directive.ts +++ b/frontend/src/app/shared/dropdown/dropdown-menu.directive.ts @@ -4,7 +4,7 @@ import { takeUntil } from 'rxjs/operators'; import { Subject } from 'rxjs'; import * as _ from 'lodash'; import { DropdownService } from './dropdown.service'; -import { modulo } from '../../utils/utils'; +import { modulo } from '@utils/utils'; @Directive({ selector: '[iaDropdownMenu]' diff --git a/frontend/src/app/shared/icons.ts b/frontend/src/app/shared/icons.ts index 21b16626d..65803f01d 100644 --- a/frontend/src/app/shared/icons.ts +++ b/frontend/src/app/shared/icons.ts @@ -1,15 +1,18 @@ import { IconDefinition as RegularIconDefinition, + faClock, faNewspaper, } from '@fortawesome/free-regular-svg-icons'; import { IconDefinition as SolidIconDefinition, - faAngleDown, faAngleUp, faArrowLeft, faArrowRight, faAt, faBook, faBookmark, faBookOpen, faBuilding, faChartColumn, - faCheck, faChevronLeft, faChevronRight, faCog, faCogs, faDatabase, faDiagramProject, - faDownload, faEnvelope, faEye, faFilter, faHistory, faImage, faInfo, faInfoCircle, faLink, faList, faLocationDot, faLock, - faMinus, faPalette, faPencil, faPlus, faQuestionCircle, faSearch, faSearchMinus, faSearchPlus, faSignOut, - faSortAlphaAsc, faSortAlphaDesc, faSortNumericAsc, faSortNumericDesc, faSquare, - faTable, faTags, faTimes, faTrashCan, faUndo, faUpload, faUser + faAngleDown, faAngleUp, faArrowLeft, faArrowRight, faAt, faBook, faBookmark, + faBookOpen, faBuilding, faChartColumn, faCheck, faChevronLeft, faChevronRight, faCog, + faCogs, faDatabase, faDiagramProject, faDownload, faEnvelope, faEye, faFilter, + faHistory, faImage, faInfo, faInfoCircle, faLink, faList, faLocationDot, faLock, + faMinus, faPalette, faPencil, faPlus, faQuestionCircle, faSearch, faSearchMinus, + faSearchPlus, faSignOut, faSortAlphaAsc, faSortAlphaDesc, faSortNumericAsc, + faSortNumericDesc, faSquare, faTable, faTags, faTimes, faTrashCan, faUndo, faUpload, + faUser } from '@fortawesome/free-solid-svg-icons'; type IconDefinition = SolidIconDefinition | RegularIconDefinition; @@ -55,6 +58,7 @@ export const actionIcons: Icons = { delete: faTrashCan, edit: faPencil, view: faEye, + wait: faClock, }; export const formIcons: Icons = { diff --git a/frontend/src/app/shared/pipes/elasticsearch-highlight.pipe.ts b/frontend/src/app/shared/pipes/elasticsearch-highlight.pipe.ts index ae1d374da..9ab5905d6 100644 --- a/frontend/src/app/shared/pipes/elasticsearch-highlight.pipe.ts +++ b/frontend/src/app/shared/pipes/elasticsearch-highlight.pipe.ts @@ -1,7 +1,7 @@ import { Pipe, PipeTransform } from '@angular/core'; import * as _ from 'lodash'; -import { CorpusField, FoundDocument } from '../../models'; +import { CorpusField, FoundDocument } from '@models'; @Pipe({ name: 'elasticsearchHighlight' diff --git a/frontend/src/app/shared/pipes/geo-data.pipe.ts b/frontend/src/app/shared/pipes/geo-data.pipe.ts index 08d9df047..64726ee7e 100644 --- a/frontend/src/app/shared/pipes/geo-data.pipe.ts +++ b/frontend/src/app/shared/pipes/geo-data.pipe.ts @@ -1,6 +1,6 @@ import { Pipe, PipeTransform } from '@angular/core'; import { DomSanitizer } from '@angular/platform-browser'; -import { CorpusField, FoundDocument } from '../../models'; +import { CorpusField, FoundDocument } from '@models'; @Pipe({ name: 'geoData' }) diff --git a/frontend/src/app/shared/pipes/regex-highlight.pipe.ts b/frontend/src/app/shared/pipes/regex-highlight.pipe.ts index 595499503..058f89908 100644 --- a/frontend/src/app/shared/pipes/regex-highlight.pipe.ts +++ b/frontend/src/app/shared/pipes/regex-highlight.pipe.ts @@ -1,6 +1,6 @@ import { Pipe, PipeTransform, SecurityContext } from '@angular/core'; import { DomSanitizer } from '@angular/platform-browser'; -import { HighlightService } from '../../services/highlight.service'; +import { HighlightService } from '@services/highlight.service'; @Pipe({ name: 'regexHighlight' @@ -18,7 +18,7 @@ export class RegexHighlightPipe implements PipeTransform { const highlightedText = parts.map(part => { const sanitizedText = this.sanitizedLineBreaks(part.substring, '
    '); - return part.isHit ? `${sanitizedText}` : sanitizedText; + return part.isHit ? `${sanitizedText}` : sanitizedText; }).join(''); return this.sanitizer.bypassSecurityTrustHtml(highlightedText); diff --git a/frontend/src/app/shared/shared.module.ts b/frontend/src/app/shared/shared.module.ts index ee400898a..9b0fa3bae 100644 --- a/frontend/src/app/shared/shared.module.ts +++ b/frontend/src/app/shared/shared.module.ts @@ -17,6 +17,7 @@ import { TabPanelDirective } from './tabs/tab-panel.directive'; import { TabsComponent } from './tabs/tabs.component'; import { ToggleComponent } from './toggle/toggle.component'; import { SlugifyPipe } from './pipes/slugify.pipe'; +import { ToggleButtonDirective } from './toggle-button.directive'; @NgModule({ declarations: [ @@ -28,6 +29,7 @@ import { SlugifyPipe } from './pipes/slugify.pipe'; TabPanelDirective, ToggleComponent, SlugifyPipe, + ToggleButtonDirective, ], exports: [ // shared components @@ -36,6 +38,7 @@ import { SlugifyPipe } from './pipes/slugify.pipe'; ScrollToDirective, TabsComponent, TabPanelDirective, + ToggleButtonDirective, // shared modules BrowserAnimationsModule, diff --git a/frontend/src/app/shared/tabs/tabs.component.html b/frontend/src/app/shared/tabs/tabs.component.html index 854c2bc58..738ef3020 100644 --- a/frontend/src/app/shared/tabs/tabs.component.html +++ b/frontend/src/app/shared/tabs/tabs.component.html @@ -1,20 +1,27 @@
    -
    +
    diff --git a/frontend/src/app/shared/tabs/tabs.component.spec.ts b/frontend/src/app/shared/tabs/tabs.component.spec.ts index 24182bc1f..792c19c5f 100644 --- a/frontend/src/app/shared/tabs/tabs.component.spec.ts +++ b/frontend/src/app/shared/tabs/tabs.component.spec.ts @@ -16,9 +16,11 @@ describe('TabsComponent', () => { component = fixture.componentInstance; component.tabs = [{ label: 'First tab', + elementId: 'tab-1', id: 1 }, { label: 'Second tab', + elementId: 'tab-2', id: 2 }]; fixture.detectChanges(); diff --git a/frontend/src/app/shared/tabs/tabs.component.ts b/frontend/src/app/shared/tabs/tabs.component.ts index e8a25ba0d..34ee33ede 100644 --- a/frontend/src/app/shared/tabs/tabs.component.ts +++ b/frontend/src/app/shared/tabs/tabs.component.ts @@ -5,12 +5,13 @@ import { import * as _ from 'lodash'; import { TabPanelDirective } from './tab-panel.directive'; import { IconDefinition } from '@fortawesome/free-solid-svg-icons'; -import { modulo } from '../../utils/utils'; -import { SlugifyPipe } from '../pipes/slugify.pipe'; +import { modulo } from '@utils/utils'; +import { SlugifyPipe } from '@shared/pipes/slugify.pipe'; interface Tab { label: string; // display name id: string | number; + elementId: string; icon?: IconDefinition; }; @@ -34,6 +35,7 @@ export class TabsComponent implements AfterContentInit { ngAfterContentInit(): void { this.tabs = this.tabPanels.map(tabPanel => ({ id: tabPanel.id, + elementId: this.tabLinkId(tabPanel.id), label: tabPanel.title, icon: tabPanel.icon, })); @@ -48,8 +50,7 @@ export class TabsComponent implements AfterContentInit { cycleTab(event: KeyboardEvent) { const target = event.target as Element; - const id = target.id; - const tabIndex = this.tabs.findIndex(tab => this.tabLinkId(tab.id) === id); + const tabIndex = this.tabs.findIndex(tab => tab.elementId === target.id); const keyBindings = { ArrowLeft: -1, @@ -59,14 +60,14 @@ export class TabsComponent implements AfterContentInit { const shift = keyBindings[event.key]; const newIndex = modulo(tabIndex + shift, this.tabs.length); const newTab = this.tabs[newIndex]; - this.setTabLinkFocus(newTab.id); + this.setTabLinkFocus(newTab.elementId); this.selectTab(newTab); } - setTabLinkFocus(id: string | number) { + setTabLinkFocus(elementId: string) { this.tabLinks.forEach(tabLink => { const element = tabLink.nativeElement; - const focus = element.id === this.tabLinkId(id); + const focus = element.id === elementId; element.tabIndex = focus ? 0 : -1; if (focus) { element.focus(); diff --git a/frontend/src/app/shared/toggle-button.directive.spec.ts b/frontend/src/app/shared/toggle-button.directive.spec.ts new file mode 100644 index 000000000..244a5514d --- /dev/null +++ b/frontend/src/app/shared/toggle-button.directive.spec.ts @@ -0,0 +1,54 @@ +import { Component } from '@angular/core'; +import { ToggleButtonDirective } from './toggle-button.directive'; +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { CommonModule } from '@angular/common'; + +@Component({ + template: ` + + `, +}) +class ToggleButtonTestComponent { + active = false; + class = 'is-primary'; +} + +describe('ToggleButtonDirective', () => { + let fixture: ComponentFixture; + let component: ToggleButtonTestComponent; + let button: HTMLButtonElement; + + beforeEach(() => { + TestBed.configureTestingModule({ + declarations: [ToggleButtonTestComponent, ToggleButtonDirective], + imports: [CommonModule], + }); + fixture = TestBed.createComponent(ToggleButtonTestComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + const element = fixture.debugElement.nativeElement as Element; + button = element.querySelector('button'); + + }); + + it('should show toggle state', () => { + expect(button.className).toEqual('button'); + expect(button.getAttribute('aria-pressed')).toBe('false'); + + component.active = true; + fixture.detectChanges(); + + expect(button.className).toEqual('button is-primary'); + expect(button.getAttribute('aria-pressed')).toBe('true'); + }); + + it('should set the CSS class through input', () => { + component.class = 'is-danger'; + component.active = true; + fixture.detectChanges(); + + expect(button.className).toEqual('button is-danger'); + }); +}); diff --git a/frontend/src/app/shared/toggle-button.directive.ts b/frontend/src/app/shared/toggle-button.directive.ts new file mode 100644 index 000000000..30132d3ae --- /dev/null +++ b/frontend/src/app/shared/toggle-button.directive.ts @@ -0,0 +1,17 @@ +import { Directive, HostBinding, Input } from '@angular/core'; + +@Directive({ + selector: 'button[iaToggleButton]', +}) +export class ToggleButtonDirective { + @HostBinding('attr.aria-pressed') + @Input() active: boolean; + + /** name of the CSS class that should be applied when active */ + @Input() activeClass: string = 'is-primary'; + + @HostBinding('class') + get classes(): Record { + return { [this.activeClass]: this.active } + } +} diff --git a/frontend/src/app/store/router-store.service.ts b/frontend/src/app/store/router-store.service.ts index 1f681aae1..a017117d3 100644 --- a/frontend/src/app/store/router-store.service.ts +++ b/frontend/src/app/store/router-store.service.ts @@ -3,7 +3,7 @@ import { Params, Router } from '@angular/router'; import { Store } from './types'; import { Observable, Subject } from 'rxjs'; import { bufferTime, filter, map } from 'rxjs/operators'; -import { mergeAllParams } from '../utils/params'; +import { mergeAllParams } from '@utils/params'; import * as _ from 'lodash'; /** diff --git a/frontend/src/app/store/simple-store.ts b/frontend/src/app/store/simple-store.ts index dab556cd5..d81495fd0 100644 --- a/frontend/src/app/store/simple-store.ts +++ b/frontend/src/app/store/simple-store.ts @@ -3,7 +3,7 @@ import { Store } from './types'; import { Params } from '@angular/router'; import * as _ from 'lodash'; import { map, scan } from 'rxjs/operators'; -import { mergeParams, omitNullParameters } from '../utils/params'; +import { mergeParams, omitNullParameters } from '@utils/params'; /** simple store that does not depend on services or routing * @@ -18,7 +18,7 @@ export class SimpleStore implements Store { this.paramUpdates$.pipe( scan(mergeParams), map(omitNullParameters), - map(obj => _.mapValues(obj, _.toString)) + map(obj => _.mapValues(obj, _.toString)), ).subscribe(params => this.params$.next(params) ); diff --git a/frontend/src/app/tag/document-tags/document-tags.component.ts b/frontend/src/app/tag/document-tags/document-tags.component.ts index 45ca9db6f..3889f1ab7 100644 --- a/frontend/src/app/tag/document-tags/document-tags.component.ts +++ b/frontend/src/app/tag/document-tags/document-tags.component.ts @@ -1,7 +1,7 @@ import { Component, Input, OnChanges, SimpleChanges } from '@angular/core'; -import { FoundDocument, Tag } from '../../models'; +import { FoundDocument, Tag } from '@models'; import * as _ from 'lodash'; -import { formIcons, actionIcons } from '../../shared/icons'; +import { formIcons, actionIcons } from '@shared/icons'; @Component({ selector: 'ia-document-tags', diff --git a/frontend/src/app/tag/tag-overview/tag-overview.component.spec.ts b/frontend/src/app/tag/tag-overview/tag-overview.component.spec.ts index 85e3916d0..81ef58006 100644 --- a/frontend/src/app/tag/tag-overview/tag-overview.component.spec.ts +++ b/frontend/src/app/tag/tag-overview/tag-overview.component.spec.ts @@ -2,7 +2,7 @@ import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing'; import { TagOverviewComponent } from './tag-overview.component'; import { HttpClientTestingModule } from '@angular/common/http/testing'; -import { ApiRetryService } from '../../services'; +import { ApiRetryService } from '@services'; import { RouterTestingModule } from '@angular/router/testing'; import { commonTestBed } from '../../common-test-bed'; diff --git a/frontend/src/app/tag/tag-overview/tag-overview.component.ts b/frontend/src/app/tag/tag-overview/tag-overview.component.ts index 41c442f08..64e346394 100644 --- a/frontend/src/app/tag/tag-overview/tag-overview.component.ts +++ b/frontend/src/app/tag/tag-overview/tag-overview.component.ts @@ -1,13 +1,13 @@ import { Component, OnInit } from '@angular/core'; import { isUndefined } from 'lodash'; -import { Corpus, QueryModel, Tag } from '../../models'; -import { isTagFilter } from '../../models/tag-filter'; -import { CorpusService } from '../../services'; -import { TagService } from '../../services/tag.service'; -import { actionIcons, formIcons } from '../../shared/icons'; -import { findByName } from '../../utils/utils'; +import { Corpus, QueryModel, Tag } from '@models'; +import { isTagFilter } from '@models/tag-filter'; +import { CorpusService } from '@services'; +import { TagService } from '@services/tag.service'; +import { actionIcons, formIcons } from '@shared/icons'; +import { findByName } from '@utils/utils'; import { Title } from '@angular/platform-browser'; -import { pageTitle } from '../../utils/app'; +import { pageTitle } from '@utils/app'; @Component({ selector: 'ia-tag-overview', diff --git a/frontend/src/app/tag/tag-select/tag-select.component.ts b/frontend/src/app/tag/tag-select/tag-select.component.ts index 79f447189..2f3a44a6c 100644 --- a/frontend/src/app/tag/tag-select/tag-select.component.ts +++ b/frontend/src/app/tag/tag-select/tag-select.component.ts @@ -9,10 +9,10 @@ import { } from '@angular/core'; import * as _ from 'lodash'; import { Observable, Subject } from 'rxjs'; -import { Tag } from '../../models'; -import { TagService } from '../../services/tag.service'; +import { Tag } from '@models'; +import { TagService } from '@services/tag.service'; import { takeUntil } from 'rxjs/operators'; -import { actionIcons, formIcons } from '../../shared/icons'; +import { actionIcons, formIcons } from '@shared/icons'; @Component({ selector: 'ia-tag-select', diff --git a/frontend/src/app/tag/tag.module.ts b/frontend/src/app/tag/tag.module.ts index 80517ef58..e7244072b 100644 --- a/frontend/src/app/tag/tag.module.ts +++ b/frontend/src/app/tag/tag.module.ts @@ -1,5 +1,5 @@ import { NgModule } from '@angular/core'; -import { SharedModule } from '../shared/shared.module'; +import { SharedModule } from '@shared/shared.module'; import { TagSelectComponent } from './tag-select/tag-select.component'; import { DocumentTagsComponent } from './document-tags/document-tags.component'; import { TagOverviewComponent } from './tag-overview/tag-overview.component'; diff --git a/frontend/src/app/utils/api-query.ts b/frontend/src/app/utils/api-query.ts index c9d50935d..1a1a8c697 100644 --- a/frontend/src/app/utils/api-query.ts +++ b/frontend/src/app/utils/api-query.ts @@ -1,9 +1,11 @@ -import { FilterInterface } from '../models'; -import { APITagFilter } from '../models/search-requests'; +import { FilterInterface } from '@models'; +import { APITagFilter } from '@models/search-requests'; -import { TAG_FILTER, TagFilter } from '../models/tag-filter'; +import { TAG_FILTER, TagFilter } from '@models/tag-filter'; -export const makeTagSpecification = (filters: FilterInterface[]): APITagFilter => { +export const makeTagSpecification = ( + filters: FilterInterface[] +): APITagFilter => { const tagFilter = filters.find(isTagFilter); return tagFilter.dataToAPI(); }; diff --git a/frontend/src/app/utils/app.ts b/frontend/src/app/utils/app.ts index 0c9270492..8860867cd 100644 --- a/frontend/src/app/utils/app.ts +++ b/frontend/src/app/utils/app.ts @@ -1,4 +1,4 @@ -import { environment } from '../../environments/environment'; +import { environment } from '@environments/environment'; export const pageTitle = (pageName: string) => `${pageName} - ${environment.appName}`; diff --git a/frontend/src/app/utils/document-context.ts b/frontend/src/app/utils/document-context.ts index 42571f0ec..acafa9f63 100644 --- a/frontend/src/app/utils/document-context.ts +++ b/frontend/src/app/utils/document-context.ts @@ -1,6 +1,6 @@ import { Params } from '@angular/router'; -import { Corpus, FoundDocument, QueryModel } from '../models'; -import { PageResultsParameters } from '../models/page-results'; +import { Corpus, FoundDocument, QueryModel } from '@models'; +import { PageResultsParameters } from '@models/page-results'; import { omitNullParameters, pageResultsParametersToParams } from './params'; const documentContextQuery = (corpus: Corpus, document: FoundDocument): [QueryModel, PageResultsParameters] => { diff --git a/frontend/src/app/utils/download-history.ts b/frontend/src/app/utils/download-history.ts index 13ffd4691..b1a451122 100644 --- a/frontend/src/app/utils/download-history.ts +++ b/frontend/src/app/utils/download-history.ts @@ -1,12 +1,21 @@ -import { Corpus, Download, QueryModel } from '../models'; -import { APIQuery } from '../models/search-requests'; +import { Corpus, Download, QueryModel } from '@models'; +import { APIQuery } from '@models/search-requests'; import { apiQueryToQueryModel } from './es-query'; import * as _ from 'lodash'; -export const downloadQueryModel = (download: Download, corpus: Corpus): QueryModel => - _.first(downloadQueryModels(download, corpus)); +export const downloadQueryModel = ( + download: Download, + corpus: Corpus +): QueryModel => _.first(downloadQueryModels(download, corpus)); -export const downloadQueryModels = (download: Download, corpus: Corpus): QueryModel[] => { - const queries = (_.isArray(download.parameters) ? download.parameters : [download.parameters]) as APIQuery[]; - return queries.map(query => apiQueryToQueryModel(query, corpus)); +export const downloadQueryModels = ( + download: Download, + corpus: Corpus +): QueryModel[] => { + const queries = ( + _.isArray(download.parameters) + ? download.parameters + : [download.parameters] + ) as APIQuery[]; + return queries.map((query) => apiQueryToQueryModel(query, corpus)); }; diff --git a/frontend/src/app/utils/es-query.spec.ts b/frontend/src/app/utils/es-query.spec.ts index 0bf4b0dd1..020fcc064 100644 --- a/frontend/src/app/utils/es-query.spec.ts +++ b/frontend/src/app/utils/es-query.spec.ts @@ -5,10 +5,10 @@ import { makeEsSearchClause, makeHighlightSpecification, makeSimpleQueryString, makeSortSpecification, resultsParamsToAPIQuery } from './es-query'; -import { QueryModel } from '../models'; -import { PageResultsParameters } from '../models/page-results'; -import { APIQuery } from '../models/search-requests'; -import { isTagFilter } from '../models/tag-filter'; +import { QueryModel } from '@models'; +import { PageResultsParameters } from '@models/page-results'; +import { APIQuery } from '@models/search-requests'; +import { isTagFilter } from '@models/tag-filter'; describe('es-query utils', () => { it('should make a simple query string clause', () => { @@ -49,8 +49,8 @@ describe('es-query utils', () => { expect(makeHighlightSpecification(mockCorpus3, 'test', 100)).toEqual({ highlight: { fragment_size: 100, - pre_tags: [''], - post_tags: [''], + pre_tags: [''], + post_tags: [''], order: 'score', fields: [{speech: {}}] } diff --git a/frontend/src/app/utils/es-query.ts b/frontend/src/app/utils/es-query.ts index 5368470bb..cb23946fc 100644 --- a/frontend/src/app/utils/es-query.ts +++ b/frontend/src/app/utils/es-query.ts @@ -2,15 +2,24 @@ import * as _ from 'lodash'; import { - BooleanQuery, Corpus, CorpusField, EsFilter, EsSearchClause, FilterInterface, MatchAll, + BooleanQuery, + Corpus, + CorpusField, + EsFilter, + EsSearchClause, + FilterInterface, + MatchAll, QueryModel, - SimpleQueryString, SortBy, SortDirection } from '../models'; -import { EsQuery } from '../models'; + SimpleQueryString, + SortBy, + SortDirection, +} from '@models'; +import { EsQuery } from '@models'; import { findByName } from './utils'; -import { SearchFilter } from '../models/field-filter'; -import { APIQuery } from '../models/search-requests'; -import { TagFilter } from '../models/tag-filter'; -import { PageResultsParameters } from '../models/page-results'; +import { SearchFilter } from '@models/field-filter'; +import { APIQuery } from '@models/search-requests'; +import { TagFilter } from '@models/tag-filter'; +import { PageResultsParameters } from '@models/page-results'; import { DeepPartial } from 'chart.js/dist/types/utils'; import { SimpleStore } from '../store/simple-store'; @@ -89,8 +98,8 @@ export const makeHighlightSpecification = (corpus: Corpus, queryText?: string, h return { highlight: { fragment_size: highlightSize, - pre_tags: [''], - post_tags: [''], + pre_tags: [''], + post_tags: [''], order: 'score', fields: highlightFields.map((field) => field.displayType === 'text_content' && diff --git a/frontend/src/app/utils/params.spec.ts b/frontend/src/app/utils/params.spec.ts index 909556380..4fbe7f03f 100644 --- a/frontend/src/app/utils/params.spec.ts +++ b/frontend/src/app/utils/params.spec.ts @@ -3,9 +3,9 @@ import { sortSettingsFromParams, sortSettingsToParams } from './params'; import { mockCorpus, mockCorpus3, mockField2, mockField } from '../../mock-data/corpus'; -import { SortState } from '../models'; +import { SortState } from '@models'; import * as _ from 'lodash'; -import { PageParameters, PageResultsParameters } from '../models/page-results'; +import { PageParameters, PageResultsParameters } from '@models/page-results'; describe('searchFieldsFromParams', () => { it('should parse field parameters', () => { diff --git a/frontend/src/app/utils/params.ts b/frontend/src/app/utils/params.ts index 52c0ee3d8..d830a7dff 100644 --- a/frontend/src/app/utils/params.ts +++ b/frontend/src/app/utils/params.ts @@ -1,8 +1,21 @@ import { ParamMap, Params, convertToParamMap } from '@angular/router'; import * as _ from 'lodash'; -import { Corpus, CorpusField, FilterInterface, QueryModel, SearchFilter, SortBy, SortDirection, SortState } from '../models'; -import { TagFilter } from '../models/tag-filter'; -import { PageParameters, PageResultsParameters, RESULTS_PER_PAGE } from '../models/page-results'; +import { + Corpus, + CorpusField, + FilterInterface, + QueryModel, + SearchFilter, + SortBy, + SortDirection, + SortState, +} from '@models'; +import { TagFilter } from '@models/tag-filter'; +import { + PageParameters, + PageResultsParameters, + RESULTS_PER_PAGE, +} from '@models/page-results'; import { findByName } from './utils'; import { SimpleStore } from '../store/simple-store'; @@ -25,6 +38,10 @@ export const mergeAllParams = (values: Params[]): Params => export const queryFromParams = (params: Params): string => params['query']; +export const queryToParams = (queryText: string): Params => ({ + query: queryText || null +}); + export const searchFieldsFromParams = (params: Params, corpus: Corpus): CorpusField[] => { if (params['fields']) { const fieldNames = params['fields'].split(','); diff --git a/frontend/src/app/utils/user.spec.ts b/frontend/src/app/utils/user.spec.ts index a43600813..9772c2b95 100644 --- a/frontend/src/app/utils/user.spec.ts +++ b/frontend/src/app/utils/user.spec.ts @@ -1,5 +1,5 @@ import * as _ from 'lodash'; -import { User, UserResponse } from '../models'; +import { User, UserResponse } from '@models'; import { parseUserData, encodeUserData } from './user'; /** diff --git a/frontend/src/app/utils/user.ts b/frontend/src/app/utils/user.ts index c0d44e644..abe78907a 100644 --- a/frontend/src/app/utils/user.ts +++ b/frontend/src/app/utils/user.ts @@ -1,5 +1,5 @@ import * as _ from 'lodash'; -import { User, UserResponse } from '../models'; +import { User, UserResponse } from '@models'; /* Transforms backend user response to User object * diff --git a/frontend/src/app/visualization/barchart/barchart-options.component.html b/frontend/src/app/visualization/barchart/barchart-options.component.html index 05fc79abb..029c0e4d7 100644 --- a/frontend/src/app/visualization/barchart/barchart-options.component.html +++ b/frontend/src/app/visualization/barchart/barchart-options.component.html @@ -59,7 +59,6 @@
    - +
    diff --git a/frontend/src/app/visualization/barchart/barchart-options.component.ts b/frontend/src/app/visualization/barchart/barchart-options.component.ts index 5a339cdfd..320fea717 100644 --- a/frontend/src/app/visualization/barchart/barchart-options.component.ts +++ b/frontend/src/app/visualization/barchart/barchart-options.component.ts @@ -2,8 +2,8 @@ import { Component, EventEmitter, Input, OnChanges, Output, SimpleChanges } from import { ActivatedRoute, ParamMap, Router } from '@angular/router'; import * as _ from 'lodash'; import { ParamDirective } from '../../param/param-directive'; -import { Normalizer, ChartType, ChartParameters } from '../../models'; -import { ParamService } from '../../services'; +import { Normalizer, ChartType, ChartParameters } from '@models'; +import { ParamService } from '@services'; @Component({ selector: 'ia-barchart-options', @@ -23,14 +23,11 @@ export class BarchartOptionsComponent @Output() chartParameters = new EventEmitter(); @Output() queriesChanged = new EventEmitter(); - @Output() clearQueries = new EventEmitter(); currentNormalizer: Normalizer; currentChartType: ChartType = 'bar'; - public queries: string[] = []; - showEdit = false; nullableParameters = ['normalize']; @@ -38,24 +35,12 @@ export class BarchartOptionsComponent constructor( route: ActivatedRoute, router: Router, - paramService: ParamService + paramService: ParamService, ) { super(route, router, paramService); } - get showTermFrequency(): boolean { - return _.some(this.queries); - } - ngOnChanges(changes: SimpleChanges): void { - if (changes.queryText) { - if (this.queryText) { - this.queries = [this.queryText]; - } else { - this.queries = []; - } - } - if ( changes.showTokenCountOption && changes.showTokenCountOption.currentValue && @@ -86,6 +71,9 @@ export class BarchartOptionsComponent teardown() {} setStateFromParams(params: ParamMap) { + // show term comparison editor if there are terms in the route ; + // don't hide the editor if already displayed + this.showEdit = this.showEdit || params.has('compareTerms'); if (params.has('normalize')) { this.currentNormalizer = params.get('normalize') as Normalizer; } else { @@ -101,17 +89,10 @@ export class BarchartOptionsComponent } updateQueries(queries: string[]) { - this.queries = queries; - if (this.queries.length === 1 && this.queries[0] === this.queryText) { + if (_.isEqual(queries, [this.queryText])) { this.showEdit = false; } - this.queriesChanged.emit(this.queries); + this.queriesChanged.emit(queries); } - signalClearQueries() { - this.queries = [this.queryText]; - this.showEdit = false; - this.setParams({ visualizeTerm: null }); - this.clearQueries.emit(); - } } diff --git a/frontend/src/app/visualization/barchart/barchart.directive.ts b/frontend/src/app/visualization/barchart/barchart.directive.ts index 1907339b6..72030353a 100644 --- a/frontend/src/app/visualization/barchart/barchart.directive.ts +++ b/frontend/src/app/visualization/barchart/barchart.directive.ts @@ -3,19 +3,33 @@ import { Directive, EventEmitter, Host, HostBinding, Input, OnChanges, OnDestroy import * as _ from 'lodash'; -import { ApiService, NotificationService, SearchService } from '../../services/index'; +import { + ApiService, + NotificationService, + SearchService, +} from '@services/index'; import { Chart, ChartOptions } from 'chart.js'; import { - Corpus, FreqTableHeaders, QueryModel, CorpusField, TaskResult, - BarchartSeries, TimelineDataPoint, HistogramDataPoint, TermFrequencyResult, ChartParameters -} from '../../models'; + Corpus, + FreqTableHeaders, + QueryModel, + CorpusField, + TaskResult, + BarchartSeries, + TimelineDataPoint, + HistogramDataPoint, + TermFrequencyResult, + ChartParameters, +} from '@models'; import Zoom from 'chartjs-plugin-zoom'; import { BehaviorSubject, Subject } from 'rxjs'; -import { selectColor } from '../../utils/select-color'; -import { VisualizationService } from '../../services/visualization.service'; -import { showLoading } from '../../utils/utils'; +import { selectColor } from '@utils/select-color'; +import { VisualizationService } from '@services/visualization.service'; +import { showLoading } from '@utils/utils'; import { takeUntil } from 'rxjs/operators'; -import { DateHistogramResult, TermsResult } from '../../models/aggregation'; +import { DateHistogramResult, TermsResult } from '@models/aggregation'; +import { ComparedQueries } from '@models/compared-queries'; +import { RouterStoreService } from '../../store/router-store.service'; const hintSeenSessionStorageKey = 'hasSeenTimelineZoomingHint'; const hintHidingMinDelay = 500; // milliseconds @@ -49,9 +63,10 @@ export abstract class BarchartDirective< @Input() palette: string[]; @Input() frequencyMeasure: 'documents' | 'tokens' = 'documents'; - normalizer: 'raw' | 'percent' | 'documents' | 'terms' = 'raw'; + normalizer: 'raw' | 'percent' | 'documents' | 'terms' = 'raw'; chartType: 'bar' | 'line' | 'scatter' = 'bar'; + comparedQueries: ComparedQueries; documentLimit = 1000; // maximum number of documents to search through for term frequency documentLimitExceeded = false; // whether the results include documents than the limit @@ -61,8 +76,6 @@ export abstract class BarchartDirective< tableHeaders: FreqTableHeaders; tableData: any[]; - /** list of query used by each series in te graph */ - queries: string[] = []; /** Stores the key that can be used in a DataPoint object * to retrieve the y-axis value. @@ -148,13 +161,16 @@ export abstract class BarchartDirective< public searchService: SearchService, public visualizationService: VisualizationService, public apiService: ApiService, - private notificationService: NotificationService + private notificationService: NotificationService, + private routerStoreService: RouterStoreService, ) { const chartDefault = Chart.defaults; chartDefault.elements.bar.backgroundColor = selectColor(); chartDefault.elements.bar.hoverBackgroundColor = selectColor(); chartDefault.plugins.tooltip.displayColors = false; chartDefault.plugins.tooltip.intersect = false; + this.comparedQueries = new ComparedQueries(this.routerStoreService); + this.comparedQueries.allQueries$.subscribe(this.updateQueries.bind(this)); } get isLoading() { @@ -182,6 +198,7 @@ export abstract class BarchartDirective< ngOnDestroy(): void { this.stopPolling$.next(); this.destroy$.next(undefined); + this.comparedQueries.complete(); } /** check whether input changes should force reloading the data */ @@ -218,8 +235,10 @@ export abstract class BarchartDirective< } initQueries(): void { - this.rawData = [this.newSeries(this.queryText)]; - this.queries = [this.queryText]; + this.rawData = [ + this.newSeries(this.queryText), + ...this.comparedQueries.state$.value.compare.map(this.newSeries) + ]; } /** if a chart is active, clear canvas and reset chart object */ @@ -233,13 +252,15 @@ export abstract class BarchartDirective< /** update the queries in the graph to the input array. Preserve results if possible, and kick off loading the rest. */ updateQueries(queries: string[]) { - this.rawData = queries.map((queryText) => { - const existingSeries = this.rawData.find( - (series) => series.queryText === queryText - ); - return existingSeries || this.newSeries(queryText); - }); - this.prepareChart(); + if (this.rawData) { + this.rawData = queries.map((queryText) => { + const existingSeries = this.rawData.find( + (series) => series.queryText === queryText + ); + return existingSeries || this.newSeries(queryText); + }); + this.prepareChart(); + } } /** make a blank series object */ @@ -252,14 +273,6 @@ export abstract class BarchartDirective< }; } - /** Remove any additional queries from the BarchartOptions component. - * Only keep the original query */ - clearAddedQueries() { - this.rawData = this.rawData.slice(0, 1); - this.queries = [this.queryText]; - this.prepareChart(); - } - /** Show a loading spinner and load data for the graph. * This function should be called after (potential) changes to parameters. */ diff --git a/frontend/src/app/visualization/barchart/histogram.component.html b/frontend/src/app/visualization/barchart/histogram.component.html index 7b274133d..90f17cf1b 100644 --- a/frontend/src/app/visualization/barchart/histogram.component.html +++ b/frontend/src/app/visualization/barchart/histogram.component.html @@ -9,7 +9,7 @@ [queryText]="queryText" [showTokenCountOption]="totalTokenCountAvailable" [isLoading]="isLoading" [frequencyMeasure]="frequencyMeasure" [freqTable]="asTable" [histogram]="true" - (chartParameters)="onOptionChange($event)" (queriesChanged)="updateQueries($event)" (clearQueries)="clearAddedQueries()"> + (chartParameters)="onOptionChange($event)" (queriesChanged)="updateQueries($event)"> diff --git a/frontend/src/app/visualization/barchart/histogram.component.spec.ts b/frontend/src/app/visualization/barchart/histogram.component.spec.ts index 7ca935465..08c68ba86 100644 --- a/frontend/src/app/visualization/barchart/histogram.component.spec.ts +++ b/frontend/src/app/visualization/barchart/histogram.component.spec.ts @@ -1,5 +1,5 @@ import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing'; -import { QueryModel } from '../../models'; +import { QueryModel } from '@models'; import { commonTestBed } from '../../common-test-bed'; diff --git a/frontend/src/app/visualization/barchart/histogram.component.ts b/frontend/src/app/visualization/barchart/histogram.component.ts index 24f952f09..9f32f05aa 100644 --- a/frontend/src/app/visualization/barchart/histogram.component.ts +++ b/frontend/src/app/visualization/barchart/histogram.component.ts @@ -6,10 +6,11 @@ import { HistogramSeries, MultipleChoiceFilterOptions, QueryModel, - RangeFilterOptions} from '../../models/index'; -import { selectColor } from '../../utils/select-color'; + RangeFilterOptions, +} from '@models/index'; +import { selectColor } from '@utils/select-color'; import { BarchartDirective } from './barchart.directive'; -import { TermsAggregator, TermsResult } from '../../models/aggregation'; +import { TermsAggregator, TermsResult } from '@models/aggregation'; function formatXAxisLabel(value): string { const label = this.getLabelForValue(value); // from chartJS api diff --git a/frontend/src/app/visualization/barchart/term-comparison-editor/term-comparison-editor.component.html b/frontend/src/app/visualization/barchart/term-comparison-editor/term-comparison-editor.component.html index 2dd8837cd..fb13d8032 100644 --- a/frontend/src/app/visualization/barchart/term-comparison-editor/term-comparison-editor.component.html +++ b/frontend/src/app/visualization/barchart/term-comparison-editor/term-comparison-editor.component.html @@ -6,7 +6,7 @@
    -
    -
    -
    diff --git a/frontend/src/app/visualization/barchart/timeline.component.ts b/frontend/src/app/visualization/barchart/timeline.component.ts index b7be5a80a..eb3e11d36 100644 --- a/frontend/src/app/visualization/barchart/timeline.component.ts +++ b/frontend/src/app/visualization/barchart/timeline.component.ts @@ -2,16 +2,22 @@ import { Component, OnChanges, OnInit } from '@angular/core'; import * as _ from 'lodash'; -import { QueryModel, TimelineSeries, TimelineDataPoint, +import { + QueryModel, + TimelineSeries, + TimelineDataPoint, TimeCategory, DateFilterData, -} from '../../models/index'; +} from '@models/index'; import { BarchartDirective } from './barchart.directive'; import * as moment from 'moment'; import 'chartjs-adapter-moment'; -import { selectColor } from '../../utils/select-color'; -import { showLoading } from '../../utils/utils'; -import { DateHistogramAggregator, DateHistogramResult } from '../../models/aggregation'; +import { selectColor } from '@utils/select-color'; +import { showLoading } from '@utils/utils'; +import { + DateHistogramAggregator, + DateHistogramResult, +} from '@models/aggregation'; @Component({ diff --git a/frontend/src/app/visualization/freqtable.component.html b/frontend/src/app/visualization/freqtable.component.html index bdfe2855d..7663f269d 100644 --- a/frontend/src/app/visualization/freqtable.component.html +++ b/frontend/src/app/visualization/freqtable.component.html @@ -26,15 +26,17 @@
    - +
    - +
    -
    diff --git a/frontend/src/app/visualization/freqtable.component.ts b/frontend/src/app/visualization/freqtable.component.ts index ce1525432..5ffec2be3 100644 --- a/frontend/src/app/visualization/freqtable.component.ts +++ b/frontend/src/app/visualization/freqtable.component.ts @@ -1,8 +1,8 @@ import { Input, Component, OnChanges, OnDestroy, ViewEncapsulation, SimpleChanges } from '@angular/core'; import * as _ from 'lodash'; import { saveAs } from 'file-saver'; -import { FreqTableHeader, FreqTableHeaders } from '../models'; -import { actionIcons } from '../shared/icons'; +import { FreqTableHeader, FreqTableHeaders } from '@models'; +import { actionIcons } from '@shared/icons'; @Component({ selector: 'ia-freqtable', diff --git a/frontend/src/app/visualization/full-data-button/full-data-button.component.ts b/frontend/src/app/visualization/full-data-button/full-data-button.component.ts index fdbc979a8..d00b1c9b6 100644 --- a/frontend/src/app/visualization/full-data-button/full-data-button.component.ts +++ b/frontend/src/app/visualization/full-data-button/full-data-button.component.ts @@ -1,5 +1,5 @@ import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'; -import { actionIcons } from '../../shared/icons'; +import { actionIcons } from '@shared/icons'; @Component({ selector: 'ia-full-data-button', diff --git a/frontend/src/app/visualization/map/map.component.html b/frontend/src/app/visualization/map/map.component.html index 4324ae9fa..894ccc28e 100644 --- a/frontend/src/app/visualization/map/map.component.html +++ b/frontend/src/app/visualization/map/map.component.html @@ -1,2 +1,3 @@ -

    Map component

    -
    {{ results | json }}
    +
    +
    +
    diff --git a/frontend/src/app/visualization/map/map.component.ts b/frontend/src/app/visualization/map/map.component.ts index 531056176..03e7b2e08 100644 --- a/frontend/src/app/visualization/map/map.component.ts +++ b/frontend/src/app/visualization/map/map.component.ts @@ -1,76 +1,252 @@ -import { Component, EventEmitter, Input, Output, OnChanges, SimpleChanges } from '@angular/core'; +import { Component, ElementRef, EventEmitter, Input, Output, OnChanges, SimpleChanges, ViewChild } from '@angular/core'; import { BehaviorSubject } from 'rxjs'; +import embed, { VisualizationSpec } from 'vega-embed'; -import { Corpus, CorpusField, GeoDocument, QueryModel } from '../../models'; -import { VisualizationService } from '../../services'; -import { showLoading } from '../../utils/utils'; +import { Corpus, CorpusField, GeoDocument, GeoLocation, QueryModel } from '@models'; +import { VisualizationService } from '@services'; +import { showLoading } from '@utils/utils'; @Component({ - selector: 'ia-map', - templateUrl: './map.component.html', - styleUrls: ['./map.component.scss'] + selector: 'ia-map', + templateUrl: './map.component.html', + styleUrls: ['./map.component.scss'] }) export class MapComponent implements OnChanges { + @ViewChild('vegaMap') vegaMap!: ElementRef; + @Input() visualizedField: CorpusField; + @Input() queryModel: QueryModel; + @Input() corpus: Corpus; + @Input() resultsCount: number; + @Input() asTable: boolean; - @Input() visualizedField: CorpusField; - @Input() queryModel: QueryModel; - @Input() corpus: Corpus; - @Input() resultsCount: number; - @Input() asTable: boolean; - - @Output() mapError = new EventEmitter(); - - results: GeoDocument[]; - - isLoading$ = new BehaviorSubject(false); - - constructor(private visualizationService: VisualizationService) { } - - get readyToLoad() { - return ( - this.corpus && - this.visualizedField && - this.queryModel - ); - } - - ngOnChanges(changes: SimpleChanges) { - if ( - this.readyToLoad && - (changes.corpus || changes.visualizedField || changes.queryModel) - ) { - if (changes.queryModel) { - this.queryModel.update.subscribe(this.loadData.bind(this)); - } - this.loadData(); - } else { - // this.makeChart(); + @Output() mapError = new EventEmitter(); + + mapCenter: GeoLocation; + results: GeoDocument[]; + + isLoading$ = new BehaviorSubject(false); + + constructor( + private visualizationService: VisualizationService + ) { } + + get readyToLoad() { + return ( + this.corpus && + this.visualizedField && + this.queryModel + ); + } + + + ngOnChanges(changes: SimpleChanges) { + if (this.readyToLoad) { + this.loadMapCenter(); + + if ( changes.corpus || changes.visualizedField || changes.queryModel ) { + this.queryModel.update.subscribe(this.loadData.bind(this)); + this.loadData(); + } + } + } + + + loadMapCenter() { + this.visualizationService.getGeoCentroid(this.visualizedField.name, this.corpus) + .then(centroid => { + this.mapCenter = centroid; + }) + .catch(this.emitError.bind(this)); + } + + + loadData() { + showLoading( + this.isLoading$, + this.visualizationService + .getGeoData( + this.visualizedField.name, + this.queryModel, + this.corpus, + ) + .then(geoData => { + this.results = geoData; + this.renderChart(); + }) + .catch(this.emitError.bind(this)) + ); + } + + getVegaSpec(): VisualizationSpec { + // Returns a Vega map specification + // Uses pan/zoom signals from https://vega.github.io/vega/examples/zoomable-world-map/ + return { + "$schema": "https://vega.github.io/schema/vega/v5.json", + "description": "An interactive map supporting pan and zoom.", + "width": { "signal": "width" }, + "height": { "signal": "height" }, + "autosize": "none", + + "signals": [ + { + name: "corpusName", + value: this.corpus.name + }, + { "name": "tx", "update": "width / 2" }, + { "name": "ty", "update": "height / 2" }, + { + "name": "scale", + "value": 500, + "on": [{ + "events": { "type": "wheel", "consume": true }, + "update": "clamp(scale * pow(1.0005, -event.deltaY * pow(16, event.deltaMode)), 150, 3000)" + }] + }, + { + "name": "angles", + "value": [0, 0], + "on": [{ + "events": "pointerdown", + "update": "[rotateX, centerY]" + }] + }, + { + "name": "cloned", + "value": null, + "on": [{ + "events": "pointerdown", + "update": "copy('projection')" + }] + }, + { + "name": "start", + "value": null, + "on": [{ + "events": "pointerdown", + "update": "invert(cloned, xy())" + }] + }, + { + "name": "drag", "value": null, + "on": [{ + "events": "[pointerdown, window:pointerup] > window:pointermove", + "update": "invert(cloned, xy())" + }] + }, + { + "name": "delta", "value": null, + "on": [{ + "events": { "signal": "drag" }, + "update": "[drag[0] - start[0], start[1] - drag[1]]" + }] + }, + { + "name": "rotateX", "value": 0, + "on": [{ + "events": { "signal": "delta" }, + "update": "angles[0] + delta[0]" + }] + }, + { + "name": "centerY", "value": this.mapCenter.location.lat, + "on": [{ + "events": { "signal": "delta" }, + "update": "clamp(angles[1] + delta[1], -60, 60)" + }] + } + ], + + "projections": [ + { + "name": "projection", + "type": "mercator", + "scale": { "signal": "scale" }, + "rotate": [{ "signal": "rotateX" }, 0, 0], + "center": [this.mapCenter.location.lon, { "signal": "centerY" }], + "translate": [{ "signal": "tx" }, { "signal": "ty" }], + } + ], + + "data": [ + { + "name": "world", + "url": "assets/world-atlas/land-110m.json", + "format": { + "type": "topojson", + "mesh": "land", + "filter": "exterior" + } + }, + { + "name": "points", + "format": { "type": "json" }, + "values": this.results + } + ], + + "marks": [ + { + "type": "shape", + "from": { "data": "world" }, + "encode": { + "enter": { + "strokeWidth": { "value": 0.75 }, + "fill": { "value": "#E5D3B3" }, + "stroke": { "value": "#BDAE8A" }, + } + }, + "transform": [ + { "type": "geoshape", "projection": "projection" } + ] + }, + { + "type": "shape", + "from": { "data": "points" }, + "encode": { + "enter": { + "width": { "value": 3 }, + "height": { "value": 3 }, + "fill": { "value": "#303F9F" }, + "stroke": { "value": "grey" }, + "tooltip": { "field": "properties.id" }, + "href": { "signal": "'/document/' + corpusName + '/' + datum.properties.id" }, + }, + }, + "transform": [ + { + "type": "geoshape", + "projection": "projection", + } + ] + } + ] + }; + } + + + async renderChart(): Promise { + const spec = this.getVegaSpec(); + const aspectRatio = 2 / 3; + const width = this.vegaMap.nativeElement.offsetWidth; + const height = width * aspectRatio; + + try { + await embed(this.vegaMap.nativeElement, spec, { + mode: 'vega', + renderer: 'canvas', + width: width, + height: height, + actions: false, + tooltip: true, + }); + } catch (error) { + this.emitError(error); + } + } + + emitError(error: { message: string }) { + this.mapError.emit(error?.message); } - } - - loadData() { - showLoading( - this.isLoading$, - this.visualizationService - .getGeoData( - this.visualizedField.name, - this.queryModel, - this.corpus, - ) - .then(geoData => { - this.results = geoData; - }) - .catch(this.emitError.bind(this)) - ); - } - - makeChart(geoData: GeoDocument[]) { - } - - - emitError(error: { message: string }) { - this.mapError.emit(error?.message); - } } diff --git a/frontend/src/app/visualization/ngram/joyplot/joyplot.component.ts b/frontend/src/app/visualization/ngram/joyplot/joyplot.component.ts index 0eecfa738..f57245da8 100644 --- a/frontend/src/app/visualization/ngram/joyplot/joyplot.component.ts +++ b/frontend/src/app/visualization/ngram/joyplot/joyplot.component.ts @@ -1,8 +1,8 @@ import { Component, Input, OnChanges, SimpleChanges } from '@angular/core'; import { Chart, ChartData, ChartOptions } from 'chart.js'; import * as _ from 'lodash'; -import { NgramResults } from '../../../models'; -import { selectColor } from '../../../utils/select-color'; +import { NgramResults } from '@models'; +import { selectColor } from '@utils/select-color'; @Component({ selector: 'ia-joyplot', diff --git a/frontend/src/app/visualization/ngram/ngram.component.spec.ts b/frontend/src/app/visualization/ngram/ngram.component.spec.ts index 3cac77151..60e44a175 100644 --- a/frontend/src/app/visualization/ngram/ngram.component.spec.ts +++ b/frontend/src/app/visualization/ngram/ngram.component.spec.ts @@ -1,12 +1,12 @@ import { ComponentFixture, TestBed, waitForAsync } from '@angular/core/testing'; import { By } from '@angular/platform-browser'; -import { QueryModel } from '../../models'; +import { QueryModel } from '@models'; import { mockCorpus } from '../../../mock-data/corpus'; import { MockCorpusResponse } from '../../../mock-data/corpus-response'; import { commonTestBed } from '../../common-test-bed'; import { NgramComponent } from './ngram.component'; -import { ApiService } from '../../services'; +import { ApiService } from '@services'; import { ApiServiceMock } from '../../../mock-data/api'; import { Subject } from 'rxjs'; diff --git a/frontend/src/app/visualization/ngram/ngram.component.ts b/frontend/src/app/visualization/ngram/ngram.component.ts index fccfdf264..43f9d1887 100644 --- a/frontend/src/app/visualization/ngram/ngram.component.ts +++ b/frontend/src/app/visualization/ngram/ngram.component.ts @@ -3,10 +3,22 @@ import { ActivatedRoute, ParamMap, Params, Router } from '@angular/router'; import * as _ from 'lodash'; import { Subject } from 'rxjs'; -import { formIcons } from '../../shared/icons'; -import { Corpus, FreqTableHeaders, QueryModel, - CorpusField, NgramResults, NgramParameters, SuccessfulTask } from '../../models'; -import { ApiService, NotificationService, ParamService, VisualizationService } from '../../services'; +import { formIcons } from '@shared/icons'; +import { + Corpus, + FreqTableHeaders, + QueryModel, + CorpusField, + NgramResults, + NgramParameters, + SuccessfulTask, +} from '@models'; +import { + ApiService, + NotificationService, + ParamService, + VisualizationService, +} from '@services'; import { ParamDirective } from '../../param/param-directive'; @Component({ diff --git a/frontend/src/app/visualization/visualization-footer/palette-select/palette-select.component.ts b/frontend/src/app/visualization/visualization-footer/palette-select/palette-select.component.ts index e80350d1a..86b99f6a8 100644 --- a/frontend/src/app/visualization/visualization-footer/palette-select/palette-select.component.ts +++ b/frontend/src/app/visualization/visualization-footer/palette-select/palette-select.component.ts @@ -1,6 +1,6 @@ import { Component, EventEmitter, Output } from '@angular/core'; -import { PALETTES } from '../../../utils/select-color'; -import { visualizationIcons } from '../../../shared/icons'; +import { PALETTES } from '@utils/select-color'; +import { visualizationIcons } from '@shared/icons'; @Component({ selector: 'ia-palette-select', diff --git a/frontend/src/app/visualization/visualization-footer/visualization-footer.component.ts b/frontend/src/app/visualization/visualization-footer/visualization-footer.component.ts index 3ec1ff924..e9880dc8d 100644 --- a/frontend/src/app/visualization/visualization-footer/visualization-footer.component.ts +++ b/frontend/src/app/visualization/visualization-footer/visualization-footer.component.ts @@ -1,9 +1,9 @@ import { Component, Input, OnInit, Output } from '@angular/core'; import { BehaviorSubject } from 'rxjs'; -import { DialogService, NotificationService } from '../../services'; +import { DialogService, NotificationService } from '@services'; import * as htmlToImage from 'html-to-image'; -import { PALETTES } from '../../utils/select-color'; -import { actionIcons } from '../../shared/icons'; +import { PALETTES } from '@utils/select-color'; +import { actionIcons } from '@shared/icons'; @Component({ selector: 'ia-visualization-footer', diff --git a/frontend/src/app/visualization/visualization.component.html b/frontend/src/app/visualization/visualization.component.html index befb6bd82..2ea328485 100644 --- a/frontend/src/app/visualization/visualization.component.html +++ b/frontend/src/app/visualization/visualization.component.html @@ -61,13 +61,15 @@
    -
    -
    diff --git a/frontend/src/app/visualization/visualization.component.ts b/frontend/src/app/visualization/visualization.component.ts index d667e0b21..820fd0fb2 100644 --- a/frontend/src/app/visualization/visualization.component.ts +++ b/frontend/src/app/visualization/visualization.component.ts @@ -6,10 +6,13 @@ import { SimpleChanges, } from '@angular/core'; import * as _ from 'lodash'; -import { Corpus, CorpusField, QueryModel } from '../models/index'; -import { actionIcons, visualizationIcons } from '../shared/icons'; +import { Corpus, CorpusField, QueryModel } from '@models/index'; +import { actionIcons, visualizationIcons } from '@shared/icons'; import { RouterStoreService } from '../store/router-store.service'; -import { VisualizationOption, VisualizationSelector } from '../models/visualization-selector'; +import { + VisualizationOption, + VisualizationSelector, +} from '@models/visualization-selector'; import { Observable, Subject, merge } from 'rxjs'; import { map } from 'rxjs/operators'; diff --git a/frontend/src/app/visualization/visualization.module.ts b/frontend/src/app/visualization/visualization.module.ts index 511353c59..85d879063 100644 --- a/frontend/src/app/visualization/visualization.module.ts +++ b/frontend/src/app/visualization/visualization.module.ts @@ -8,8 +8,8 @@ import { DialogService, SearchService, VisualizationService, -} from '../services'; -import { SharedModule } from '../shared/shared.module'; +} from '@services'; +import { SharedModule } from '@shared/shared.module'; import { BarchartOptionsComponent } from './barchart/barchart-options.component'; import { HistogramComponent } from './barchart/histogram.component'; import { TermComparisonEditorComponent } from './barchart/term-comparison-editor/term-comparison-editor.component'; diff --git a/frontend/src/app/visualization/wordcloud/wordcloud.component.ts b/frontend/src/app/visualization/wordcloud/wordcloud.component.ts index 53cacee52..0e8d6baed 100644 --- a/frontend/src/app/visualization/wordcloud/wordcloud.component.ts +++ b/frontend/src/app/visualization/wordcloud/wordcloud.component.ts @@ -3,13 +3,24 @@ import { } from '@angular/core'; -import { MostFrequentWordsResult, QueryModel, FreqTableHeaders } from '../../models/index'; -import { VisualizationService } from '../../services/visualization.service'; -import { Chart, ChartData, ChartDataset, ChartOptions, ScriptableContext, TooltipItem } from 'chart.js'; +import { + MostFrequentWordsResult, + QueryModel, + FreqTableHeaders, +} from '@models/index'; +import { VisualizationService } from '@services/visualization.service'; +import { + Chart, + ChartData, + ChartDataset, + ChartOptions, + ScriptableContext, + TooltipItem, +} from 'chart.js'; import { WordCloudChart } from 'chartjs-chart-wordcloud'; import * as _ from 'lodash'; -import { selectColor } from '../../utils/select-color'; -import { FrequentWordsResults } from '../../models/frequent-words'; +import { selectColor } from '@utils/select-color'; +import { FrequentWordsResults } from '@models/frequent-words'; import { RouterStoreService } from '../../store/router-store.service'; // maximum font size in px diff --git a/frontend/src/app/word-models/query-feedback/query-feedback.component.ts b/frontend/src/app/word-models/query-feedback/query-feedback.component.ts index 705241169..6621f0dc1 100644 --- a/frontend/src/app/word-models/query-feedback/query-feedback.component.ts +++ b/frontend/src/app/word-models/query-feedback/query-feedback.component.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core'; import * as _ from 'lodash'; -import { QueryFeedback } from '../../models'; +import { QueryFeedback } from '@models'; @Component({ selector: 'ia-query-feedback', diff --git a/frontend/src/app/word-models/related-words/related-words.component.ts b/frontend/src/app/word-models/related-words/related-words.component.ts index 963c16ab2..43482a24c 100644 --- a/frontend/src/app/word-models/related-words/related-words.component.ts +++ b/frontend/src/app/word-models/related-words/related-words.component.ts @@ -2,11 +2,11 @@ import { Component, EventEmitter, HostBinding, Input, OnChanges, Output, SimpleC import { ActivatedRoute, ParamMap, Params, Router } from '@angular/router'; import { BehaviorSubject } from 'rxjs'; import * as _ from 'lodash'; -import { showLoading } from '../../utils/utils'; -import { Corpus, WordSimilarity } from '../../models'; -import { ParamService, WordmodelsService } from '../../services/index'; +import { showLoading } from '@utils/utils'; +import { Corpus, WordSimilarity } from '@models'; +import { ParamService, WordmodelsService } from '@services/index'; import { ParamDirective } from '../../param/param-directive'; -import { formIcons } from '../../shared/icons'; +import { formIcons } from '@shared/icons'; @Component({ diff --git a/frontend/src/app/word-models/similarity-chart/similarity-chart.component.ts b/frontend/src/app/word-models/similarity-chart/similarity-chart.component.ts index 8b825f391..c9b544f88 100644 --- a/frontend/src/app/word-models/similarity-chart/similarity-chart.component.ts +++ b/frontend/src/app/word-models/similarity-chart/similarity-chart.component.ts @@ -3,8 +3,8 @@ import { Chart, ChartData, ChartOptions, ChartType, Filler, TooltipItem } from ' import Zoom from 'chartjs-plugin-zoom'; import * as _ from 'lodash'; import { BehaviorSubject } from 'rxjs'; -import { selectColor } from '../../utils/select-color'; -import { FreqTableHeaders, WordSimilarity } from '../../models'; +import { selectColor } from '@utils/select-color'; +import { FreqTableHeaders, WordSimilarity } from '@models'; /** * Child component of the related words and compare similarity graphs. diff --git a/frontend/src/app/word-models/word-models.component.html b/frontend/src/app/word-models/word-models.component.html index ca5255730..b0601ae00 100644 --- a/frontend/src/app/word-models/word-models.component.html +++ b/frontend/src/app/word-models/word-models.component.html @@ -34,13 +34,21 @@
    -
    -
    @@ -55,7 +63,7 @@ diff --git a/frontend/src/app/word-models/word-models.component.ts b/frontend/src/app/word-models/word-models.component.ts index 3951aa885..69dc99f3d 100644 --- a/frontend/src/app/word-models/word-models.component.ts +++ b/frontend/src/app/word-models/word-models.component.ts @@ -1,12 +1,16 @@ import { Component, ElementRef, HostListener, ViewChild } from '@angular/core'; import { ActivatedRoute, ParamMap, Router } from '@angular/router'; -import * as _ from 'lodash'; -import { Corpus, QueryFeedback, User, WordInModelResult } from '../models'; -import { AuthService, CorpusService, ParamService, WordmodelsService } from '../services'; -import { ParamDirective } from '../param/param-directive'; -import { visualizationIcons } from '../shared/icons'; import { Title } from '@angular/platform-browser'; +import { + AuthService, + CorpusService, + ParamService, + WordmodelsService, +} from '@services'; +import { visualizationIcons } from '@shared/icons'; +import { Corpus, QueryFeedback, WordInModelResult } from '@models'; +import { ParamDirective } from '../param/param-directive'; @Component({ selector: 'ia-word-models', diff --git a/frontend/src/app/word-models/word-models.module.ts b/frontend/src/app/word-models/word-models.module.ts index 7b8fb4147..1b98723c9 100644 --- a/frontend/src/app/word-models/word-models.module.ts +++ b/frontend/src/app/word-models/word-models.module.ts @@ -1,14 +1,14 @@ import { NgModule } from '@angular/core'; -import { WordModelsComponent } from './word-models.component'; +import { WordmodelsService } from '@services'; +import { SharedModule } from '@shared/shared.module'; +import { CorpusModule } from '../corpus-header/corpus.module'; import { VisualizationModule } from '../visualization/visualization.module'; import { QueryFeedbackComponent } from './query-feedback/query-feedback.component'; import { RelatedWordsComponent } from './related-words/related-words.component'; -import { WordSimilarityComponent } from './word-similarity/word-similarity.component'; import { SimilarityChartComponent } from './similarity-chart/similarity-chart.component'; -import { WordmodelsService } from '../services'; import { TimeIntervalSliderComponent } from './similarity-chart/time-interval-slider/time-interval-slider.component'; -import { SharedModule } from '../shared/shared.module'; -import { CorpusModule } from '../corpus-header/corpus.module'; +import { WordModelsComponent } from './word-models.component'; +import { WordSimilarityComponent } from './word-similarity/word-similarity.component'; @NgModule({ diff --git a/frontend/src/app/word-models/word-similarity/word-similarity.component.html b/frontend/src/app/word-models/word-similarity/word-similarity.component.html index 4ed2c52eb..c81af757d 100644 --- a/frontend/src/app/word-models/word-similarity/word-similarity.component.html +++ b/frontend/src/app/word-models/word-similarity/word-similarity.component.html @@ -2,8 +2,7 @@
    + [termLimit]="comparisonTermLimit">
    diff --git a/frontend/src/app/word-models/word-similarity/word-similarity.component.ts b/frontend/src/app/word-models/word-similarity/word-similarity.component.ts index 2b0018c51..288a5570d 100644 --- a/frontend/src/app/word-models/word-similarity/word-similarity.component.ts +++ b/frontend/src/app/word-models/word-similarity/word-similarity.component.ts @@ -1,19 +1,20 @@ -import { Component, EventEmitter, HostBinding, Input, OnChanges, Output, SimpleChanges } from '@angular/core'; +import { Component, EventEmitter, HostBinding, Input, OnChanges, OnDestroy, Output, SimpleChanges } from '@angular/core'; import * as _ from 'lodash'; import { BehaviorSubject } from 'rxjs'; -import { showLoading } from '../../utils/utils'; -import { Corpus, WordSimilarity } from '../../models'; -import { WordmodelsService } from '../../services'; +import { RouterStoreService } from '../../store/router-store.service'; +import { ComparedQueries } from '@models/compared-queries'; +import { showLoading } from '@utils/utils'; +import { Corpus, WordSimilarity } from '@models'; +import { WordmodelsService } from '@services'; @Component({ selector: 'ia-word-similarity', templateUrl: './word-similarity.component.html', styleUrls: ['./word-similarity.component.scss'], }) -export class WordSimilarityComponent implements OnChanges { +export class WordSimilarityComponent implements OnChanges, OnDestroy { @HostBinding('style.display') display = 'block'; // needed for loading spinner positioning - @Input() queryText: string; @Input() corpus: Corpus; @Input() asTable: boolean; @Input() palette: string[]; @@ -23,27 +24,41 @@ export class WordSimilarityComponent implements OnChanges { isLoading$ = new BehaviorSubject(false); comparisonTermLimit = Infinity; - comparisonTerms: string[] = []; + + comparedQueries: ComparedQueries; results: WordSimilarity[][]; timeIntervals: string[]; data: WordSimilarity[]; - constructor(private wordModelsService: WordmodelsService) {} + constructor( + private wordModelsService: WordmodelsService, + private routerStoreService: RouterStoreService) { + this.comparedQueries = new ComparedQueries(this.routerStoreService); + this.comparedQueries.allQueries$.subscribe(this.onTermsUpdate.bind(this)) + } @HostBinding('class.is-loading') get isLoading() { return this.isLoading$.value; } + get queryText(): string { + return this.comparedQueries.state$.value.primary; + } + + get comparisonTerms(): string[] { + return this.comparedQueries.state$.value.compare; + } + get tableFileName(): string { return `word similarity - ${this.queryText} - ${this.corpus?.title}`; } ngOnChanges(changes: SimpleChanges): void { if ( - (changes.queryText || changes.corpus) && + (changes.corpus) && this.comparisonTerms.length ) { this.getData(); @@ -54,9 +69,16 @@ export class WordSimilarityComponent implements OnChanges { } } - updateComparisonTerms(terms: string[] = []) { - this.comparisonTerms = terms; - this.getData(); + ngOnDestroy(): void { + this.comparedQueries.complete(); + } + + onTermsUpdate() { + if (this.corpus && this.comparisonTerms.length >= 1) { + this.getData(); + } else { + this.clearData(); + } } getData(): void { @@ -76,6 +98,12 @@ export class WordSimilarityComponent implements OnChanges { .catch(this.onError.bind(this)); } + clearData() { + this.results = undefined; + this.timeIntervals = undefined; + this.data = undefined; + } + getTimePoints(points: WordSimilarity[]) { return points.map((point) => point.time); } diff --git a/frontend/src/assets/world-atlas/LICENSE b/frontend/src/assets/world-atlas/LICENSE new file mode 100644 index 000000000..a24a8ec02 --- /dev/null +++ b/frontend/src/assets/world-atlas/LICENSE @@ -0,0 +1,13 @@ +Copyright 2013-2019 Michael Bostock + +Permission to use, copy, modify, and/or distribute this software for any purpose +with or without fee is hereby granted, provided that the above copyright notice +and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF +THIS SOFTWARE. diff --git a/frontend/src/assets/world-atlas/land-110m.json b/frontend/src/assets/world-atlas/land-110m.json new file mode 100644 index 000000000..fbd897cb1 --- /dev/null +++ b/frontend/src/assets/world-atlas/land-110m.json @@ -0,0 +1 @@ +{"type":"Topology","objects":{"land":{"type":"GeometryCollection","geometries":[{"type":"MultiPolygon","arcs":[[[0]],[[1]],[[2]],[[3]],[[4]],[[5]],[[6]],[[7]],[[8]],[[9]],[[10]],[[11]],[[12]],[[13]],[[14]],[[15]],[[16,18]],[[19]],[[20]],[[21]],[[22]],[[23]],[[24]],[[25]],[[26]],[[27]],[[28]],[[29]],[[30]],[[31]],[[32]],[[33]],[[34]],[[35]],[[36]],[[37]],[[38]],[[39]],[[40]],[[41]],[[42]],[[43]],[[44]],[[45]],[[46]],[[47]],[[48]],[[49]],[[50]],[[51]],[[52]],[[53]],[[54]],[[55]],[[56]],[[57]],[[58]],[[59]],[[60]],[[61]],[[62]],[[63]],[[64]],[[65]],[[66]],[[67]],[[68]],[[69]],[[70]],[[71]],[[72]],[[73]],[[74]],[[75]],[[76]],[[77]],[[78]],[[79]],[[80]],[[81]],[[82]],[[83]],[[84]],[[85]],[[86]],[[87]],[[88]],[[89]],[[90]],[[91]],[[92,115],[114]],[[94]],[[95]],[[96]],[[97]],[[98]],[[99]],[[100]],[[101]],[[102]],[[103]],[[104]],[[105]],[[106]],[[107]],[[108]],[[109]],[[110]],[[111]],[[112]],[[113]],[[116]],[[117]],[[118]],[[119]],[[120]],[[121]],[[122]],[[123]],[[124]],[[125]],[[126]],[[127]],[[128]],[[129]]]}]}},"arcs":[[[33452,3290],[-82,-301],[-81,-266],[-582,81],[-621,-35],[-348,197],[0,23],[-152,174],[625,-23],[599,-58],[207,243],[147,208],[288,-243]],[[5775,3611],[-533,-81],[-364,208],[-163,209],[-11,35],[-180,162],[169,220],[517,-93],[277,-185],[212,-209],[76,-266]],[[37457,4468],[342,-255],[120,-359],[33,-254],[11,-301],[-430,-186],[-452,-150],[-522,-139],[-582,-116],[-658,35],[-365,197],[49,243],[593,162],[239,197],[174,254],[126,220],[168,209],[180,243],[141,0],[414,127],[419,-127]],[[16330,7154],[359,-93],[332,104],[-158,-208],[-261,-151],[-386,47],[-278,208],[60,197],[332,-104]],[[15122,7165],[425,-231],[-164,23],[-359,58],[-381,162],[202,127],[277,-139]],[[22505,8080],[305,-81],[304,69],[163,-335],[-217,46],[-337,-23],[-343,23],[-376,-35],[-283,116],[-146,243],[174,104],[353,-81],[403,-46]],[[30985,8657],[33,-266],[-49,-231],[-76,-220],[-326,-81],[-311,-116],[-364,11],[136,232],[-327,-81],[-310,-81],[-212,174],[-16,243],[305,231],[190,70],[321,-23],[82,301],[16,219],[-6,475],[158,278],[256,93],[147,-220],[65,-220],[120,-267],[92,-254],[76,-267]],[[0,529],[16,-5],[245,344],[501,-185],[32,21],[78,49],[94,61],[81,52],[41,26],[41,-1],[29,-10],[402,-246],[352,246],[63,34],[816,104],[265,-138],[130,-71],[419,-196],[789,-151],[625,-185],[1072,-139],[800,162],[1181,-116],[669,-185],[734,174],[773,162],[60,278],[-1094,23],[-898,139],[-234,231],[-745,128],[49,266],[103,243],[104,220],[-55,243],[-462,162],[-212,209],[-430,185],[675,-35],[642,93],[402,-197],[495,173],[457,220],[223,197],[-98,243],[-359,162],[-408,174],[-571,35],[-500,81],[-539,58],[-180,220],[-359,185],[-217,208],[-87,672],[136,-58],[250,-185],[457,58],[441,81],[228,-255],[441,58],[370,127],[348,162],[315,197],[419,58],[-11,220],[-97,220],[81,208],[359,104],[163,-196],[425,115],[321,151],[397,12],[375,57],[376,139],[299,128],[337,127],[218,-35],[190,-46],[414,81],[370,-104],[381,11],[364,81],[375,-57],[414,-58],[386,23],[403,-12],[413,-11],[381,23],[283,174],[337,92],[349,-127],[331,104],[300,208],[179,-185],[98,-208],[180,-197],[288,174],[332,-220],[375,-70],[321,-162],[392,35],[354,104],[418,-23],[376,-81],[381,-104],[147,254],[-180,197],[-136,209],[-359,46],[-158,220],[-60,220],[-98,440],[213,-81],[364,-35],[359,35],[327,-93],[283,-174],[119,-208],[376,-35],[359,81],[381,116],[342,70],[283,-139],[370,46],[239,451],[224,-266],[321,-104],[348,58],[228,-232],[365,-23],[337,-69],[332,-128],[218,220],[108,209],[278,-232],[381,58],[283,-127],[190,-197],[370,58],[288,127],[283,151],[337,81],[392,69],[354,81],[272,127],[163,186],[65,254],[-32,244],[-87,231],[-98,232],[-87,231],[-71,209],[-16,231],[27,232],[130,220],[109,243],[44,231],[-55,255],[-32,232],[136,266],[152,173],[180,220],[190,186],[223,173],[109,255],[152,162],[174,151],[267,34],[174,186],[196,115],[228,70],[202,150],[157,186],[218,69],[163,-151],[-103,-196],[-283,-174],[-120,-127],[-206,92],[-229,-58],[-190,-139],[-202,-150],[-136,-174],[-38,-231],[17,-220],[130,-197],[-190,-139],[-261,-46],[-153,-197],[-163,-185],[-174,-255],[-44,-220],[98,-243],[147,-185],[229,-139],[212,-185],[114,-232],[60,-220],[82,-232],[130,-196],[82,-220],[38,-544],[81,-220],[22,-232],[87,-231],[-38,-313],[-152,-243],[-163,-197],[-370,-81],[-125,-208],[-169,-197],[-419,-220],[-370,-93],[-348,-127],[-376,-128],[-223,-243],[-446,-23],[-489,23],[-441,-46],[-468,0],[87,-232],[424,-104],[311,-162],[174,-208],[-310,-185],[-479,58],[-397,-151],[-17,-243],[-11,-232],[327,-196],[60,-220],[353,-220],[588,-93],[500,-162],[398,-185],[506,-186],[690,-92],[681,-162],[473,-174],[517,-197],[272,-278],[136,-220],[337,209],[457,173],[484,186],[577,150],[495,162],[691,12],[680,-81],[560,-139],[180,255],[386,173],[702,12],[550,127],[522,128],[577,81],[614,104],[430,150],[-196,209],[-119,208],[0,220],[-539,-23],[-571,-93],[-544,0],[-77,220],[39,440],[125,128],[397,138],[468,139],[337,174],[337,174],[251,231],[380,104],[376,81],[190,47],[430,23],[408,81],[343,116],[337,139],[305,139],[386,185],[245,197],[261,173],[82,232],[-294,139],[98,243],[185,185],[288,116],[305,139],[283,185],[217,232],[136,277],[202,163],[331,-35],[136,-197],[332,-23],[11,220],[142,231],[299,-58],[71,-220],[331,-34],[360,104],[348,69],[315,-34],[120,-243],[305,196],[283,105],[315,81],[310,81],[283,139],[310,92],[240,128],[168,208],[207,-151],[288,81],[202,-277],[157,-209],[316,116],[125,232],[283,162],[365,-35],[108,-220],[229,220],[299,69],[326,23],[294,-11],[310,-70],[300,-34],[130,-197],[180,-174],[304,104],[327,24],[315,0],[310,11],[278,81],[294,70],[245,162],[261,104],[283,58],[212,162],[152,324],[158,197],[288,-93],[109,-208],[239,-139],[289,46],[196,-208],[206,-151],[283,139],[98,255],[250,104],[289,197],[272,81],[326,116],[218,127],[228,139],[218,127],[261,-69],[250,208],[180,162],[261,-11],[229,139],[54,208],[234,162],[228,116],[278,93],[256,46],[244,-35],[262,-58],[223,-162],[27,-254],[245,-197],[168,-162],[332,-70],[185,-162],[229,-162],[266,-35],[223,116],[240,243],[261,-127],[272,-70],[261,-69],[272,-46],[277,0],[229,-614],[-11,-150],[-33,-267],[-266,-150],[-218,-220],[38,-232],[310,12],[-38,-232],[-141,-220],[-131,-243],[212,-185],[321,-58],[321,104],[153,232],[92,220],[153,185],[174,174],[70,208],[147,289],[174,58],[316,24],[277,69],[283,93],[136,231],[82,220],[190,220],[272,151],[234,115],[153,197],[157,104],[202,93],[277,-58],[250,58],[272,69],[305,-34],[201,162],[142,393],[103,-162],[131,-278],[234,-115],[266,-47],[267,70],[283,-46],[261,-12],[174,58],[234,-35],[212,-127],[250,81],[300,0],[255,81],[289,-81],[185,197],[141,196],[191,163],[348,439],[179,-81],[212,-162],[185,-208],[354,-359],[272,-12],[256,0],[299,70],[299,81],[229,162],[190,174],[310,23],[207,127],[218,-116],[141,-185],[196,-185],[305,23],[190,-150],[332,-151],[348,-58],[288,47],[218,185],[185,185],[250,46],[251,-81],[288,-58],[261,93],[250,0],[245,-58],[256,-58],[250,104],[299,93],[283,23],[316,0],[255,58],[251,46],[76,290],[11,243],[174,-162],[49,-266],[92,-244],[115,-196],[234,-105],[315,35],[365,12],[250,35],[364,0],[262,11],[364,-23],[310,-46],[196,-186],[-54,-220],[179,-173],[299,-139],[310,-151],[360,-104],[375,-92],[283,-93],[315,-12],[180,197],[245,-162],[212,-185],[245,-139],[337,-58],[321,-69],[136,-232],[316,-139],[212,-208],[310,-93],[321,12],[299,-35],[332,12],[332,-47],[310,-81],[288,-139],[289,-116],[195,-173],[-32,-232],[-147,-208],[-125,-266],[-98,-209],[-131,-243],[-364,-93],[-163,-208],[-360,-127],[-125,-232],[-190,-220],[-201,-185],[-115,-243],[-70,-220],[-28,-266],[6,-220],[158,-232],[60,-220],[130,-208],[517,-81],[109,-255],[-501,-93],[-424,-127],[-528,-23],[-234,-336],[-49,-278],[-119,-220],[-147,-220],[370,-196],[141,-244],[239,-219],[338,-197],[386,-186],[419,-185],[636,-185],[142,-289],[800,-128],[53,-45],[208,-175],[767,151],[636,-186],[-99520,-142]],[[31180,18764],[361,-355],[389,-147],[-125,-296],[-264,-29],[-141,208],[-92,-239],[-238,-183],[-301,67],[-202,177],[-291,86],[-350,330],[-283,317],[-383,662],[229,-124],[390,-395],[369,-212],[143,271],[90,405],[256,244],[198,-70],[106,-274],[139,-443]],[[33736,20389],[222,-266],[-83,-207],[-375,-177],[-125,207],[-236,-266],[-139,266],[333,354],[236,-148],[167,237]],[[69522,21210],[-427,-38],[-7,314],[41,244],[19,121],[179,-186],[263,-74],[9,-112],[-77,-269]],[[90387,26479],[269,-204],[151,81],[217,113],[166,-39],[20,-702],[-95,-203],[-29,-476],[-97,162],[-193,-412],[-57,32],[-171,19],[-171,505],[-38,390],[-160,515],[7,271],[181,-52]],[[98060,26404],[63,-244],[198,239],[80,-249],[0,-249],[-103,-274],[-182,-435],[-142,-238],[103,-284],[-214,-7],[-238,-223],[-75,-387],[-157,-597],[-219,-264],[-138,-169],[-256,13],[-180,194],[-302,42],[-46,217],[149,438],[349,583],[179,111],[200,225],[238,310],[167,306],[123,441],[106,149],[41,330],[195,273],[61,-251]],[[98502,29218],[202,-622],[5,403],[126,-161],[41,-447],[224,-192],[188,-48],[158,226],[141,-69],[-67,-524],[-85,-345],[-212,12],[-74,-179],[26,-254],[-41,-110],[-105,-319],[-138,-404],[-214,-236],[-48,155],[-116,85],[160,486],[-91,326],[-299,236],[8,214],[201,206],[47,455],[-13,382],[-113,396],[8,104],[-133,244],[-218,523],[-117,418],[104,46],[151,-328],[216,-153],[78,-526]],[[96421,37487],[-105,-142],[-153,160],[-199,266],[-179,313],[-184,416],[-38,201],[119,-9],[156,-201],[122,-200],[89,-166],[228,-366],[144,-272]],[[99547,40335],[96,-171],[-46,-308],[-172,-81],[-153,73],[-27,260],[107,203],[126,-74],[69,98]],[[0,40798],[99822,-145],[-177,-124],[-36,220],[139,121],[88,33],[-99836,184]],[[0,41087],[0,-289]],[[0,41087],[57,27],[-34,-284],[-23,-32]],[[96623,40851],[-92,-78],[-93,259],[10,158],[175,-339]],[[96418,41756],[45,-476],[-75,74],[-58,-32],[-39,163],[-6,453],[133,-182]],[[63904,42571],[45,-711],[72,-276],[-28,-284],[-49,-174],[-94,347],[-53,-175],[53,-438],[-24,-250],[-77,-137],[-18,-500],[-109,-689],[-137,-814],[-172,-1120],[-106,-821],[-125,-685],[-226,-140],[-243,-250],[-160,151],[-220,211],[-77,312],[-18,524],[-98,471],[-26,425],[50,426],[128,102],[1,197],[133,447],[25,377],[-65,280],[-52,372],[-23,544],[97,331],[38,375],[138,22],[155,121],[103,107],[122,7],[158,337],[229,364],[83,297],[-38,253],[118,-71],[153,410],[6,356],[92,264],[96,-254],[74,-251],[69,-390]],[[89877,42448],[100,-464],[179,223],[92,-250],[133,-231],[-29,-262],[60,-506],[42,-295],[70,-72],[75,-505],[-27,-307],[90,-400],[301,-309],[197,-281],[186,-257],[-37,-143],[159,-371],[108,-639],[111,130],[113,-256],[68,91],[48,-626],[197,-363],[129,-226],[217,-478],[78,-475],[7,-337],[-19,-365],[132,-502],[-16,-523],[-48,-274],[-75,-527],[6,-339],[-55,-423],[-123,-538],[-205,-290],[-102,-458],[-93,-292],[-82,-510],[-107,-294],[-70,-442],[-36,-407],[14,-187],[-159,-205],[-311,-22],[-257,-242],[-127,-229],[-168,-254],[-230,262],[-170,104],[43,308],[-152,-112],[-243,-428],[-240,160],[-158,94],[-159,42],[-269,171],[-179,364],[-52,449],[-64,298],[-137,240],[-267,71],[91,287],[-67,438],[-136,-408],[-247,-109],[146,327],[42,341],[107,289],[-22,438],[-226,-504],[-174,-202],[-106,-470],[-217,243],[9,313],[-174,429],[-147,221],[52,137],[-356,358],[-195,17],[-267,287],[-498,-56],[-359,-211],[-317,-197],[-265,39],[-294,-303],[-241,-137],[-53,-309],[-103,-240],[-236,-15],[-174,-52],[-246,107],[-199,-64],[-191,-27],[-165,-315],[-81,26],[-140,-167],[-133,-187],[-203,23],[-186,0],[-295,377],[-149,113],[6,338],[138,81],[47,134],[-10,212],[34,411],[-31,350],[-147,598],[-45,337],[12,336],[-111,385],[-7,174],[-123,235],[-35,463],[-158,467],[-39,252],[122,-255],[-93,548],[137,-171],[83,-229],[-5,303],[-138,465],[-26,186],[-65,177],[31,341],[56,146],[38,295],[-29,346],[114,425],[21,-450],[118,406],[225,198],[136,252],[212,217],[126,46],[77,-73],[219,220],[168,66],[42,129],[74,54],[153,-14],[292,173],[151,262],[71,316],[163,300],[13,236],[7,321],[194,502],[117,-510],[119,118],[-99,279],[87,287],[122,-128],[34,449],[152,291],[67,233],[140,101],[4,165],[122,-69],[5,148],[122,85],[134,80],[205,-271],[155,-350],[173,-4],[177,-56],[-59,325],[133,473],[126,155],[-44,147],[121,338],[168,208],[142,-70],[234,111],[-5,302],[-204,195],[148,86],[184,-147],[148,-242],[234,-151],[79,60],[172,-182],[162,169],[105,-51],[65,113],[127,-292],[-74,-316],[-105,-239],[-96,-20],[32,-236],[-81,-295],[-99,-291],[20,-166],[221,-327],[214,-189],[143,-204],[201,-350],[78,1],[145,-151],[43,-183],[265,-200],[183,202],[55,317],[56,262],[34,324],[85,470],[-39,286],[20,171],[-32,339],[37,445],[53,120],[-43,197],[67,313],[52,325],[7,168],[104,222],[78,-289],[19,-371],[70,-71],[11,-249],[101,-300],[21,-335],[-10,-214]],[[95032,44386],[78,-203],[-194,4],[-106,363],[166,-142],[56,-22]],[[83531,44530],[-117,-11],[-368,414],[259,116],[146,-180],[97,-180],[-17,-159]],[[94680,44747],[-108,-14],[-170,60],[-58,91],[17,235],[183,-93],[91,-124],[45,-155]],[[94910,44908],[-42,-109],[-206,512],[-57,353],[94,0],[100,-473],[111,-283]],[[84565,44589],[-238,-130],[-33,71],[25,201],[119,360],[275,235],[32,139],[239,133],[194,20],[87,74],[105,-74],[-102,-160],[-289,-258],[-233,-170],[-181,-441]],[[82749,45797],[100,-158],[172,48],[69,-251],[-321,-119],[-193,-79],[-149,5],[95,340],[153,5],[74,209]],[[84139,45797],[-41,-328],[-417,-168],[-370,73],[0,216],[220,123],[174,-177],[185,45],[249,216]],[[94409,45654],[12,-119],[-218,251],[-152,212],[-104,197],[41,60],[128,-142],[228,-272],[65,-187]],[[93760,46238],[-56,-33],[-121,134],[-114,243],[14,99],[166,-250],[111,-193]],[[80172,46575],[533,-59],[61,244],[515,-284],[101,-383],[417,-108],[341,-351],[-317,-225],[-306,238],[-251,-16],[-288,44],[-260,106],[-322,225],[-204,59],[-116,-74],[-506,243],[-48,254],[-255,44],[191,564],[337,-35],[224,-231],[115,-45],[38,-210]],[[87423,46908],[-143,-402],[-27,445],[49,212],[58,200],[63,-173],[0,-282]],[[93299,46550],[-78,-59],[-120,227],[-122,375],[-59,450],[38,57],[30,-175],[84,-134],[135,-375],[131,-200],[-39,-166]],[[92217,47343],[-146,-48],[-44,-166],[-152,-144],[-142,-138],[-148,1],[-228,171],[-158,165],[23,183],[249,-86],[152,46],[42,283],[40,15],[27,-314],[158,45],[78,202],[155,211],[-30,348],[166,11],[56,-97],[-5,-327],[-93,-361]],[[85346,48536],[-104,-196],[-192,108],[-54,254],[281,29],[69,-195]],[[86241,48752],[101,-452],[-234,244],[-232,49],[-157,-39],[-192,21],[65,325],[344,24],[305,-172]],[[92538,47921],[-87,-157],[-52,348],[-65,229],[-126,193],[-158,252],[-200,174],[77,143],[150,-166],[94,-130],[117,-142],[111,-248],[106,-189],[33,-307]],[[87261,49899],[78,-955],[287,-354],[232,627],[319,356],[247,1],[238,-206],[206,-212],[298,-113],[482,-407],[513,-338],[192,-302],[154,-297],[43,-349],[462,-365],[68,-313],[-256,-64],[62,-393],[248,-388],[180,-627],[159,20],[-11,-262],[215,-100],[-84,-111],[295,-249],[-30,-171],[-184,-41],[-69,153],[-238,66],[-281,89],[-216,377],[-158,325],[-144,517],[-362,259],[-235,-169],[-170,-195],[35,-436],[-218,-203],[-155,99],[-288,25],[-247,485],[-282,118],[-69,-168],[-352,-18],[118,481],[175,164],[-72,642],[-134,496],[-538,500],[-229,50],[-417,546],[-82,-287],[-107,-52],[-63,216],[-1,257],[-212,290],[299,213],[198,-11],[-23,156],[-407,1],[-110,352],[-248,109],[-117,293],[374,143],[142,192],[446,-242],[44,-220]],[[84788,51419],[-223,-587],[-209,-113],[-267,115],[-463,-29],[-243,-85],[-39,-447],[248,-526],[150,268],[518,201],[-22,-272],[-121,86],[-121,-347],[-245,-229],[263,-757],[-50,-203],[249,-682],[-2,-388],[-148,-173],[-109,207],[134,484],[-273,-229],[-69,164],[36,228],[-200,346],[21,576],[-186,-179],[24,-689],[11,-846],[-176,-85],[-119,173],[79,544],[-43,570],[-117,4],[-86,405],[115,387],[40,469],[139,891],[58,243],[237,439],[217,-174],[350,-82],[319,25],[275,429],[48,-132]],[[85746,51249],[-15,-517],[-143,58],[-42,-359],[114,-312],[-78,-71],[-112,374],[-82,755],[56,472],[92,215],[20,-322],[164,-52],[26,-241]],[[79393,47122],[-308,-12],[-234,494],[-356,482],[-119,358],[-210,481],[-138,443],[-212,827],[-244,493],[-81,508],[-103,461],[-250,372],[-145,506],[-209,330],[-290,652],[-24,300],[178,-24],[430,-114],[246,-577],[215,-401],[153,-246],[263,-635],[283,-9],[233,-405],[161,-495],[211,-270],[-111,-482],[159,-205],[100,-15],[47,-412],[97,-330],[204,-52],[135,-374],[-70,-735],[-11,-914]],[[82742,51659],[312,-546],[-329,-70],[-93,-403],[12,-535],[-267,-404],[-7,-589],[-107,-903],[-41,210],[-316,-266],[-110,361],[-198,34],[-139,189],[-330,-212],[-101,285],[-182,-32],[-229,68],[-43,793],[-138,164],[-134,505],[-38,517],[32,548],[165,392],[204,-202],[214,110],[56,500],[119,112],[333,128],[199,467],[137,374],[110,221],[236,323],[214,411],[140,462],[112,2],[143,-299],[13,-257],[183,-165],[231,-177],[-20,-232],[-186,-29],[50,-289],[-205,-201],[-158,-533],[204,-560],[-48,-272]],[[85104,55551],[28,-392],[16,-332],[-94,-540],[-102,602],[-130,-300],[89,-435],[-79,-277],[-327,343],[-78,428],[84,280],[-176,280],[-87,-245],[-131,23],[-205,-330],[-46,173],[109,498],[175,166],[151,223],[98,-268],[212,162],[45,264],[196,15],[-16,457],[225,-280],[23,-297],[20,-218]],[[72560,54241],[-242,-135],[-132,470],[-49,849],[126,959],[192,-328],[129,-416],[134,-616],[-42,-615],[-116,-168]],[[33073,56553],[-232,-65],[-50,53],[81,163],[-6,233],[160,77],[58,-21],[-11,-440]],[[84439,56653],[-100,-195],[-87,-373],[-87,-175],[-171,409],[57,158],[70,165],[30,367],[153,35],[-44,-398],[205,570],[-26,-563]],[[82917,56084],[-369,-561],[136,414],[200,364],[167,409],[146,587],[49,-482],[-183,-325],[-146,-406]],[[83856,57606],[166,-183],[177,1],[-5,-247],[-129,-251],[-176,-178],[-10,275],[20,301],[-43,282]],[[84861,57766],[78,-660],[-214,157],[5,-199],[68,-364],[-132,-133],[-11,416],[-84,31],[-43,357],[163,-47],[-4,224],[-169,451],[266,-13],[77,-220]],[[83757,58301],[-74,-510],[-119,295],[-142,450],[238,-22],[97,-213]],[[83700,61512],[171,-168],[85,153],[26,-150],[-46,-245],[95,-423],[-73,-491],[-164,-196],[-43,-476],[62,-471],[147,-65],[123,70],[347,-328],[-27,-321],[91,-142],[-29,-272],[-216,290],[-103,310],[-71,-217],[-177,354],[-253,-87],[-138,130],[14,244],[87,151],[-83,136],[-36,-213],[-137,340],[-41,257],[-11,566],[112,-195],[29,925],[90,535],[169,-1]],[[31780,61349],[-71,-149],[-209,4],[-163,-21],[-16,253],[40,86],[227,-3],[142,-52],[50,-118]],[[28638,61137],[-84,-99],[-156,95],[-159,215],[34,135],[116,41],[64,-20],[187,-53],[147,-142],[46,-161],[-195,-11]],[[29839,62320],[241,-93],[34,101],[217,-3],[165,-152],[73,15],[50,-209],[152,11],[-9,-176],[124,-21],[136,-217],[-103,-240],[-132,128],[-127,-25],[-92,28],[-50,-107],[-106,-37],[-43,144],[-92,-85],[-111,-405],[-71,94],[-14,170],[-185,100],[-131,-41],[-169,43],[-130,-110],[-149,184],[24,190],[256,-82],[210,-47],[100,131],[-127,256],[2,226],[-175,92],[62,163],[170,-26]],[[80649,61615],[-240,-284],[-228,183],[-8,509],[137,267],[304,166],[159,-14],[62,-226],[-122,-260],[-64,-341]],[[6794,61855],[-41,-99],[-69,84],[8,165],[-46,216],[14,65],[48,97],[-19,116],[16,55],[21,-11],[107,-100],[49,-51],[45,-79],[71,-207],[-7,-33],[-108,-126],[-89,-92]],[[6645,62777],[-94,-43],[-47,125],[-32,48],[-3,37],[27,50],[99,-56],[73,-90],[-23,-71]],[[6456,63091],[-9,-63],[-149,17],[21,72],[137,-26]],[[6207,63177],[-15,-34],[-19,8],[-97,21],[-35,133],[-11,24],[74,82],[23,-38],[80,-196]],[[5737,63567],[-33,-58],[-93,107],[14,43],[43,58],[64,-12],[5,-138]],[[27867,64030],[110,-216],[260,66],[98,-138],[235,-366],[173,-267],[92,8],[165,-120],[-20,-167],[205,-24],[210,-242],[-33,-138],[-185,-75],[-187,-29],[-191,46],[-398,-57],[186,329],[-113,154],[-179,39],[-96,171],[-66,336],[-157,-23],[-259,159],[-83,124],[-362,91],[-97,115],[104,148],[-273,30],[-199,-307],[-115,-8],[-40,-144],[-138,-65],[-118,56],[146,183],[60,213],[126,131],[142,116],[210,56],[67,65],[240,-42],[219,-7],[261,-201]],[[28462,64617],[-68,-29],[-70,340],[-104,171],[60,375],[84,-23],[97,-491],[1,-343]],[[83659,64045],[-119,-485],[-146,499],[-32,438],[163,581],[223,447],[127,-176],[-49,-357],[-167,-947]],[[28383,66284],[-303,-95],[-19,219],[130,47],[184,-18],[8,-153]],[[28611,66290],[-48,-420],[-51,75],[4,309],[-124,234],[-1,67],[220,-265]],[[87399,70756],[35,-203],[-156,-357],[-114,189],[-143,-137],[-73,-346],[-181,168],[2,281],[154,352],[158,-68],[114,248],[204,-127]],[[59604,71655],[-188,-251],[21,-111],[8,-48],[-285,-240],[-136,77],[-64,237],[132,22],[19,3],[40,143],[200,-8],[253,176]],[[56583,71675],[152,-199],[216,34],[207,-42],[-7,-103],[151,71],[-35,-175],[-400,-50],[3,98],[-339,115],[52,251]],[[54311,73167],[-100,-465],[41,-183],[-58,-303],[-213,222],[-141,64],[-387,300],[38,304],[325,-54],[284,64],[211,51]],[[52558,74927],[166,-419],[-39,-782],[-126,38],[-113,-197],[-105,156],[-11,713],[-64,338],[153,-30],[139,183]],[[89159,72524],[-104,-472],[48,-296],[-145,-416],[-355,-278],[-488,-36],[-396,-675],[-186,227],[-12,442],[-483,-130],[-329,-279],[-325,-11],[282,-435],[-186,-1004],[-179,-248],[-135,229],[69,533],[-176,172],[-113,405],[263,182],[145,371],[280,306],[203,403],[553,177],[297,-121],[291,1050],[185,-282],[408,591],[158,229],[174,723],[-47,664],[117,374],[295,108],[152,-819],[-9,-479],[-256,-595],[4,-610]],[[52655,75484],[-92,-456],[-126,120],[-64,398],[56,219],[179,226],[47,-507]],[[89974,76679],[195,-126],[197,250],[62,-663],[-412,-162],[-244,-587],[-436,404],[-152,-646],[-308,-9],[-39,587],[138,455],[296,33],[81,817],[83,460],[326,-615],[213,-198]],[[32315,78082],[202,-79],[257,16],[-137,-242],[-102,-38],[-353,250],[-69,198],[105,183],[97,-288]],[[32831,79592],[-135,-11],[-360,186],[-258,279],[96,49],[365,-148],[284,-247],[8,-108]],[[15692,79240],[-140,-82],[-456,269],[-84,209],[-248,207],[-50,168],[-286,107],[-107,321],[24,137],[291,-129],[171,-89],[261,-63],[94,-204],[138,-280],[277,-244],[115,-327]],[[34407,80527],[-184,-517],[181,199],[187,-126],[-98,-206],[247,-162],[128,144],[277,-182],[-86,-433],[194,101],[36,-313],[86,-367],[-117,-520],[-125,-22],[-183,111],[60,484],[-77,75],[-322,-513],[-166,21],[196,277],[-267,144],[-298,-35],[-539,18],[-43,175],[173,208],[-121,160],[234,356],[287,941],[172,336],[241,204],[129,-26],[-54,-160],[-148,-372]],[[13136,82508],[267,47],[-84,-671],[242,-475],[-111,1],[-167,270],[-103,272],[-140,184],[-51,260],[16,188],[131,-76]],[[89901,80562],[280,-1046],[-411,195],[-171,-854],[271,-605],[-8,-413],[-211,356],[-182,-457],[-51,496],[31,575],[-32,638],[64,446],[13,790],[-163,581],[24,808],[257,271],[-110,274],[123,83],[73,-391],[96,-569],[-7,-581],[114,-597]],[[48114,81456],[-493,-349],[-393,89],[225,617],[-145,601],[378,463],[210,276],[233,24],[298,-365],[-149,-406],[46,-422],[-210,-528]],[[53524,83435],[-166,-478],[-291,333],[-39,246],[408,195],[88,-296]],[[7498,84325],[-277,-225],[-142,152],[-43,277],[252,210],[148,90],[185,-40],[117,-183],[-240,-281]],[[49165,85222],[-297,-639],[283,81],[304,-3],[-72,-481],[-250,-530],[287,-38],[270,-759],[190,-95],[171,-673],[79,-233],[337,-113],[-34,-378],[-142,-173],[111,-305],[-250,-310],[-371,6],[-473,-163],[-130,116],[-183,-276],[-257,67],[-195,-226],[-148,118],[407,621],[249,127],[-436,99],[-79,235],[291,183],[-152,319],[52,387],[414,-54],[40,343],[-190,372],[-337,104],[-66,160],[101,264],[-92,163],[-149,-279],[-17,569],[-140,301],[101,611],[216,480],[222,-47],[335,49]],[[4006,85976],[-171,-92],[-182,110],[-168,161],[274,101],[220,-54],[27,-226]],[[27981,87304],[-108,-310],[-123,50],[-73,176],[13,41],[107,177],[114,-13],[70,-121]],[[27250,87631],[-325,-326],[-196,13],[-61,160],[207,273],[381,-6],[-6,-114]],[[2297,88264],[171,-113],[173,61],[225,-156],[276,-79],[-23,-64],[-211,-125],[-211,128],[-106,107],[-245,-34],[-66,52],[17,223]],[[26344,89371],[51,-259],[143,91],[161,-155],[304,-203],[318,-184],[25,-281],[204,46],[199,-196],[-247,-186],[-432,142],[-156,266],[-275,-314],[-396,-306],[-95,346],[-377,-57],[242,292],[35,465],[95,542],[201,-49]],[[45969,89843],[-64,-382],[314,-403],[-361,-451],[-801,-405],[-240,-107],[-365,87],[-775,187],[273,261],[-605,289],[492,114],[-12,174],[-583,137],[188,385],[421,87],[433,-400],[422,321],[349,-167],[453,315],[461,-42]],[[28926,90253],[-312,-30],[-69,289],[118,331],[255,82],[217,-163],[3,-253],[-32,-82],[-180,-174]],[[0,91325],[681,-451],[728,-588],[-24,-367],[187,-147],[-64,429],[754,-88],[544,-553],[-276,-257],[-455,-61],[-7,-578],[-111,-122],[-260,17],[-212,206],[-369,172],[-62,257],[-283,96],[-315,-76],[-151,207],[60,219],[-333,-140],[126,-278],[-158,-251]],[[0,88971],[0,2354]],[[23431,91410],[-173,-207],[-374,179],[-226,-65],[-380,266],[245,183],[194,256],[295,-168],[166,-106],[84,-112],[169,-226]],[[0,92833],[99999,-404],[-305,-30],[-49,187],[-99645,247]],[[0,92833],[36,24],[235,-1],[402,-169],[-24,-81],[-286,-141],[-363,-36],[0,404]],[[24848,91640],[-1,-604],[371,463],[332,-380],[-83,-438],[269,-399],[290,427],[202,510],[16,648],[394,-45],[411,-87],[373,-293],[17,-293],[-207,-315],[196,-316],[-36,-288],[-544,-413],[-386,-91],[-287,178],[-83,-297],[-268,-498],[-81,-258],[-322,-400],[-397,-39],[-220,-250],[-18,-384],[-323,-74],[-340,-479],[-301,-665],[-108,-466],[-15,-686],[408,-99],[125,-553],[130,-448],[388,117],[517,-256],[277,-225],[199,-279],[348,-162],[294,-249],[459,-34],[302,-58],[-45,-511],[86,-594],[201,-661],[414,-561],[214,192],[150,607],[-145,934],[-196,311],[445,276],[314,415],[154,411],[-22,395],[-189,502],[-338,445],[328,619],[-121,535],[-93,922],[194,137],[476,-161],[286,-57],[230,155],[258,-200],[342,-343],[85,-229],[495,-45],[-8,-496],[92,-747],[254,-92],[201,-348],[402,328],[266,652],[184,274],[216,-527],[362,-754],[307,-709],[-112,-371],[370,-333],[250,-338],[442,-152],[179,-189],[110,-500],[216,-78],[112,-223],[20,-664],[-202,-222],[-199,-207],[-458,-210],[-349,-486],[-470,-96],[-594,125],[-417,4],[-287,-41],[-233,-424],[-354,-262],[-401,-782],[-320,-545],[236,97],[446,776],[583,493],[416,59],[245,-290],[-262,-397],[88,-637],[91,-446],[361,-295],[459,86],[278,664],[19,-429],[180,-214],[-344,-387],[-615,-351],[-276,-239],[-310,-426],[-211,44],[-11,500],[483,488],[-445,-19],[-309,-72],[48,-194],[-296,-286],[-286,-204],[-293,-175],[-159,-386],[-35,-98],[-3,-313],[92,-313],[115,-15],[-29,216],[83,-131],[-22,-169],[-188,-96],[-133,12],[-205,-104],[-121,-29],[-162,-29],[-231,-171],[408,111],[82,-112],[-389,-177],[-177,-1],[8,72],[-84,-164],[82,-27],[-60,-424],[-203,-455],[-20,152],[-61,31],[-91,147],[57,-318],[66,-106],[8,-222],[-89,-230],[-157,-472],[-25,24],[86,402],[-142,226],[-33,490],[-53,-255],[59,-375],[-175,88],[183,-186],[12,-562],[79,-41],[29,-204],[39,-591],[-176,-439],[-288,-175],[-182,-346],[-139,-38],[-141,-217],[-39,-199],[-305,-383],[-157,-281],[-131,-351],[-43,-419],[50,-411],[92,-505],[124,-418],[1,-256],[132,-685],[-9,-398],[-12,-230],[-69,-361],[-83,-74],[-137,71],[-44,259],[-106,136],[-147,508],[-129,452],[-42,231],[57,393],[-77,325],[-217,494],[-108,91],[-281,-269],[-49,30],[-135,276],[-174,146],[-314,-75],[-247,66],[-212,-41],[-118,-83],[54,-166],[-5,-240],[59,-117],[-53,-77],[-103,87],[-104,-112],[-202,18],[-207,313],[-242,-74],[-202,137],[-173,-42],[-234,-138],[-253,-438],[-276,-255],[-152,-282],[-63,-266],[-3,-407],[14,-284],[52,-201],[1,-1],[-1,-1],[-107,-516],[-49,-426],[-20,-791],[-27,-289],[48,-322],[86,-288],[56,-458],[184,-440],[65,-337],[109,-291],[295,-157],[114,-247],[244,165],[212,60],[208,106],[175,101],[176,241],[67,345],[22,496],[48,173],[188,155],[294,137],[246,-21],[169,50],[66,-125],[-9,-285],[-149,-351],[-66,-360],[51,-103],[-42,-255],[-69,-461],[-71,152],[-58,-10],[1,-87],[53,-3],[-5,-160],[-45,-256],[24,-91],[-29,-212],[18,-56],[-32,-299],[-55,-156],[-50,-19],[-55,-205],[90,-107],[24,88],[82,-75],[29,-23],[61,104],[79,8],[26,-48],[43,29],[129,-53],[128,16],[90,65],[32,66],[89,-31],[66,-40],[73,14],[55,51],[127,-82],[44,-13],[85,-110],[80,-132],[101,-91],[73,-162],[-24,-57],[-14,-132],[29,-216],[-64,-202],[-30,-237],[-9,-261],[15,-152],[7,-266],[-43,-58],[-26,-253],[19,-156],[-56,-151],[12,-159],[43,-97],[70,-321],[108,-238],[130,-252],[100,-212],[-6,-125],[111,-27],[26,48],[77,-145],[136,42],[119,150],[168,119],[95,176],[153,-34],[-10,-58],[155,-21],[124,-102],[90,-177],[105,-164],[143,-18],[209,412],[114,63],[3,195],[51,500],[159,274],[175,11],[22,123],[218,-49],[218,298],[109,132],[134,285],[98,-36],[73,-156],[-54,-199],[-8,-139],[-163,-69],[91,-268],[-3,-309],[-123,-343],[105,-469],[120,38],[62,427],[-86,208],[-14,447],[346,241],[-38,278],[97,186],[100,-415],[195,-9],[180,-330],[11,-195],[249,-6],[297,61],[159,-264],[213,-73],[155,184],[4,149],[344,35],[333,9],[-236,-175],[95,-279],[222,-44],[210,-291],[45,-473],[144,13],[109,-139],[183,-217],[171,-385],[8,-304],[105,-14],[149,-289],[109,-205],[333,-119],[30,107],[225,43],[298,-159],[95,-65],[204,-140],[294,-499],[46,-242],[95,28],[69,-327],[155,-1033],[149,-97],[7,-408],[-208,-487],[86,-178],[491,-92],[10,-593],[211,388],[349,-212],[462,-361],[135,-346],[-45,-327],[323,182],[540,-313],[415,23],[411,-489],[355,-662],[214,-170],[237,-24],[101,-186],[94,-752],[46,-358],[-110,-977],[-142,-385],[-391,-822],[-177,-668],[-206,-513],[-69,-11],[-78,-435],[20,-1107],[-77,-910],[-30,-390],[-88,-233],[-49,-790],[-282,-771],[-47,-610],[-225,-256],[-65,-355],[-302,2],[-437,-227],[-195,-263],[-311,-173],[-327,-470],[-235,-586],[-41,-441],[46,-326],[-51,-597],[-63,-289],[-195,-325],[-308,-1040],[-244,-468],[-189,-277],[-127,-562],[-183,-337],[-121,-372],[-313,-328],[-205,118],[-151,-63],[-256,253],[-189,-19],[-169,327],[-19,-308],[353,-506],[-38,-408],[173,-257],[-14,-289],[-267,-757],[-412,-317],[-557,-123],[-305,59],[59,-352],[-57,-442],[51,-298],[-167,-208],[-284,-82],[-267,216],[-108,-155],[39,-587],[188,-178],[152,186],[82,-307],[-255,-183],[-223,-367],[-41,-595],[-66,-316],[-262,-2],[-218,-302],[-80,-443],[274,-433],[265,-119],[-96,-531],[-328,-333],[-180,-692],[-254,-234],[-113,-276],[89,-614],[185,-342],[-117,30],[-247,4],[-134,-145],[-250,-213],[-45,-552],[-118,-14],[-313,192],[-318,412],[-346,338],[-87,374],[79,346],[-140,393],[-36,1007],[119,568],[293,457],[-422,172],[265,522],[94,982],[309,-208],[145,1224],[-186,157],[-87,-738],[-175,83],[87,845],[95,1095],[127,404],[-79,576],[-23,666],[117,19],[170,954],[192,945],[118,881],[-64,885],[83,487],[-34,730],[163,721],[50,1143],[89,1227],[87,1321],[-20,967],[-58,832],[-279,340],[-24,242],[-551,593],[-498,646],[-214,365],[-115,488],[46,170],[-236,775],[-274,1090],[-262,1177],[-114,269],[-87,435],[-216,386],[-198,239],[90,264],[-134,563],[86,414],[221,373],[148,442],[-60,258],[-106,-275],[-166,259],[56,167],[-47,536],[97,89],[52,368],[105,381],[-20,241],[153,126],[190,236],[-37,183],[103,44],[-12,296],[65,214],[138,40],[117,371],[106,310],[-102,141],[52,343],[-62,540],[59,155],[-44,500],[-112,315],[-93,170],[-59,319],[68,158],[-70,40],[-52,195],[-138,165],[-122,-38],[-56,-205],[-112,-149],[-61,-20],[-27,-123],[132,-321],[-75,-76],[-40,-87],[-130,-30],[-48,353],[-36,-101],[-92,35],[-56,238],[-114,39],[-72,69],[-119,-1],[-8,-128],[-32,89],[-151,131],[-56,124],[32,103],[-11,130],[-77,142],[-109,116],[-95,76],[-19,173],[-73,105],[18,-172],[-55,-141],[-64,164],[-89,58],[-38,120],[2,179],[36,187],[-78,83],[64,114],[-96,186],[-130,238],[-61,200],[-117,185],[-140,267],[31,92],[46,-89],[21,41],[-48,185],[-84,52],[-31,-140],[-161,9],[-100,57],[-115,117],[-154,37],[-79,127],[-142,103],[-174,11],[-127,117],[-149,244],[-314,636],[-144,192],[-226,154],[-156,-43],[-223,-223],[-140,-58],[-196,156],[-208,112],[-260,271],[-208,83],[-314,275],[-233,282],[-70,158],[-155,35],[-284,187],[-116,270],[-299,335],[-139,373],[-66,288],[93,57],[-29,169],[64,153],[1,204],[-93,266],[-25,235],[-94,298],[-244,587],[-280,462],[-135,368],[-238,241],[-51,145],[42,365],[-142,137],[-164,288],[-69,412],[-149,48],[-162,311],[-130,288],[-12,184],[-149,446],[-99,452],[5,227],[-201,235],[-93,-26],[-159,163],[-44,-240],[46,-284],[27,-444],[95,-243],[206,-407],[46,-139],[42,-42],[37,-203],[49,8],[56,-381],[85,-150],[59,-210],[174,-300],[92,-550],[83,-259],[77,-277],[15,-311],[134,-20],[112,-268],[100,-264],[-6,-106],[-117,-217],[-49,3],[-74,359],[-182,337],[-200,286],[-142,150],[9,432],[-42,320],[-132,183],[-191,264],[-37,-76],[-70,154],[-171,143],[-164,343],[20,44],[115,-33],[103,221],[10,266],[-214,422],[-163,163],[-102,369],[-103,388],[-129,472],[-113,531],[-46,302],[-180,340],[-130,71],[-30,169],[-156,30],[-100,159],[-258,59],[-70,95],[-34,324],[-270,594],[-231,821],[10,137],[-123,195],[-215,495],[-38,482],[-148,323],[61,489],[-10,507],[-89,453],[109,557],[67,1072],[-50,792],[-88,506],[-80,274],[33,115],[402,-200],[148,-558],[68,156],[-44,485],[-94,484],[-38,1],[-537,581],[-199,255],[-503,245],[-155,523],[40,362],[-356,252],[-48,476],[-336,429],[-6,304],[-153,223],[-245,188],[-78,515],[-358,478],[-150,558],[-267,38],[-441,15],[-326,170],[-574,613],[-266,112],[-486,211],[-385,-50],[-546,271],[-330,252],[-309,-125],[58,-411],[-154,-38],[-321,-123],[-245,-199],[-307,-126],[-40,348],[125,580],[295,182],[-76,148],[-354,-329],[-190,-394],[-400,-420],[203,-287],[-262,-424],[-299,-247],[-278,-181],[-69,-261],[-434,-305],[-87,-278],[-325,-252],[-191,45],[-259,-165],[-282,-201],[-231,-197],[-477,-169],[-43,99],[304,276],[271,182],[296,324],[345,66],[137,243],[385,353],[62,119],[205,208],[48,448],[141,349],[-320,-179],[-90,102],[-150,-215],[-181,300],[-75,-212],[-104,294],[-278,-236],[-170,0],[-24,352],[50,217],[-179,210],[-361,-113],[-235,277],[-190,142],[-1,334],[-214,252],[108,340],[226,330],[99,303],[225,43],[191,-94],[224,285],[201,-51],[212,183],[-52,270],[-155,106],[205,228],[-170,-7],[-295,-128],[-85,-131],[-219,131],[-392,-67],[-407,142],[-117,238],[-351,343],[390,247],[620,289],[228,0],[-38,-295],[586,22],[-225,366],[-342,226],[-197,295],[-267,252],[-381,187],[155,309],[493,19],[350,270],[66,287],[284,281],[271,68],[526,262],[256,-40],[427,315],[421,-124],[201,-266],[123,114],[469,-35],[-16,-136],[425,-101],[283,59],[585,-186],[534,-56],[214,-77],[370,96],[421,-177],[302,-83],[518,-142],[438,-284],[289,-55],[244,247],[336,184],[413,-72],[416,259],[455,148],[191,-245],[207,138],[62,278],[192,-63],[470,-530],[369,401],[38,-448],[341,96],[105,173],[337,-34],[424,-248],[650,-217],[383,-100],[272,38],[375,-300],[-391,-293],[502,-127],[750,70],[236,103],[296,-354],[302,299],[-283,251],[179,202],[338,27],[223,59],[224,-141],[279,-321],[310,47],[491,-266],[431,94],[405,-14],[-32,367],[247,103],[431,-200],[-2,-559],[177,471],[223,-16],[126,594],[-298,364],[-324,239],[22,653],[329,429],[366,-95],[281,-261],[378,-666],[-247,-290],[517,-120]],[[18287,93781],[-139,-277],[618,179],[386,-298],[314,302],[254,-194],[227,-580],[140,244],[-197,606],[244,86],[276,-94],[311,-239],[175,-575],[86,-417],[466,-293],[502,-279],[-31,-260],[-456,-48],[178,-227],[-94,-217],[-503,93],[-478,160],[-322,-36],[-522,-201],[-824,-103],[-374,-41],[-151,279],[-379,161],[-246,-66],[-343,468],[185,62],[429,101],[392,-26],[362,103],[-537,138],[-594,-47],[-394,12],[-146,217],[644,237],[-428,-9],[-485,156],[233,443],[193,235],[744,359],[284,-114]],[[20972,93958],[-244,-390],[-434,413],[95,83],[372,24],[211,-130]],[[28794,93770],[25,-163],[-296,17],[-299,13],[-304,-80],[-80,36],[-306,313],[12,213],[133,39],[636,-63],[479,-325]],[[25955,93803],[219,-369],[256,477],[704,242],[477,-611],[-42,-387],[550,172],[263,235],[616,-299],[383,-282],[36,-258],[515,134],[290,-376],[670,-234],[242,-238],[263,-553],[-510,-275],[654,-386],[441,-130],[400,-543],[437,-39],[-87,-414],[-487,-687],[-342,253],[-437,568],[-359,-74],[-35,-338],[292,-344],[377,-272],[114,-157],[181,-584],[-96,-425],[-350,160],[-697,473],[393,-509],[289,-357],[45,-206],[-753,236],[-596,343],[-337,287],[97,167],[-414,304],[-405,286],[5,-171],[-803,-94],[-235,203],[183,435],[522,10],[571,76],[-92,211],[96,294],[360,576],[-77,261],[-107,203],[-425,286],[-563,201],[178,150],[-294,367],[-245,34],[-219,201],[-149,-175],[-503,-76],[-1011,132],[-588,174],[-450,89],[-231,207],[290,270],[-394,2],[-88,599],[213,528],[286,241],[717,158],[-204,-382]],[[22123,94208],[331,-124],[496,75],[72,-172],[-259,-283],[420,-254],[-50,-532],[-455,-229],[-268,50],[-192,225],[-690,456],[5,189],[567,-73],[-306,386],[329,286]],[[89889,93835],[-421,-4],[-569,66],[-49,31],[263,234],[348,54],[394,-226],[34,-155]],[[24112,93575],[-298,-442],[-317,22],[-173,519],[4,294],[145,251],[276,161],[579,-20],[530,-144],[-415,-526],[-331,-115]],[[16539,92764],[-731,-294],[-147,259],[-641,312],[93,193],[218,489],[241,388],[-272,362],[939,93],[397,-123],[709,-33],[270,-171],[298,-249],[-349,-149],[-681,-415],[-344,-414],[0,-248]],[[91869,94941],[-321,-234],[-444,53],[-516,233],[66,192],[518,-89],[697,-155]],[[23996,94879],[-151,-229],[-403,44],[-337,155],[148,266],[399,159],[243,-208],[101,-187]],[[90301,95224],[-219,-439],[-1023,16],[-461,-139],[-550,384],[149,406],[366,111],[734,-26],[1004,-313]],[[22639,95907],[212,-273],[9,-303],[-127,-440],[-458,-60],[-298,94],[5,345],[-455,-46],[-18,457],[299,-18],[419,201],[390,-34],[22,77]],[[19941,95601],[109,-210],[247,99],[291,-26],[49,-289],[-169,-281],[-940,-91],[-701,-256],[-423,-14],[-35,193],[577,261],[-1255,-70],[-389,106],[379,577],[262,165],[782,-199],[493,-350],[485,-45],[-397,565],[255,215],[286,-68],[94,-282]],[[65981,92363],[-164,-52],[-907,77],[-74,262],[-503,158],[-40,320],[284,126],[-10,323],[551,503],[-255,73],[665,518],[-75,268],[621,312],[917,380],[925,110],[475,220],[541,76],[193,-233],[-187,-184],[-984,-293],[-848,-282],[-863,-562],[-414,-577],[-435,-568],[56,-491],[531,-484]],[[23699,96131],[308,-190],[547,1],[240,-194],[-64,-222],[319,-134],[177,-140],[374,-26],[406,-50],[441,128],[566,51],[451,-42],[298,-223],[62,-244],[-174,-157],[-414,-127],[-355,72],[-797,-91],[-570,-11],[-449,73],[-738,190],[-96,325],[-34,293],[-279,258],[-574,72],[-322,183],[104,242],[573,-37]],[[17722,96454],[-38,-454],[-214,-205],[-259,-29],[-517,-252],[-444,-91],[-377,128],[472,442],[570,383],[426,-9],[381,87]],[[63641,74970],[141,-419],[130,-28],[85,-159],[-228,-47],[-49,-459],[-47,-207],[-102,-138],[7,-293],[88,-436],[263,-123],[193,-296],[395,-102],[434,156],[27,139],[-52,417],[40,618],[-216,200],[71,405],[-184,34],[61,498],[262,-145],[244,189],[-202,355],[-80,338],[-224,-151],[-28,-433],[-87,383],[-15,144],[68,246],[-53,206],[-322,202],[-125,530],[-154,150],[-9,192],[270,-56],[11,432],[236,96],[243,-88],[50,576],[-50,365],[-278,-28],[-236,144],[-321,-260],[-259,-124],[-127,-350],[-269,-97],[-276,-610],[252,-561],[-27,-398],[303,-696],[146,-311]],[[0,88971],[99997,-3],[-357,-260],[-360,44],[250,-315],[166,-487],[128,-159],[32,-244],[-71,-157],[-518,129],[-777,-445],[-247,-69],[-425,-415],[-403,-362],[-102,-269],[-397,409],[-724,-464],[-126,220],[-268,-254],[-371,81],[-90,-388],[-333,-572],[10,-239],[316,-132],[-37,-860],[-258,-22],[-119,-494],[116,-255],[-486,-301],[-96,-675],[-415,-144],[-83,-600],[-400,-551],[-103,407],[-119,862],[-155,1313],[134,819],[234,353],[15,276],[431,132],[496,744],[479,608],[499,471],[223,833],[-337,-50],[-167,-487],[-705,-648],[-227,726],[-717,-201],[-696,-990],[230,-362],[-620,-154],[-430,-61],[20,427],[-431,90],[-344,-291],[-850,102],[-913,-175],[-900,-1153],[-1065,-1394],[438,-74],[136,-370],[270,-132],[178,296],[305,-39],[401,-650],[9,-502],[-217,-591],[-23,-705],[-126,-945],[-418,-855],[-94,-409],[-377,-688],[-374,-682],[-179,-349],[-370,-346],[-175,-8],[-175,287],[-373,-432],[-43,-197],[-106,36],[-120,-201],[-83,-201],[10,-424],[-143,-130],[-50,-105],[-104,-174],[-185,-97],[-121,-159],[-9,-256],[-32,-65],[111,-96],[157,-259],[240,-697],[68,-383],[3,-681],[-105,-325],[-252,-113],[-222,-245],[-250,-51],[-31,322],[52,443],[-123,615],[206,99],[-190,506],[-135,113],[-34,-112],[-81,-49],[-10,112],[-72,54],[-75,94],[77,260],[65,69],[-25,108],[71,319],[-18,97],[-163,64],[-131,158],[-388,-171],[-204,-277],[-300,-161],[148,274],[-58,230],[220,397],[-147,310],[-242,-209],[-314,-411],[-171,-381],[-272,-29],[-142,-275],[147,-400],[227,-97],[9,-265],[220,-172],[311,421],[247,-230],[179,-15],[46,-310],[-394,-165],[-130,-319],[-270,-296],[-142,-414],[299,-324],[109,-582],[169,-541],[189,-454],[-5,-439],[-174,-161],[66,-315],[164,-184],[-43,-481],[-71,-468],[-155,-53],[-203,-640],[-225,-775],[-258,-705],[-382,-545],[-386,-498],[-313,-68],[-170,-262],[-96,192],[-157,-294],[-388,-296],[-294,-90],[-95,-624],[-154,-35],[-73,429],[66,228],[-373,190],[-131,-97],[-371,-505],[-231,-558],[-61,-410],[212,-623],[260,-772],[252,-365],[169,-475],[127,-1093],[-37,-1039],[-232,-389],[-318,-381],[-227,-492],[-346,-550],[-101,378],[78,401],[-206,335],[-233,87],[-112,307],[-141,611],[-249,271],[-238,-11],[41,464],[-245,-3],[-22,-650],[-150,-863],[-90,-522],[19,-428],[181,-18],[113,-539],[50,-512],[155,-338],[168,-69],[144,-306],[64,-56],[164,-356],[116,-396],[16,-398],[-29,-269],[27,-203],[20,-349],[98,-163],[109,-523],[-5,-199],[-197,-40],[-263,438],[-329,469],[-32,301],[-161,395],[-38,489],[-100,322],[30,431],[-61,250],[-110,227],[-47,292],[-148,334],[-135,280],[-45,-347],[-53,328],[30,369],[82,566],[-27,439],[86,452],[-94,350],[23,644],[-113,306],[-90,707],[-50,746],[-121,490],[-183,-297],[-315,-421],[-156,53],[-172,138],[96,732],[-58,554],[-218,681],[34,213],[-163,76],[-197,481],[-79,309],[-16,301],[-53,284],[-116,344],[-256,23],[25,-243],[-87,-329],[-118,120],[-41,-108],[-78,65],[-108,53],[-39,-216],[-189,7],[-343,-122],[16,-445],[-148,-349],[-400,-398],[-311,-695],[-209,-373],[-276,-386],[-1,-272],[-138,-146],[-250,-212],[-130,-31],[-84,-450],[58,-769],[15,-490],[-118,-561],[-1,-1004],[-144,-29],[-126,-450],[84,-195],[-253,-167],[-93,-402],[-112,-170],[-263,552],[-128,827],[-107,596],[-97,279],[-148,568],[-69,739],[-48,369],[-253,811],[-115,1145],[-83,756],[1,716],[-54,553],[-404,-353],[-196,70],[-362,716],[133,214],[-82,232],[-326,501],[-203,150],[-83,425],[-215,449],[-512,-111],[-451,-11],[-391,-83],[-523,179],[-302,136],[-314,76],[-118,725],[-133,105],[-214,-106],[-280,-286],[-339,196],[-281,454],[-267,168],[-186,561],[-205,788],[-149,-96],[-177,196],[-103,-231],[-165,29],[58,-261],[-25,-135],[89,-445],[109,-510],[137,-135],[47,-207],[190,-248],[16,-244],[-27,-197],[35,-199],[80,-165],[37,-194],[41,-145],[-18,430],[75,310],[76,64],[84,-186],[5,-345],[-61,-348],[53,-226],[49,29],[11,-162],[217,93],[230,-15],[168,-18],[190,400],[207,379],[176,364],[80,201],[35,-51],[-26,-244],[-37,-108],[38,-466],[125,-404],[155,-214],[204,-78],[164,-107],[125,-339],[75,-196],[100,-75],[-1,-132],[-101,-352],[-44,-166],[-117,-189],[-104,-404],[-126,31],[-58,-141],[-44,-300],[34,-395],[-26,-72],[-128,2],[-174,-221],[-27,-288],[-63,-125],[-173,5],[-109,-149],[1,-239],[-134,-164],[-153,56],[-186,-199],[-128,-33],[-201,-159],[-54,-263],[-6,-201],[-277,-249],[-444,-276],[-249,-417],[-122,-32],[-83,34],[-163,-245],[-177,-113],[-233,-31],[-70,-34],[-61,-156],[-73,-43],[-42,-150],[-138,13],[-89,-80],[-192,30],[-72,345],[8,323],[-46,174],[-54,437],[-80,243],[56,29],[-29,270],[34,114],[-12,257],[-36,253],[-84,177],[-22,236],[-143,212],[-148,495],[-79,482],[-192,406],[-124,97],[-184,563],[-32,411],[12,350],[-159,655],[-130,231],[-150,122],[-92,339],[15,133],[-77,307],[-81,131],[-108,440],[-170,476],[-141,406],[-139,-3],[44,325],[12,206],[34,236],[-9,86],[-78,-238],[-60,-446],[-75,-308],[-65,-103],[-93,191],[-125,263],[-198,847],[-29,-53],[115,-624],[171,-594],[210,-920],[102,-321],[90,-334],[249,-654],[-55,-103],[9,-384],[323,-530],[49,-121],[90,-580],[-61,-107],[40,-608],[102,-706],[106,-145],[152,-219],[161,-683],[77,-543],[152,-288],[379,-558],[154,-336],[151,-341],[87,-203],[136,-178],[66,-183],[-9,-245],[-158,-142],[119,-161],[91,-109],[54,-244],[125,-248],[138,-2],[262,151],[302,70],[245,184],[138,39],[99,108],[158,20],[89,12],[128,88],[147,59],[132,202],[105,2],[6,-163],[-25,-344],[1,-310],[-59,-214],[-78,-639],[-134,-659],[-172,-755],[-238,-866],[-237,-661],[-327,-806],[-278,-479],[-415,-586],[-259,-450],[-304,-715],[-64,-312],[-63,-140],[-195,-236],[-68,-246],[-104,-44],[-40,-416],[-89,-238],[-54,-393],[-112,-195],[-128,-728],[16,-335],[178,-216],[8,-153],[-76,-357],[16,-180],[-18,-282],[97,-370],[115,-583],[101,-129],[45,-265],[-11,-588],[34,-519],[11,-923],[49,-290],[-83,-422],[-108,-410],[-177,-366],[-254,-225],[-313,-287],[-313,-634],[-107,-108],[-194,-420],[-115,-136],[-23,-421],[132,-448],[54,-346],[4,-177],[49,29],[-8,-579],[-45,-274],[65,-102],[-41,-246],[-116,-210],[-229,-199],[-334,-320],[-122,-219],[24,-248],[71,-40],[-24,-311],[-70,-430],[-32,-491],[-72,-267],[-190,-298],[-54,-86],[-118,-300],[-77,-303],[-158,-424],[-314,-609],[-196,-355],[-209,-269],[-291,-229],[-141,-31],[-36,-164],[-169,88],[-138,-113],[-301,114],[-168,-72],[-115,31],[-286,-233],[-238,-94],[-171,-223],[-127,-14],[-117,210],[-94,11],[-120,264],[-13,-82],[-37,159],[2,346],[-90,396],[89,108],[-7,453],[-182,553],[-139,501],[-1,1],[-199,768],[-207,446],[-108,432],[-62,575],[-68,428],[-93,910],[-7,707],[-35,322],[-108,243],[-144,489],[-146,708],[-60,371],[-226,577],[-17,453],[-26,372],[38,519],[96,541],[15,254],[90,532],[66,243],[159,386],[90,263],[29,438],[-15,335],[-83,211],[-74,358],[-68,355],[15,122],[85,235],[-84,570],[-57,396],[-139,374],[26,115],[-39,183],[-74,444],[-228,626],[-285,596],[-184,488],[-169,610],[9,196],[61,189],[67,430],[56,438],[-52,90],[96,663],[40,467],[-108,390],[-127,100],[-56,265],[-71,85],[3,163],[-289,-213],[-105,32],[-107,-133],[-222,13],[-149,370],[-91,427],[-197,390],[-209,-8],[-245,1],[-229,-69],[-224,-126],[-436,-346],[-154,-203],[-250,-171],[-248,168],[-126,-7],[-194,116],[-178,-7],[-329,-103],[-193,-170],[-275,-217],[-54,15],[-73,-5],[-286,282],[-252,450],[-237,323],[-187,381],[-75,44],[-200,238],[-144,316],[-49,216],[-34,437],[-122,349],[-108,232],[-71,76],[-69,118],[-32,261],[-41,130],[-80,97],[-149,247],[-117,39],[-63,166],[1,90],[-84,125],[-18,127],[-46,453],[36,262],[-115,460],[-138,210],[122,112],[134,415],[66,304],[-24,318],[78,291],[34,557],[-30,583],[-34,294],[28,295],[-72,281],[-146,255],[12,249],[13,274],[106,161],[91,308],[-18,200],[96,417],[155,376],[93,95],[74,344],[6,315],[100,365],[185,216],[177,603],[144,235],[259,66],[219,403],[139,158],[232,493],[-70,735],[106,508],[37,312],[179,399],[278,270],[206,244],[186,612],[87,362],[205,-2],[167,-251],[264,41],[288,-131],[121,-6],[267,323],[300,102],[175,244],[268,180],[471,105],[459,48],[140,-87],[262,232],[297,5],[113,-137],[190,35],[302,239],[195,-71],[-9,-299],[236,217],[20,-113],[-139,-289],[-2,-274],[96,-147],[-36,-511],[-183,-297],[53,-322],[143,-10],[70,-281],[106,-92],[326,-204],[117,51],[232,-98],[368,-264],[130,-526],[250,-114],[391,-248],[296,-293],[136,153],[133,272],[-65,452],[87,288],[200,277],[192,80],[375,-121],[95,-264],[104,-2],[88,-101],[276,-69],[68,-196],[369,10],[268,-156],[275,-175],[129,-92],[214,188],[114,169],[245,49],[198,-75],[75,-293],[65,193],[222,-140],[217,-33],[137,149],[80,194],[-19,34],[74,276],[56,446],[40,149],[8,6],[99,482],[138,416],[5,21],[-26,452],[68,243],[-102,268],[105,222],[-169,-51],[-233,136],[-191,-340],[-421,-66],[-225,317],[-300,20],[-64,-245],[-192,-71],[-268,315],[-303,-10],[-165,587],[-203,328],[135,459],[-176,283],[308,565],[428,23],[117,449],[529,-78],[334,383],[324,167],[459,13],[485,-416],[399,-229],[323,91],[239,-53],[328,309],[42,252],[-70,403],[-160,218],[-154,68],[-102,181],[-354,499],[-317,223],[-240,347],[202,95],[231,494],[-156,234],[410,241],[-8,129],[-249,-95],[-222,-48],[-185,-191],[-260,-31],[-239,-220],[16,-368],[136,-142],[284,35],[-55,-210],[-304,-103],[-377,-342],[-154,121],[61,277],[-304,173],[50,113],[265,197],[-80,135],[-432,149],[-19,221],[-257,-73],[-103,-325],[-215,-437],[6,-152],[-135,-128],[-84,56],[-78,-713],[-144,-245],[-101,-422],[89,-337],[33,-228],[243,-190],[-51,-145],[-330,-33],[-118,-182],[-232,-319],[-87,275],[3,122],[-169,17],[-145,56],[-336,-154],[192,-332],[-141,-96],[-154,0],[-147,304],[-52,-130],[62,-353],[139,-277],[-105,-130],[155,-272],[137,-171],[4,-334],[-257,157],[82,-302],[-176,-62],[105,-521],[-184,-7],[-228,257],[-104,472],[-49,393],[-108,272],[-143,337],[-18,168],[-48,41],[-5,130],[-154,199],[-24,281],[23,403],[38,184],[-46,93],[-59,46],[-78,192],[-120,118],[-261,218],[-161,213],[-254,176],[-233,435],[56,44],[-127,248],[-5,200],[-179,93],[-85,-255],[-82,198],[6,205],[10,9],[62,54],[-221,86],[-226,-210],[15,-293],[-34,-168],[91,-301],[261,-298],[140,-488],[309,-476],[217,3],[68,-130],[-78,-118],[249,-213],[204,-179],[238,-308],[29,-111],[-52,-211],[-154,276],[-242,97],[-116,-382],[200,-219],[-33,-309],[-116,-35],[-148,-506],[-116,-46],[1,181],[57,317],[60,126],[-108,342],[-85,298],[-115,74],[-82,255],[-179,107],[-120,238],[-206,38],[-217,267],[-254,384],[-189,341],[-86,584],[-138,68],[-226,195],[-128,-80],[-161,-274],[-115,-43],[-252,-334],[-548,160],[-404,-192],[-32,-355],[15,-344],[-263,-393],[-356,-125],[-25,-199],[-171,-327],[-107,-481],[108,-338],[-160,-263],[-60,-384],[-210,-118],[-197,-455],[-352,-8],[-265,11],[-174,-209],[-106,-223],[-136,49],[-103,199],[-79,340],[-259,92],[-112,-153],[-146,83],[-143,-65],[42,462],[-26,363],[-124,55],[-67,224],[22,386],[111,215],[20,239],[58,355],[-6,250],[-56,212],[-12,200],[14,420],[-114,257],[393,426],[340,-107],[373,4],[296,-101],[230,31],[449,-19],[144,354],[53,1177],[-287,620],[-205,299],[-424,228],[-28,430],[360,129],[466,-152],[-88,669],[263,-254],[646,461],[84,484],[243,119],[222,117],[143,162],[244,870],[380,247],[231,-17],[54,125],[232,32],[52,-130],[188,291],[-63,222],[-13,335],[-113,328],[-8,604],[46,159],[80,178],[244,36],[98,163],[223,167],[-9,-304],[-82,-192],[33,-166],[151,-89],[-68,-223],[-83,64],[-200,-425],[76,-288],[4,-228],[281,-138],[-3,-210],[283,111],[156,162],[313,-233],[132,-189],[189,174],[434,273],[350,200],[277,-100],[21,-144],[268,-7],[63,260],[383,191],[-59,497],[10,445],[136,371],[262,202],[221,-442],[223,12],[53,453],[32,349],[-102,-75],[-176,210],[-24,340],[351,164],[350,86],[301,-97],[287,17],[316,327],[-291,280],[-504,-47],[-489,-216],[-452,-125],[-161,322],[-269,195],[62,581],[-135,534],[133,344],[252,371],[635,640],[185,124],[-28,250],[-387,279],[-478,-167],[-269,-413],[43,-361],[-441,-475],[-537,-509],[-202,-832],[198,-416],[265,-328],[-255,-666],[-289,-138],[-106,-992],[-157,-554],[-337,57],[-158,-468],[-321,-27],[-89,558],[-232,671],[-211,835],[-187,363],[-548,-684],[-370,-138],[-385,301],[-99,635],[-88,1364],[256,380],[733,496],[549,609],[508,824],[668,1141],[465,444],[763,741],[610,259],[457,-31],[423,489],[506,-26],[499,118],[869,-433],[-358,-158],[305,-371],[286,206],[456,-358],[761,-140],[1050,-668],[213,-281],[18,-393],[-308,-311],[-454,-157],[-1240,449],[-204,-75],[453,-433],[36,-878],[358,-180],[217,-153],[36,286],[-174,263],[183,215],[672,-368],[234,144],[-187,433],[647,578],[256,-34],[260,-206],[161,406],[-231,352],[136,353],[-204,367],[777,-190],[158,-331],[-351,-73],[2,-328],[218,-203],[429,128],[68,377],[581,282],[969,507],[209,-29],[-273,-359],[344,-61],[199,202],[521,16],[412,245],[317,-356],[315,391],[-291,343],[145,195],[820,-179],[385,-185],[1006,-675],[186,309],[-282,313],[-8,125],[-335,58],[92,280],[-149,461],[-8,189],[512,535],[182,537],[207,116],[735,-156],[58,-328],[-263,-479],[173,-189],[89,-413],[-63,-809],[307,-362],[-120,-395],[-544,-839],[318,-87],[110,213],[306,151],[74,293],[240,281],[-162,336],[130,390],[-304,49],[-67,328],[222,594],[-361,481],[497,398],[-64,421],[139,13],[145,-328],[-109,-570],[297,-108],[-127,426],[465,233],[577,31],[513,-337],[-247,492],[-28,630],[484,119],[668,-26],[602,77],[-226,309],[321,388],[319,16],[540,293],[734,79],[93,162],[729,55],[227,-133],[624,314],[510,-10],[77,255],[265,252],[656,242],[476,-191],[-378,-146],[629,-90],[75,-292],[254,143],[812,-7],[626,-289],[223,-221],[-69,-307],[-307,-175],[-730,-328],[-209,-175],[345,-83],[410,-149],[250,112],[142,-379],[122,153],[444,93],[892,-97],[67,-276],[1162,-88],[15,451],[590,-103],[443,3],[449,-312],[128,-378],[-165,-247],[349,-465],[437,-240],[268,620],[446,-266],[473,159],[538,-182],[204,166],[455,-83],[-201,549],[367,256],[2509,-384],[236,-351],[727,-451],[1122,112],[553,-98],[231,-244],[-33,-432],[342,-168],[372,121],[492,15],[525,-116],[526,66],[484,-526],[344,189],[-224,378],[123,263],[886,-166],[578,36],[799,-282],[-99610,-258]],[[23933,96380],[-126,-17],[-521,38],[-74,165],[559,-9],[195,-109],[-33,-68]],[[19392,96485],[-518,-170],[-411,191],[224,188],[406,60],[392,-92],[-93,-177]],[[56867,96577],[-620,-241],[-490,137],[191,152],[-167,189],[575,119],[110,-222],[401,-134]],[[19538,97019],[-339,-115],[-461,1],[5,84],[285,177],[149,-27],[361,-120]],[[23380,96697],[-411,-122],[-226,138],[-119,221],[-22,245],[360,-24],[162,-39],[332,-205],[-76,-214]],[[22205,96856],[108,-247],[-453,66],[-457,192],[-619,21],[268,176],[-335,142],[-21,227],[546,-81],[751,-215],[212,-281]],[[79187,96845],[-1566,-228],[507,776],[229,66],[208,-38],[704,-336],[-82,-240]],[[55069,97669],[915,-440],[-699,-233],[-155,-435],[-243,-111],[-132,-490],[-335,-23],[-598,361],[252,210],[-416,170],[-541,499],[-216,463],[757,212],[152,-207],[396,8],[105,202],[408,20],[350,-206]],[[57068,98086],[545,-207],[-412,-318],[-806,-70],[-819,98],[-50,163],[-398,11],[-304,271],[858,165],[403,-142],[281,177],[702,-148]],[[64204,98169],[-373,-78],[-250,-45],[-39,-97],[-324,-98],[-301,140],[158,185],[-618,18],[542,107],[422,8],[57,-160],[159,142],[262,97],[412,-129],[-107,-90]],[[77760,97184],[-606,-73],[-773,170],[-462,226],[-213,423],[-379,117],[722,404],[600,133],[540,-297],[640,-572],[-69,-531]],[[25828,97644],[334,-190],[-381,-176],[-513,-445],[-492,-42],[-575,76],[-299,240],[4,215],[220,157],[-508,-4],[-306,196],[-176,268],[193,262],[192,180],[285,42],[-122,135],[646,30],[355,-315],[468,-127],[455,-112],[220,-390]],[[30972,99681],[742,-47],[597,-75],[508,-161],[-12,-157],[-678,-257],[-672,-119],[-251,-133],[605,3],[-656,-358],[-452,-167],[-476,-483],[-573,-98],[-177,-120],[-841,-64],[383,-74],[-192,-105],[230,-292],[-264,-202],[-429,-167],[-132,-232],[-388,-176],[39,-134],[475,23],[6,-144],[-742,-355],[-726,163],[-816,-91],[-414,71],[-525,31],[-35,284],[514,133],[-137,427],[170,41],[742,-255],[-379,379],[-450,113],[225,229],[492,141],[79,206],[-392,231],[-118,304],[759,-26],[220,-64],[433,216],[-625,68],[-972,-38],[-491,201],[-232,239],[-324,173],[-61,202],[413,112],[324,19],[545,96],[409,220],[344,-30],[300,-166],[211,319],[367,95],[498,65],[849,24],[148,-63],[802,100],[601,-38],[602,-37]],[[42472,99925],[1737,-469],[-513,-227],[-1062,-26],[-1496,-58],[140,-105],[984,65],[836,-204],[540,181],[231,-212],[-305,-344],[707,220],[1348,229],[833,-114],[156,-253],[-1132,-420],[-157,-136],[-888,-102],[643,-28],[-324,-431],[-224,-383],[9,-658],[333,-386],[-434,-24],[-457,-187],[513,-313],[65,-502],[-297,-55],[360,-508],[-617,-42],[322,-241],[-91,-208],[-391,-91],[-388,-2],[348,-400],[4,-263],[-549,244],[-143,-158],[375,-148],[364,-361],[105,-476],[-495,-114],[-214,228],[-344,340],[95,-401],[-322,-311],[732,-25],[383,-32],[-745,-515],[-755,-466],[-813,-204],[-306,-2],[-288,-228],[-386,-624],[-597,-414],[-192,-24],[-370,-145],[-399,-138],[-238,-365],[-4,-415],[-141,-388],[-453,-472],[112,-462],[-125,-488],[-142,-577],[-391,-36],[-410,482],[-556,3],[-269,324],[-186,577],[-481,735],[-141,385],[-38,530],[-384,546],[100,435],[-186,208],[275,691],[418,220],[110,247],[58,461],[-318,-209],[-151,-88],[-249,-84],[-341,193],[-19,401],[109,314],[258,9],[567,-157],[-478,375],[-249,202],[-276,-83],[-232,147],[310,550],[-169,220],[-220,409],[-335,626],[-353,230],[3,247],[-745,346],[-590,43],[-743,-24],[-677,-44],[-323,188],[-482,372],[729,186],[559,31],[-1188,154],[-627,241],[39,229],[1051,285],[1018,284],[107,214],[-750,213],[243,235],[961,413],[404,63],[-115,265],[658,156],[854,93],[853,5],[303,-184],[737,325],[663,-221],[390,-46],[577,-192],[-660,318],[38,253],[932,353],[975,-27],[354,218],[982,57],[2219,-74]]],"bbox":[-180,-85.60903777459777,180,83.64513000000002],"transform":{"scale":[0.0036000360003600037,0.0016925586033320111],"translate":[-180,-85.60903777459777]}} diff --git a/frontend/src/mock-data/api.ts b/frontend/src/mock-data/api.ts index adaca1a90..4bb98c09c 100644 --- a/frontend/src/mock-data/api.ts +++ b/frontend/src/mock-data/api.ts @@ -2,7 +2,7 @@ import * as _ from 'lodash'; import { Observable, Subject, from, of } from 'rxjs'; import { takeUntil } from 'rxjs/operators'; import { mockUserResponse } from './user'; -import { CorpusDocumentationPage, TaskResult, TasksOutcome } from '../app/models'; +import { Corpus, CorpusDocumentationPage, TaskResult, TasksOutcome } from '../app/models'; import { LimitedResultsDownloadParameters } from '../app/models/search-results'; import { mockCorpusDefinition } from './corpus-definition'; import { APIEditableCorpus } from '../app/models/corpus-definition'; @@ -73,10 +73,13 @@ export class ApiServiceMock { return of({ username: 'Thomas', email: 'thomas@cromwell.com' }); } - public corpusDocumentation(): Observable { + public corpusDocumentationPages(corpus: Corpus): Observable { return of([{ + id: 1, + corpus: corpus.name, type: 'General', - content: 'Example of _documentation_.' + content: 'Example of _documentation_.', + index: 1, }]); } diff --git a/frontend/tsconfig.json b/frontend/tsconfig.json index f3472e852..6dc561c67 100644 --- a/frontend/tsconfig.json +++ b/frontend/tsconfig.json @@ -1,24 +1,27 @@ { "compileOnSave": false, "compilerOptions": { - "downlevelIteration": true, - "importHelpers": true, - "outDir": "./dist/out-tsc", - "sourceMap": true, - "declaration": false, - "moduleResolution": "node", - "experimentalDecorators": true, - "target": "ES2022", - "typeRoots": [ - "node_modules/@types" - ], - "lib": [ - "es2017", - "dom", - "dom.iterable" - ], - "module": "esnext", - "baseUrl": "./", - "useDefineForClassFields": false + "downlevelIteration": true, + "importHelpers": true, + "outDir": "../dist/out-tsc", + "sourceMap": true, + "declaration": false, + "moduleResolution": "node", + "experimentalDecorators": true, + "target": "ES2022", + "typeRoots": ["node_modules/@types"], + "lib": ["es2017", "dom", "dom.iterable"], + "module": "esnext", + "baseUrl": "src", + "paths": { + "@shared/*": ["app/shared/*"], + "@models": ["app/models"], + "@models/*": ["app/models/*"], + "@services": ["app/services"], + "@services/*": ["app/services/*"], + "@utils/*": ["app/utils/*"], + "@environments/*": ["environments/*"] + }, + "useDefineForClassFields": false } -} \ No newline at end of file +} diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 156fba49f..af4f5d239 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -2376,7 +2376,7 @@ "@types/estree" "*" "@types/json-schema" "*" -"@types/estree@*", "@types/estree@1.0.5", "@types/estree@^1.0.5": +"@types/estree@*", "@types/estree@1.0.5", "@types/estree@^1.0.0", "@types/estree@^1.0.5": version "1.0.5" resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.5.tgz#a6ce3e556e00fd9895dd872dd172ad0d4bd687f4" integrity sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw== @@ -2406,6 +2406,11 @@ resolved "https://registry.yarnpkg.com/@types/file-saver/-/file-saver-1.3.1.tgz#55d76b3c78b5fcc8a7588ead428dce0822464120" integrity sha512-A+lNc0nnhtX3iTLEYd/DisKTZdNKTf1bN0aSfQD/fG8bQ6SfUe5u8Fm2ab8qQHaMY5GVZumAXLnYptwX+mmQgg== +"@types/geojson@7946.0.4": + version "7946.0.4" + resolved "https://registry.yarnpkg.com/@types/geojson/-/geojson-7946.0.4.tgz#4e049756383c3f055dd8f3d24e63fb543e98eb07" + integrity sha512-MHmwBtCb7OCv1DSivz2UNJXPGU/1btAWRKlqJ2saEhVJkpkvqHMMaOpKg0v4sAbDWSQekHGvPVMM8nQ+Jen03Q== + "@types/http-errors@*": version "2.0.4" resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-2.0.4.tgz#7eb47726c391b7345a6ec35ad7f4de469cf5ba4f" @@ -3055,9 +3060,9 @@ autoprefixer@10.4.18: postcss-value-parser "^4.2.0" axios@^1.6.0: - version "1.6.8" - resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.8.tgz#66d294951f5d988a00e87a0ffb955316a619ea66" - integrity sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ== + version "1.7.5" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.5.tgz#21eed340eb5daf47d29b6e002424b3e88c8c54b1" + integrity sha512-fZu86yCo+svH3uqJ/yTdQ0QHpQu5oL+/QE+QPSv6BZSkDAoky9vytxp7u5qk83OJFS3kEBcesWni9WTZAv3tSw== dependencies: follow-redirects "^1.15.6" form-data "^4.0.0" @@ -3221,7 +3226,7 @@ brace-expansion@^2.0.1: dependencies: balanced-match "^1.0.0" -braces@^3.0.2, braces@~3.0.2: +braces@^3.0.2, braces@^3.0.3, braces@~3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== @@ -3557,11 +3562,16 @@ combined-stream@^1.0.8: dependencies: delayed-stream "~1.0.0" -commander@^2.20.0: +commander@2, commander@^2.20.0: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ== +commander@7: + version "7.2.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" + integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw== + comment-parser@1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/comment-parser/-/comment-parser-1.3.1.tgz#3d7ea3adaf9345594aedee6563f422348f165c1b" @@ -3789,6 +3799,13 @@ custom-event@~1.0.0: resolved "https://registry.yarnpkg.com/custom-event/-/custom-event-1.0.1.tgz#5d02a46850adf1b4a317946a3928fccb5bfd0425" integrity sha1-XQKkaFCt8bSjF5RqOSj8y1v9BCU= +"d3-array@1 - 3", "d3-array@2 - 3", "d3-array@2.10.0 - 3", "d3-array@2.5.0 - 3", d3-array@3.2.4, d3-array@^3.2.2: + version "3.2.4" + resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-3.2.4.tgz#15fec33b237f97ac5d7c986dc77da273a8ed0bb5" + integrity sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg== + dependencies: + internmap "1 - 2" + d3-cloud@^1.2.7: version "1.2.7" resolved "https://registry.yarnpkg.com/d3-cloud/-/d3-cloud-1.2.7.tgz#5a733c4bae43238cbb4760bb8f2d15912a8ad7a5" @@ -3796,11 +3813,134 @@ d3-cloud@^1.2.7: dependencies: d3-dispatch "^1.0.3" +"d3-color@1 - 3", d3-color@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-3.1.0.tgz#395b2833dfac71507f12ac2f7af23bf819de24e2" + integrity sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA== + +d3-delaunay@^6.0.2: + version "6.0.4" + resolved "https://registry.yarnpkg.com/d3-delaunay/-/d3-delaunay-6.0.4.tgz#98169038733a0a5babbeda55054f795bb9e4a58b" + integrity sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A== + dependencies: + delaunator "5" + +"d3-dispatch@1 - 3": + version "3.0.1" + resolved "https://registry.yarnpkg.com/d3-dispatch/-/d3-dispatch-3.0.1.tgz#5fc75284e9c2375c36c839411a0cf550cbfc4d5e" + integrity sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg== + d3-dispatch@^1.0.3: version "1.0.6" resolved "https://registry.yarnpkg.com/d3-dispatch/-/d3-dispatch-1.0.6.tgz#00d37bcee4dd8cd97729dd893a0ac29caaba5d58" integrity sha512-fVjoElzjhCEy+Hbn8KygnmMS7Or0a9sI2UzGwoB7cCtvI1XpVN9GpoYlnb3xt2YV66oXYb1fLJ8GMvP4hdU1RA== +d3-dsv@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/d3-dsv/-/d3-dsv-3.0.1.tgz#c63af978f4d6a0d084a52a673922be2160789b73" + integrity sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q== + dependencies: + commander "7" + iconv-lite "0.6" + rw "1" + +d3-force@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/d3-force/-/d3-force-3.0.0.tgz#3e2ba1a61e70888fe3d9194e30d6d14eece155c4" + integrity sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg== + dependencies: + d3-dispatch "1 - 3" + d3-quadtree "1 - 3" + d3-timer "1 - 3" + +"d3-format@1 - 3", d3-format@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-3.1.0.tgz#9260e23a28ea5cb109e93b21a06e24e2ebd55641" + integrity sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA== + +d3-geo-projection@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/d3-geo-projection/-/d3-geo-projection-4.0.0.tgz#dc229e5ead78d31869a4e87cf1f45bd2716c48ca" + integrity sha512-p0bK60CEzph1iqmnxut7d/1kyTmm3UWtPlwdkM31AU+LW+BXazd5zJdoCn7VFxNCHXRngPHRnsNn5uGjLRGndg== + dependencies: + commander "7" + d3-array "1 - 3" + d3-geo "1.12.0 - 3" + +"d3-geo@1.12.0 - 3", d3-geo@^3.1.0: + version "3.1.1" + resolved "https://registry.yarnpkg.com/d3-geo/-/d3-geo-3.1.1.tgz#6027cf51246f9b2ebd64f99e01dc7c3364033a4d" + integrity sha512-637ln3gXKXOwhalDzinUgY83KzNWZRKbYubaG+fGVuc/dxO64RRljtCTnf5ecMyE1RIdtqpkVcq0IbtU2S8j2Q== + dependencies: + d3-array "2.5.0 - 3" + +d3-hierarchy@^3.1.2: + version "3.1.2" + resolved "https://registry.yarnpkg.com/d3-hierarchy/-/d3-hierarchy-3.1.2.tgz#b01cd42c1eed3d46db77a5966cf726f8c09160c6" + integrity sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA== + +"d3-interpolate@1 - 3", "d3-interpolate@1.2.0 - 3", d3-interpolate@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/d3-interpolate/-/d3-interpolate-3.0.1.tgz#3c47aa5b32c5b3dfb56ef3fd4342078a632b400d" + integrity sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g== + dependencies: + d3-color "1 - 3" + +d3-path@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/d3-path/-/d3-path-3.1.0.tgz#22df939032fb5a71ae8b1800d61ddb7851c42526" + integrity sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ== + +"d3-quadtree@1 - 3": + version "3.0.1" + resolved "https://registry.yarnpkg.com/d3-quadtree/-/d3-quadtree-3.0.1.tgz#6dca3e8be2b393c9a9d514dabbd80a92deef1a4f" + integrity sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw== + +d3-scale-chromatic@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz#34c39da298b23c20e02f1a4b239bd0f22e7f1314" + integrity sha512-A3s5PWiZ9YCXFye1o246KoscMWqf8BsD9eRiJ3He7C9OBaxKhAd5TFCdEx/7VbKtxxTsu//1mMJFrEt572cEyQ== + dependencies: + d3-color "1 - 3" + d3-interpolate "1 - 3" + +d3-scale@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/d3-scale/-/d3-scale-4.0.2.tgz#82b38e8e8ff7080764f8dcec77bd4be393689396" + integrity sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ== + dependencies: + d3-array "2.10.0 - 3" + d3-format "1 - 3" + d3-interpolate "1.2.0 - 3" + d3-time "2.1.1 - 3" + d3-time-format "2 - 4" + +d3-shape@^3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/d3-shape/-/d3-shape-3.2.0.tgz#a1a839cbd9ba45f28674c69d7f855bcf91dfc6a5" + integrity sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA== + dependencies: + d3-path "^3.1.0" + +"d3-time-format@2 - 4", d3-time-format@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-4.1.0.tgz#7ab5257a5041d11ecb4fe70a5c7d16a195bb408a" + integrity sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg== + dependencies: + d3-time "1 - 3" + +"d3-time@1 - 3", "d3-time@2.1.1 - 3", d3-time@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-3.1.0.tgz#9310db56e992e3c0175e1ef385e545e48a9bb5c7" + integrity sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q== + dependencies: + d3-array "2 - 3" + +"d3-timer@1 - 3", d3-timer@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/d3-timer/-/d3-timer-3.0.1.tgz#6284d2a2708285b1abb7e201eda4380af35e63b0" + integrity sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA== + date-format@^4.0.3: version "4.0.3" resolved "https://registry.yarnpkg.com/date-format/-/date-format-4.0.3.tgz#f63de5dc08dc02efd8ef32bf2a6918e486f35873" @@ -3893,6 +4033,13 @@ define-properties@^1.1.4: has-property-descriptors "^1.0.0" object-keys "^1.1.1" +delaunator@5: + version "5.0.1" + resolved "https://registry.yarnpkg.com/delaunator/-/delaunator-5.0.1.tgz#39032b08053923e924d6094fe2cde1a99cc51278" + integrity sha512-8nvh+XBe96aCESrGOqMp/84b13H9cdKbG5P2ejQCh4d4sK9RL4371qou9drQjMhvnPmhWl5hnmqbEE0fXr9Xnw== + dependencies: + robust-predicates "^3.0.2" + delayed-stream@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" @@ -4582,6 +4729,11 @@ fast-glob@3.3.2, fast-glob@^3.2.11, fast-glob@^3.2.9, fast-glob@^3.3.0: merge2 "^1.3.0" micromatch "^4.0.4" +fast-json-patch@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/fast-json-patch/-/fast-json-patch-3.1.1.tgz#85064ea1b1ebf97a3f7ad01e23f9337e72c66947" + integrity sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ== + fast-json-stable-stringify@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" @@ -5232,7 +5384,7 @@ iconv-lite@0.4.24, iconv-lite@^0.4.24: dependencies: safer-buffer ">= 2.1.2 < 3" -iconv-lite@^0.6.2, iconv-lite@^0.6.3: +iconv-lite@0.6, iconv-lite@^0.6.2, iconv-lite@^0.6.3: version "0.6.3" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== @@ -5347,6 +5499,11 @@ internal-slot@^1.0.3: has "^1.0.3" side-channel "^1.0.4" +"internmap@1 - 2": + version "2.0.3" + resolved "https://registry.yarnpkg.com/internmap/-/internmap-2.0.3.tgz#6685f23755e43c524e251d29cbc97248e3061009" + integrity sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg== + ip-address@^9.0.5: version "9.0.5" resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-9.0.5.tgz#117a960819b08780c3bd1f14ef3c1cc1d3f3ea5a" @@ -5756,6 +5913,11 @@ json-stable-stringify-without-jsonify@^1.0.1: resolved "https://registry.yarnpkg.com/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz#9db7b59496ad3f3cfef30a75142d2d930ad72651" integrity sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw== +json-stringify-pretty-compact@^3.0.0, json-stringify-pretty-compact@~3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/json-stringify-pretty-compact/-/json-stringify-pretty-compact-3.0.0.tgz#f71ef9d82ef16483a407869556588e91b681d9ab" + integrity sha512-Rc2suX5meI0S3bfdZuA7JMFBGkJ875ApfVyq2WHELjBiiG22My/l7/8zPpH/CfFVQHuVLd8NLR0nv6vi0BYYKA== + json5@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.2.tgz#63d98d60f21b313b77c4d6da18bfa69d80e1d593" @@ -6116,11 +6278,11 @@ methods@~1.1.2: integrity sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w== micromatch@^4.0.2, micromatch@^4.0.4: - version "4.0.5" - resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6" - integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA== + version "4.0.8" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" + integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA== dependencies: - braces "^3.0.2" + braces "^3.0.3" picomatch "^2.3.1" mime-db@1.52.0, "mime-db@>= 1.43.0 < 2": @@ -7435,6 +7597,11 @@ rimraf@^3.0.0, rimraf@^3.0.2: dependencies: glob "^7.1.3" +robust-predicates@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/robust-predicates/-/robust-predicates-3.0.2.tgz#d5b28528c4824d20fc48df1928d41d9efa1ad771" + integrity sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg== + rollup@^4.2.0: version "4.14.0" resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.14.0.tgz#c3e2cd479f1b2358b65c1f810fa05b51603d7be8" @@ -7471,6 +7638,11 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" +rw@1: + version "1.3.3" + resolved "https://registry.yarnpkg.com/rw/-/rw-1.3.3.tgz#3f862dfa91ab766b14885ef4d01124bfda074fb4" + integrity sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ== + rxjs@7.8.1, rxjs@^7.8.1: version "7.8.1" resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-7.8.1.tgz#6f6f3d99ea8044291efd92e7c7fcf562c4057543" @@ -7579,6 +7751,11 @@ semver@^7.0.0, semver@^7.1.1, semver@^7.3.5, semver@^7.3.8, semver@^7.5.3, semve dependencies: lru-cache "^6.0.0" +semver@^7.6.2: + version "7.6.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.2.tgz#1e3b34759f896e8f14d6134732ce798aeb0c6e13" + integrity sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w== + send@0.18.0: version "0.18.0" resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" @@ -8241,6 +8418,13 @@ toidentifier@1.0.1: resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" integrity sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA== +topojson-client@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/topojson-client/-/topojson-client-3.1.0.tgz#22e8b1ed08a2b922feeb4af6f53b6ef09a467b99" + integrity sha512-605uxS6bcYxGXw9qi62XyrV6Q3xwbndjachmNxu8HWTtVPxZfEJN9fd/SZS1Q54Sn2y0TMyMxFj/cJINqGHrKw== + dependencies: + commander "2" + tr46@~0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" @@ -8314,6 +8498,11 @@ tslib@^2.3.1: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.1.tgz#e8a335add5ceae51aa261d32a490158ef042ef01" integrity sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw== +tslib@^2.6.3, tslib@~2.6.3: + version "2.6.3" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.3.tgz#0438f810ad7a9edcde7a241c3d80db693c8cbfe0" + integrity sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ== + tuf-js@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/tuf-js/-/tuf-js-2.2.0.tgz#4daaa8620ba7545501d04dfa933c98abbcc959b9" @@ -8492,6 +8681,371 @@ vary@^1, vary@~1.1.2: resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg== +vega-canvas@^1.2.7: + version "1.2.7" + resolved "https://registry.yarnpkg.com/vega-canvas/-/vega-canvas-1.2.7.tgz#cf62169518f5dcd91d24ad352998c2248f8974fb" + integrity sha512-OkJ9CACVcN9R5Pi9uF6MZBF06pO6qFpDYHWSKBJsdHP5o724KrsgR6UvbnXFH82FdsiTOff/HqjuaG8C7FL+9Q== + +vega-crossfilter@~4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/vega-crossfilter/-/vega-crossfilter-4.1.2.tgz#810281c279b3592310f12814bc61206dd42ca61d" + integrity sha512-J7KVEXkpfRJBfRvwLxn5vNCzQCNkrnzmDvkvwhuiwT4gPm5sk7MK5TuUP8GCl/iKYw+kWeVXEtrVHwWtug+bcQ== + dependencies: + d3-array "^3.2.2" + vega-dataflow "^5.7.6" + vega-util "^1.17.2" + +vega-dataflow@^5.7.6, vega-dataflow@~5.7.6: + version "5.7.6" + resolved "https://registry.yarnpkg.com/vega-dataflow/-/vega-dataflow-5.7.6.tgz#21dfad9120cb18d9aeaed578658670839d1adc95" + integrity sha512-9Md8+5iUC1MVKPKDyZ7pCEHk6I9am+DgaMzZqo/27O/KI4f23/WQXPyuI8jbNmc/mkm340P0TKREmzL5M7+2Dg== + dependencies: + vega-format "^1.1.2" + vega-loader "^4.5.2" + vega-util "^1.17.2" + +vega-embed@^6.26.0: + version "6.26.0" + resolved "https://registry.yarnpkg.com/vega-embed/-/vega-embed-6.26.0.tgz#25ca51783b2819adf6e6330ae6dd5771e8da8653" + integrity sha512-AZCTdKHDAuhp6TFZRQOOs332tStCwZr/5e4uZMNEuJL69A57cT66NNZJdNiCP6u66REzIToYtMJhMTL9wl5B3A== + dependencies: + fast-json-patch "^3.1.1" + json-stringify-pretty-compact "^3.0.0" + semver "^7.6.2" + tslib "^2.6.3" + vega-interpreter "^1.0.5" + vega-schema-url-parser "^2.2.0" + vega-themes "^2.15.0" + vega-tooltip "^0.34.0" + +vega-encode@~4.10.1: + version "4.10.1" + resolved "https://registry.yarnpkg.com/vega-encode/-/vega-encode-4.10.1.tgz#1656e20396db99c414f495704ef3d9cff99631df" + integrity sha512-d25nVKZDrg109rC65M8uxE+7iUrTxktaqgK4fU3XZBgpWlh1K4UbU5nDag7kiHVVN4tKqwgd+synEotra9TiVQ== + dependencies: + d3-array "^3.2.2" + d3-interpolate "^3.0.1" + vega-dataflow "^5.7.6" + vega-scale "^7.4.1" + vega-util "^1.17.2" + +vega-event-selector@^3.0.1, vega-event-selector@~3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/vega-event-selector/-/vega-event-selector-3.0.1.tgz#b99e92147b338158f8079d81b28b2e7199c2e259" + integrity sha512-K5zd7s5tjr1LiOOkjGpcVls8GsH/f2CWCrWcpKy74gTCp+llCdwz0Enqo013ZlGaRNjfgD/o1caJRt3GSaec4A== + +vega-expression@^5.0.1, vega-expression@^5.1.1, vega-expression@~5.1.0, vega-expression@~5.1.1: + version "5.1.1" + resolved "https://registry.yarnpkg.com/vega-expression/-/vega-expression-5.1.1.tgz#9b2d287a1f34d990577c9798ae68ec88453815ef" + integrity sha512-zv9L1Hm0KHE9M7mldHyz8sXbGu3KmC0Cdk7qfHkcTNS75Jpsem6jkbu6ZAwx5cNUeW91AxUQOu77r4mygq2wUQ== + dependencies: + "@types/estree" "^1.0.0" + vega-util "^1.17.2" + +vega-force@~4.2.1: + version "4.2.1" + resolved "https://registry.yarnpkg.com/vega-force/-/vega-force-4.2.1.tgz#bdce6ec8572867b4ff2fb7e09d2894798c5358ec" + integrity sha512-2BcuuqFr77vcCyKfcpedNFeYMxi+XEFCrlgLWNx7YV0PI8pdP5y/yPkzyuE9Tb894+KkRAvfQHZRAshcnFNcMw== + dependencies: + d3-force "^3.0.0" + vega-dataflow "^5.7.6" + vega-util "^1.17.2" + +vega-format@^1.1.2, vega-format@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/vega-format/-/vega-format-1.1.2.tgz#d344ba8a2680144e92127459c149a4181e9e7f84" + integrity sha512-0kUfAj0dg0U6GcEY0Kp6LiSTCZ8l8jl1qVdQyToMyKmtZg/q56qsiJQZy3WWRr1MtWkTIZL71xSJXgjwjeUaAw== + dependencies: + d3-array "^3.2.2" + d3-format "^3.1.0" + d3-time-format "^4.1.0" + vega-time "^2.1.2" + vega-util "^1.17.2" + +vega-functions@^5.15.0, vega-functions@~5.15.0: + version "5.15.0" + resolved "https://registry.yarnpkg.com/vega-functions/-/vega-functions-5.15.0.tgz#a7905e1dd6457efe265dbf954cbc0a5721c484b0" + integrity sha512-pCqmm5efd+3M65jrJGxEy3UGuRksmK6DnWijoSNocnxdCBxez+yqUUVX9o2pN8VxMe3648vZnR9/Vk5CXqRvIQ== + dependencies: + d3-array "^3.2.2" + d3-color "^3.1.0" + d3-geo "^3.1.0" + vega-dataflow "^5.7.6" + vega-expression "^5.1.1" + vega-scale "^7.4.1" + vega-scenegraph "^4.13.0" + vega-selections "^5.4.2" + vega-statistics "^1.9.0" + vega-time "^2.1.2" + vega-util "^1.17.2" + +vega-geo@~4.4.2: + version "4.4.2" + resolved "https://registry.yarnpkg.com/vega-geo/-/vega-geo-4.4.2.tgz#da4a08ee39c9488bfc4fe6493779f584dd8bb412" + integrity sha512-unuV/UxUHf6UJu6GYxMZonC3SZlMfFXYLOkgEsRSvmsMPt3+CVv8FmG88dXNRUJUrdROrJepgecqx0jOwMSnGA== + dependencies: + d3-array "^3.2.2" + d3-color "^3.1.0" + d3-geo "^3.1.0" + vega-canvas "^1.2.7" + vega-dataflow "^5.7.6" + vega-projection "^1.6.1" + vega-statistics "^1.9.0" + vega-util "^1.17.2" + +vega-hierarchy@~4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/vega-hierarchy/-/vega-hierarchy-4.1.2.tgz#e42938c42527b392b110b1e3bf89eaa456dba1b8" + integrity sha512-m+xDtT5092YPSnV0rdTLW+AWmoCb+A54JQ66MUJwiDBpKxvfKnTiQeuiWDU2YudjUoXZN9EBOcI6QHF8H2Lu2A== + dependencies: + d3-hierarchy "^3.1.2" + vega-dataflow "^5.7.6" + vega-util "^1.17.2" + +vega-interpreter@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/vega-interpreter/-/vega-interpreter-1.0.5.tgz#19e1d1b5f84a4ea9cb25c4e90a05ce16cd058484" + integrity sha512-po6oTOmeQqr1tzTCdD15tYxAQLeUnOVirAysgVEemzl+vfmvcEP7jQmlc51jz0jMA+WsbmE6oJywisQPu/H0Bg== + +vega-label@~1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/vega-label/-/vega-label-1.3.0.tgz#21b3e5ef40e63f51ac987a449d183068c4961503" + integrity sha512-EfSFSCWAwVPsklM5g0gUEuohALgryuGC/SKMmsOH7dYT/bywmLBZhLVbrE+IHJAUauoGrMhYw1mqnXL/0giJBg== + dependencies: + vega-canvas "^1.2.7" + vega-dataflow "^5.7.6" + vega-scenegraph "^4.13.0" + vega-util "^1.17.2" + +vega-lite@^5.19.0: + version "5.19.0" + resolved "https://registry.yarnpkg.com/vega-lite/-/vega-lite-5.19.0.tgz#5d49ac363585612658e7ca1f242ec9a1ec123d74" + integrity sha512-DtSArHnomfYdKnkz7rDCLkXpuh4O6kHVU1YSDoMtQR0ewEwqelb5YY85mvJPCJRT9E7UG84RGdxjDbuwowtHRg== + dependencies: + json-stringify-pretty-compact "~3.0.0" + tslib "~2.6.3" + vega-event-selector "~3.0.1" + vega-expression "~5.1.0" + vega-util "~1.17.2" + yargs "~17.7.2" + +vega-loader@^4.5.2, vega-loader@~4.5.2: + version "4.5.2" + resolved "https://registry.yarnpkg.com/vega-loader/-/vega-loader-4.5.2.tgz#7212f093c397b153f69f7e6cfef47817c17c5c01" + integrity sha512-ktIdGz3DRIS3XfTP9lJ6oMT5cKwC86nQkjUbXZbOtwXQFVNE2xVWBuH13GP6FKUZxg5hJCMtb5v/e/fwTvhKsQ== + dependencies: + d3-dsv "^3.0.1" + node-fetch "^2.6.7" + topojson-client "^3.1.0" + vega-format "^1.1.2" + vega-util "^1.17.2" + +vega-parser@~6.4.0: + version "6.4.0" + resolved "https://registry.yarnpkg.com/vega-parser/-/vega-parser-6.4.0.tgz#6a12f07f0f9178492a17842efe7e1f51a2d36bed" + integrity sha512-/hFIJs0yITxfvLIfhhcpUrcbKvu4UZYoMGmly5PSsbgo60oAsVQW8ZbX2Ji3iNFqZJh1ifoX/P0j+9wep1OISw== + dependencies: + vega-dataflow "^5.7.6" + vega-event-selector "^3.0.1" + vega-functions "^5.15.0" + vega-scale "^7.4.1" + vega-util "^1.17.2" + +vega-projection@^1.6.1, vega-projection@~1.6.1: + version "1.6.1" + resolved "https://registry.yarnpkg.com/vega-projection/-/vega-projection-1.6.1.tgz#da687abc60f4a93bb888385beb23e0a1000f8b57" + integrity sha512-sqfnAAHumU7MWU1tQN3b6HNgKGF3legek0uLHhjLKcDJQxEc7kwcD18txFz2ffQks6d5j+AUhBiq4GARWf0DEQ== + dependencies: + d3-geo "^3.1.0" + d3-geo-projection "^4.0.0" + vega-scale "^7.4.1" + +vega-regression@~1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/vega-regression/-/vega-regression-1.3.0.tgz#3e68e234fa9460041fac082c6a3469c896d436a8" + integrity sha512-gxOQfmV7Ft/MYKpXDEo09WZyBuKOBqxqDRWay9KtfGq/E0Y4vbTPsWLv2cB1ToPJdKE6XSN6Re9tCIw5M/yMUg== + dependencies: + d3-array "^3.2.2" + vega-dataflow "^5.7.6" + vega-statistics "^1.9.0" + vega-util "^1.17.2" + +vega-runtime@^6.2.0, vega-runtime@~6.2.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/vega-runtime/-/vega-runtime-6.2.0.tgz#10f435089fff11d8e1b49cb0cbab8041731e6f06" + integrity sha512-30UXbujWjKNd5aeP+oeHuwFmzuyVYlBj4aDy9+AjfWLECu8wJt4K01vwegcaGPdCWcPLVIv4Oa9Lob4mcXn5KQ== + dependencies: + vega-dataflow "^5.7.6" + vega-util "^1.17.2" + +vega-scale@^7.4.1, vega-scale@~7.4.1: + version "7.4.1" + resolved "https://registry.yarnpkg.com/vega-scale/-/vega-scale-7.4.1.tgz#2dcd3e39ebb00269b03a8be86e44c7b48c67442a" + integrity sha512-dArA28DbV/M92O2QvswnzCmQ4bq9WwLKUoyhqFYWCltmDwkmvX7yhqiFLFMWPItIm7mi4Qyoygby6r4DKd1X2A== + dependencies: + d3-array "^3.2.2" + d3-interpolate "^3.0.1" + d3-scale "^4.0.2" + d3-scale-chromatic "^3.1.0" + vega-time "^2.1.2" + vega-util "^1.17.2" + +vega-scenegraph@^4.13.0, vega-scenegraph@~4.13.0: + version "4.13.0" + resolved "https://registry.yarnpkg.com/vega-scenegraph/-/vega-scenegraph-4.13.0.tgz#c4fa5c82773f6244a9ca8b01a44e380adf03fabd" + integrity sha512-nfl45XtuqB5CxyIZJ+bbJ+dofzosPCRlmF+eUQo+0J23NkNXsTzur+1krJDSdhcw0SOYs4sbYRoMz1cpuOM4+Q== + dependencies: + d3-path "^3.1.0" + d3-shape "^3.2.0" + vega-canvas "^1.2.7" + vega-loader "^4.5.2" + vega-scale "^7.4.1" + vega-util "^1.17.2" + +vega-schema-url-parser@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/vega-schema-url-parser/-/vega-schema-url-parser-2.2.0.tgz#a0d1e02915adfbfcb1fd517c8c2ebe2419985c1e" + integrity sha512-yAtdBnfYOhECv9YC70H2gEiqfIbVkq09aaE4y/9V/ovEFmH9gPKaEgzIZqgT7PSPQjKhsNkb6jk6XvSoboxOBw== + +vega-selections@^5.4.2: + version "5.4.2" + resolved "https://registry.yarnpkg.com/vega-selections/-/vega-selections-5.4.2.tgz#cb4f41f5d4c0ee924ebf131b8dbd43e7885bcad4" + integrity sha512-99FUhYmg0jOJr2/K4TcEURmJRkuibrCDc8KBUX7qcQEITzrZ5R6a4QE+sarCvbb3hi8aA9GV2oyST6MQeA9mgQ== + dependencies: + d3-array "3.2.4" + vega-expression "^5.0.1" + vega-util "^1.17.1" + +vega-statistics@^1.9.0, vega-statistics@~1.9.0: + version "1.9.0" + resolved "https://registry.yarnpkg.com/vega-statistics/-/vega-statistics-1.9.0.tgz#7d6139cea496b22d60decfa6abd73346f70206f9" + integrity sha512-GAqS7mkatpXcMCQKWtFu1eMUKLUymjInU0O8kXshWaQrVWjPIO2lllZ1VNhdgE0qGj4oOIRRS11kzuijLshGXQ== + dependencies: + d3-array "^3.2.2" + +vega-themes@^2.15.0: + version "2.15.0" + resolved "https://registry.yarnpkg.com/vega-themes/-/vega-themes-2.15.0.tgz#cf7592efb45406957e9beb67d7033ee5f7b7a511" + integrity sha512-DicRAKG9z+23A+rH/3w3QjJvKnlGhSbbUXGjBvYGseZ1lvj9KQ0BXZ2NS/+MKns59LNpFNHGi9us/wMlci4TOA== + +vega-time@^2.1.2, vega-time@~2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/vega-time/-/vega-time-2.1.2.tgz#0c414e74780613d6d3234fb97f19b50c0ebd9f49" + integrity sha512-6rXc6JdDt8MnCRy6UzUCsa6EeFycPDmvioMddLfKw38OYCV8pRQC5nw44gyddOwXgUTJLiCtn/sp53P0iA542A== + dependencies: + d3-array "^3.2.2" + d3-time "^3.1.0" + vega-util "^1.17.2" + +vega-tooltip@^0.34.0: + version "0.34.0" + resolved "https://registry.yarnpkg.com/vega-tooltip/-/vega-tooltip-0.34.0.tgz#e0aa4d9c9bcf155e257650ba7e670fad7b1ff5ab" + integrity sha512-TtxwkcLZ5aWQTvKGlfWDou8tISGuxmqAW1AgGZjrDpf75qsXvgtbPdRAAls2LZMqDxpr5T1kMEZs9XbSpiI8yw== + dependencies: + vega-util "^1.17.2" + +vega-transforms@~4.12.0: + version "4.12.0" + resolved "https://registry.yarnpkg.com/vega-transforms/-/vega-transforms-4.12.0.tgz#6a69e0b67934b0c0a40a6f607fdb543bf749955e" + integrity sha512-bh/2Qbj85O70mjfLRgPKAsABArgSUP0k+GjmaY54zukIRxoGxKju+85nigeX/aR/INpEqNWif+5lL+NvmyWA5w== + dependencies: + d3-array "^3.2.2" + vega-dataflow "^5.7.6" + vega-statistics "^1.9.0" + vega-time "^2.1.2" + vega-util "^1.17.2" + +vega-typings@~1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/vega-typings/-/vega-typings-1.3.1.tgz#025a6031505794b44d9b6e2c49d4551b8918d4ae" + integrity sha512-j9Sdgmvowz09jkMgTFGVfiv7ycuRP/TQkdHRPXIYwt3RDgPQn7inyFcJ8C8ABFt4MiMWdjOwbneF6KWW8TRXIw== + dependencies: + "@types/geojson" "7946.0.4" + vega-event-selector "^3.0.1" + vega-expression "^5.1.1" + vega-util "^1.17.2" + +vega-util@^1.17.1, vega-util@^1.17.2, vega-util@~1.17.2: + version "1.17.2" + resolved "https://registry.yarnpkg.com/vega-util/-/vega-util-1.17.2.tgz#f69aa09fd5d6110c19c4a0f0af9e35945b99987d" + integrity sha512-omNmGiZBdjm/jnHjZlywyYqafscDdHaELHx1q96n5UOz/FlO9JO99P4B3jZg391EFG8dqhWjQilSf2JH6F1mIw== + +vega-view-transforms@~4.6.0: + version "4.6.0" + resolved "https://registry.yarnpkg.com/vega-view-transforms/-/vega-view-transforms-4.6.0.tgz#829d56ca3c8116b0dded4ec0502f4ac70253de9a" + integrity sha512-z3z66aJTA3ZRo4oBY4iBXnn+A4KqBGZT/UrlKDbm+7Ec+Ip+hK2tF8Kmhp/WNcMsDZoUWFqLJgR2VgOgvJk9RA== + dependencies: + vega-dataflow "^5.7.6" + vega-scenegraph "^4.13.0" + vega-util "^1.17.2" + +vega-view@~5.13.0: + version "5.13.0" + resolved "https://registry.yarnpkg.com/vega-view/-/vega-view-5.13.0.tgz#8ea96da9fcdf42fe7c0e95fe6258933477524745" + integrity sha512-ZPAAQ3iYz6YrQjJoDT+0bcxJkXt9PKF5v4OO7Omw8PFhkIv++jFXeKlQTW1bBtyQ92dkdGGHv5lYY67Djqjf3A== + dependencies: + d3-array "^3.2.2" + d3-timer "^3.0.1" + vega-dataflow "^5.7.6" + vega-format "^1.1.2" + vega-functions "^5.15.0" + vega-runtime "^6.2.0" + vega-scenegraph "^4.13.0" + vega-util "^1.17.2" + +vega-voronoi@~4.2.3: + version "4.2.3" + resolved "https://registry.yarnpkg.com/vega-voronoi/-/vega-voronoi-4.2.3.tgz#54c4bb96b9b94c3fa0160bee24695dcb9d583fe1" + integrity sha512-aYYYM+3UGqwsOx+TkVtF1IZfguy0H7AN79dR8H0nONRIc+vhk/lbnlkgwY2nSzEu0EZ4b5wZxeGoDBEVmdDEcg== + dependencies: + d3-delaunay "^6.0.2" + vega-dataflow "^5.7.6" + vega-util "^1.17.2" + +vega-wordcloud@~4.1.5: + version "4.1.5" + resolved "https://registry.yarnpkg.com/vega-wordcloud/-/vega-wordcloud-4.1.5.tgz#789c9e67225c77f3f35a6fc052beec1c2bdc8b5e" + integrity sha512-p+qXU3cb9VeWzJ/HEdax0TX2mqDJcSbrCIfo2d/EalOXGkvfSLKobsmMQ8DxPbtVp0uhnpvfCGDyMJw+AzcI2A== + dependencies: + vega-canvas "^1.2.7" + vega-dataflow "^5.7.6" + vega-scale "^7.4.1" + vega-statistics "^1.9.0" + vega-util "^1.17.2" + +vega@^5.30.0: + version "5.30.0" + resolved "https://registry.yarnpkg.com/vega/-/vega-5.30.0.tgz#d12350c829878b481453ab28ce10855a954df06d" + integrity sha512-ZGoC8LdfEUV0LlXIuz7hup9jxuQYhSaWek2M7r9dEHAPbPrzSQvKXZ0BbsJbrarM100TGRpTVN/l1AFxCwDkWw== + dependencies: + vega-crossfilter "~4.1.2" + vega-dataflow "~5.7.6" + vega-encode "~4.10.1" + vega-event-selector "~3.0.1" + vega-expression "~5.1.1" + vega-force "~4.2.1" + vega-format "~1.1.2" + vega-functions "~5.15.0" + vega-geo "~4.4.2" + vega-hierarchy "~4.1.2" + vega-label "~1.3.0" + vega-loader "~4.5.2" + vega-parser "~6.4.0" + vega-projection "~1.6.1" + vega-regression "~1.3.0" + vega-runtime "~6.2.0" + vega-scale "~7.4.1" + vega-scenegraph "~4.13.0" + vega-statistics "~1.9.0" + vega-time "~2.1.2" + vega-transforms "~4.12.0" + vega-typings "~1.3.1" + vega-util "~1.17.2" + vega-view "~5.13.0" + vega-view-transforms "~4.6.0" + vega-voronoi "~4.2.3" + vega-wordcloud "~4.1.5" + vite@5.1.5: version "5.1.5" resolved "https://registry.yarnpkg.com/vite/-/vite-5.1.5.tgz#bdbc2b15e8000d9cc5172f059201178f9c9de5fb" @@ -8803,7 +9357,7 @@ yargs-parser@^20.2.2: resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.7.tgz#61df85c113edfb5a7a4e36eb8aa60ef423cbc90a" integrity sha512-FiNkvbeHzB/syOjIUxFDCnhSfzAL8R5vs40MgLFBorXACCOAEaWu0gRZl14vG8MR9AOJIZbmkjhusqBYZ3HTHw== -yargs@17.7.2, yargs@^17.2.1, yargs@^17.6.2: +yargs@17.7.2, yargs@^17.2.1, yargs@^17.6.2, yargs@~17.7.2: version "17.7.2" resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269" integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== diff --git a/package.json b/package.json index 937dc3e55..3f93a45c3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "i-analyzer", - "version": "5.9.0", + "version": "5.12.0", "license": "MIT", "scripts": { "postinstall": "yarn install-back && yarn install-front", @@ -31,7 +31,11 @@ "watch-front-p": "yarn front yarn watch", "start-back-p": "cd backend && python manage.py runserver --settings production --insecure --pythonpath ..", "start-p": "yarn static-p && yarn watch-front-p & yarn start-back-p", - "celery": "yarn back celery -A ianalyzer.celery" + "celery": "yarn back celery -A ianalyzer.celery", + "patch": "yarn version --patch --no-commit-hooks --no-git-tag-version && yarn update-citation && yarn fyarn prebuild", + "minor": "yarn version --minor --no-commit-hooks --no-git-tag-version && yarn update-citation && yarn fyarn prebuild", + "major": "yarn version --major --no-commit-hooks --no-git-tag-version && yarn update-citation && yarn fyarn prebuild", + "update-citation": "python $PWD/update_citation.py" }, "private": true, "devDependencies": {}, diff --git a/update_citation.py b/update_citation.py new file mode 100644 index 000000000..b965035b7 --- /dev/null +++ b/update_citation.py @@ -0,0 +1,35 @@ +'''Updates the CITATION.cff file: + - Sets the date-released to the current date + - Sets the version from toplevel package.json +''' + +from datetime import datetime +import json +import re + +CITATION_FILE = 'CITATION.cff' +PACKAGE_FILE = 'package.json' +VERSION_PATTERN = r'^version:\s+.*$' +DATE_RELEASED_PATTERN = r'^date-released:.*$' +VERSION = None +TODAY = datetime.today().strftime('%Y-%m-%d') + +with open(PACKAGE_FILE, 'r') as package_file: + package_json = json.load(package_file) + VERSION = package_json.get('version') + + +with open(CITATION_FILE) as citation_file: + citation_in = citation_file.readlines() + citation_out = [] + + for line in citation_in: + if re.match(VERSION_PATTERN, line): + citation_out.append(f'version: {VERSION}\n') + elif re.match(DATE_RELEASED_PATTERN, line): + citation_out.append(f"date-released: '{TODAY}'\n") + else: + citation_out.append(line) + +with open(CITATION_FILE, 'w') as citation_file: + citation_file.writelines(citation_out)