Merge pull request alephdata#3462 from alephdata/release/3.15.4

Release/3.15.4 into main
brassy-endomorph · Nov 2, 2023 · 5318c77 · 5318c77
2 parents f222d8c + 5e65006
commit 5318c77
Show file tree

Hide file tree

Showing 31 changed files with 10,567 additions and 3,001 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.15.3
+current_version = 3.15.4
 tag_name = {new_version}
 commit = True
 tag = True

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -8,10 +8,6 @@ on:
       - "**"
   workflow_dispatch: {}
 
-concurrency:
-  group: pages
-  cancel-in-progress: true
-
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -34,6 +30,10 @@ jobs:
     needs: build
     runs-on: ubuntu-latest
 
+    concurrency:
+      group: pages
+      cancel-in-progress: true
+
     permissions:
       pages: write
       id-token: write

diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml
@@ -16,12 +16,14 @@ jobs:
       - name: Get the version
         id: get_version
         run: echo ::set-output name=VERSION::$(cat helm/charts/aleph/Chart.yaml | grep version | awk '{print $2}')
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v0
+      - id: 'auth'
+        uses: 'google-github-actions/auth@v1'
         with:
+          credentials_json: '${{ secrets.GCP_CREDENTIALS }}'
           project_id: ${{ secrets.GCP_PROJECT_ID }}
-          service_account_key: ${{ secrets.GCP_CREDENTIALS }}
           export_default_credentials: true
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v1
       - name: Copy existing charts for index regeneration
         run: gsutil -m cp -r gs://aleph-helm-charts . && mv aleph-helm-charts charts
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,21 @@
+## 3.15.3 (26-10-2023)
+- Reduced docker image size
+- Updated SECURITY.md
+- New user guide
+- Make it possible to manually trigger a docs deployment by @tillprochaska in #3226
+- Added redirects for the old user guide
+- Updates to the dev environment (note the postgres container version upgrade)
+- Fix multiple token highlighting in search results
+- Updated dependencies
+
+## 3.15.1 (08-09-2023)
+- Document how to enable IAM role-based auth between EC2 and S3
+- Add a simple script to generate test emails
+- Derive "safeHtml" from all "bodyHtml" values
+- Fix user guide link
+- Recommend ingest-file 3.19.2
+- Updated dependencies
+
 ## 3.14.3 (28-06-2023)
 - Introduced two new Settings which controll the scroll window of ElasticSearch queries made during xref operations:
     - ALEPH_XREF_SCROLL (defaults to 5m) is the 'scroll' parameter used on ES scan() calls for xref operations and configures how long a consistent view of the index should be maintained for scrolled search

diff --git a/aleph/logic/xref.py b/aleph/logic/xref.py
@@ -146,6 +146,8 @@ def _iter_mentions(collection):
         collection_id=collection.id,
         schemata=["Mention"],
         sort={"properties.resolved": "desc"},
+        es_scroll=SETTINGS.XREF_SCROLL,
+        es_scroll_size=SETTINGS.XREF_SCROLL_SIZE,
     ):
         resolved_id = mention.first("resolved")
         if resolved_id != proxy.id:
@@ -212,12 +214,15 @@ def xref_entity(collection, proxy):
 
 def xref_collection(collection):
     """Cross-reference all the entities and documents in a collection."""
-    log.info("[%s] Clearing previous xref state....", collection)
+    log.info(
+        f"[{collection}] xref_collection scroll settings: scroll={SETTINGS.XREF_SCROLL}, scroll_size={SETTINGS.XREF_SCROLL_SIZE}"
+    )
+    log.info(f"[{collection}] Clearing previous xref state....")
     delete_xref(collection, sync=True)
     delete_entities(collection.id, origin=ORIGIN, sync=True)
     index_matches(collection, _query_entities(collection))
     index_matches(collection, _query_mentions(collection))
-    log.info("[%s] Xref done, re-indexing to reify mentions...", collection)
+    log.info(f"[{collection}] Xref done, re-indexing to reify mentions...")
     reindex_collection(collection, sync=False)
 
 

diff --git a/contrib/aleph-traefik-minio-keycloak/docker-compose.yml b/contrib/aleph-traefik-minio-keycloak/docker-compose.yml
@@ -36,7 +36,7 @@ services:
       - "traefik.enable=false"
 
   ingest-file:
-    image: ghcr.io/alephdata/ingest-file:3.19.2
+    image: ghcr.io/alephdata/ingest-file:3.19.3
     tmpfs:
       - /tmp:mode=777
     volumes:
@@ -54,7 +54,7 @@ services:
       - "traefik.enable=false"
 
   worker:
-    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-3.15.3}
+    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-3.15.4}
     command: aleph worker
     restart: on-failure
     links:
@@ -79,7 +79,7 @@ services:
       - "traefik.enable=false"
 
   shell:
-    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-3.15.3}
+    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-3.15.4}
     command: /bin/bash
     depends_on:
       - postgres
@@ -99,7 +99,7 @@ services:
       - "traefik.enable=false"
 
   api:
-    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-3.15.3}
+    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-3.15.4}
     command: gunicorn -w 6 -b 0.0.0.0:8000 --log-level debug --log-file - aleph.wsgi:app
     expose:
       - 8000
@@ -121,7 +121,7 @@ services:
       - "traefik.enable=false"
 
   ui:
-    image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-3.15.3}
+    image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-3.15.4}
     depends_on:
       - api
       - traefik

diff --git a/contrib/keycloak/docker-compose.dev-keycloak.yml b/contrib/keycloak/docker-compose.dev-keycloak.yml
@@ -16,7 +16,7 @@ services:
   elasticsearch:
     build:
       context: services/elasticsearch
-    image: ghcr.io/alephdata/aleph-elasticsearch:${ALEPH_TAG:-3.15.3}
+    image: ghcr.io/alephdata/aleph-elasticsearch:${ALEPH_TAG:-3.15.4}
     hostname: elasticsearch
     environment:
       - discovery.type=single-node
@@ -35,7 +35,7 @@ services:
   ingest-file:
     build:
       context: services/ingest-file
-    image: ghcr.io/alephdata/ingest-file:3.19.2
+    image: ghcr.io/alephdata/ingest-file:3.19.3
     hostname: ingest
     tmpfs: /tmp
     volumes:
@@ -55,7 +55,7 @@ services:
   app:
     build:
       context: .
-    image: alephdata/aleph:${ALEPH_TAG:-3.15.3}
+    image: alephdata/aleph:${ALEPH_TAG:-3.15.4}
     hostname: aleph
     command: /bin/bash
     links:
@@ -83,7 +83,7 @@ services:
   api:
     build:
       context: .
-    image: alephdata/aleph:${ALEPH_TAG:-3.15.3}
+    image: alephdata/aleph:${ALEPH_TAG:-3.15.4}
     command: aleph run -h 0.0.0.0 -p 5000 --with-threads --reload --debugger
     ports:
       - "127.0.0.1:5000:5000"
@@ -117,7 +117,7 @@ services:
   ui:
     build:
       context: ui
-    image: alephdata/aleph-ui:${ALEPH_TAG:-3.15.3}
+    image: alephdata/aleph-ui:${ALEPH_TAG:-3.15.4}
     links:
       - api
     command: npm run start

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -35,7 +35,7 @@ services:
       - redis-data:/data
 
   ingest-file:
-    image: ghcr.io/alephdata/ingest-file:3.19.2
+    image: ghcr.io/alephdata/ingest-file:3.19.3
     hostname: ingest
     tmpfs: /tmp
     volumes:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -25,7 +25,7 @@ services:
       - redis-data:/data
 
   ingest-file:
-    image: ghcr.io/alephdata/ingest-file:3.19.2
+    image: ghcr.io/alephdata/ingest-file:3.19.3
     tmpfs:
       - /tmp:mode=777
     volumes:
@@ -38,7 +38,7 @@ services:
       - aleph.env
 
   worker:
-    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-3.15.3}
+    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-3.15.4}
     command: aleph worker
     restart: on-failure
     depends_on:
@@ -54,7 +54,7 @@ services:
       - aleph.env
 
   shell:
-    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-3.15.3}
+    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-3.15.4}
     command: /bin/bash
     depends_on:
       - postgres
@@ -72,7 +72,7 @@ services:
       - aleph.env
 
   api:
-    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-3.15.3}
+    image: ghcr.io/alephdata/aleph:${ALEPH_TAG:-3.15.4}
     command: gunicorn -w 6 -b 0.0.0.0:8000 --timeout 3600 --log-level debug --log-file - aleph.wsgi:app
     expose:
       - 8000
@@ -90,7 +90,7 @@ services:
       - aleph.env
 
   ui:
-    image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-3.15.3}
+    image: ghcr.io/alephdata/aleph-ui-production:${ALEPH_TAG:-3.15.4}
     depends_on:
       - api
     ports:

diff --git a/docs/package-lock.json b/docs/package-lock.json
diff --git a/docs/src/pages/users/faq/account/index.mdx b/docs/src/pages/users/faq/account/index.mdx
@@ -5,6 +5,10 @@ title: Account & Recovery
 
 # Account creation and password recovery
 
+<Callout>
+  This page answers common questions about accounts on [aleph.occrp.org](https://aleph.occrp.org), an Aleph instance operated by the OCCRP. If your organization runs its own in-house Aleph instance, the information on this page may not apply.
+</Callout>
+
 ## When I enter the code from my 2FA app to log in to Aleph, it says it is incorrect. I tried several times and it did not work. What should I do?
 
 If you are receiving an error message after putting in your one time password, this issue is most likely caused by the current time on your phone or laptop. Follow the troubleshooting steps below:

diff --git a/docs/src/pages/users/getting-started/account/index.mdx b/docs/src/pages/users/getting-started/account/index.mdx
@@ -9,6 +9,10 @@ title: Manage your account
 
 <p class="lead">On this page, you will find instructions on how to create an account for OCCRP Aleph and how to reset your password.</p>
 
+<Callout>
+  This page describes how to create an account on [aleph.occrp.org](https://aleph.occrp.org), an Aleph instance operated by the OCCRP. If your organization runs its own in-house Aleph instance, the process might be different.
+</Callout>
+
 {/*TODO Add warning: Sign up / sign in may be different for in-house instances*/}
 
 <VideoTutorial youtubeId="asIh-pMApts" />

diff --git a/e2e/Dockerfile b/e2e/Dockerfile
@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/playwright/python:v1.32.1-focal
+FROM mcr.microsoft.com/playwright/python:v1.39.0-focal
 COPY entrypoint.sh wait-for ./
 RUN apt update -y \
     && apt install -y netcat wget \

diff --git a/helm/charts/aleph/Chart.yaml b/helm/charts/aleph/Chart.yaml
@@ -2,5 +2,5 @@ apiVersion: v2
 name: aleph
 description: Helm chart for Aleph
 type: application
-version: 3.15.3
-appVersion: 3.15.3
+version: 3.15.4
+appVersion: 3.15.4
diff --git a/helm/charts/aleph/README.md b/helm/charts/aleph/README.md
@@ -11,7 +11,7 @@ Helm chart for Aleph
 | global.amazon | bool | `true` | Are we using AWS services like s3? |
 | global.google | bool | `false` | Are we using GCE services like storage, vision api? |
 | global.image.repository | string | `"alephdata/aleph"` | Aleph docker image repo |
-| global.image.tag | string | `"3.15.3"` | Aleph docker image tag |
+| global.image.tag | string | `"3.15.4"` | Aleph docker image tag |
 | global.image.tag | string | `"Always"` |  |
 | global.namingPrefix | string | `"aleph"` | Prefix for the names of k8s resources |
 

diff --git a/helm/charts/aleph/templates/api.yaml b/helm/charts/aleph/templates/api.yaml
@@ -133,7 +133,7 @@ spec:
         - name: home-volume
           emptyDir: {}
 ---
-apiVersion: autoscaling/v2beta1
+apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:
   name: {{ .Values.global.namingPrefix }}-api

diff --git a/helm/charts/aleph/templates/ingest-file.yaml b/helm/charts/aleph/templates/ingest-file.yaml
@@ -90,7 +90,7 @@ spec:
           emptyDir:
             medium: Memory
 ---
-apiVersion: autoscaling/v2beta1
+apiVersion: autoscaling/v2
 kind: HorizontalPodAutoscaler
 metadata:
   name: {{ .Values.global.namingPrefix }}-ingest-file

diff --git a/helm/charts/aleph/values.yaml b/helm/charts/aleph/values.yaml
@@ -6,7 +6,7 @@ global:
 
   image:
     repository: ghcr.io/alephdata/aleph
-    tag: "3.15.3"
+    tag: "3.15.4"
     pullPolicy: Always
 
   commonEnv:
@@ -85,7 +85,9 @@ api:
       - type: Resource
         resource:
           name: cpu
-          targetAverageUtilization: 60
+          target:
+            type: Utilization
+            averageUtilization: 60
 
 # Aleph Upgrade Job - templates/aleph-upgrade-job.yaml
 upgrade:
@@ -140,7 +142,9 @@ ingestfile:
       - type: Resource
         resource:
           name: cpu
-          targetAverageUtilization: 100
+          target:
+            averageUtilization: 100
+            type: Utilization
 
 # Aleph UI - templates/ui.yaml
 ui:

diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -1,6 +1,6 @@
 bump2version==1.0.1
 Babel==2.12.1
-black==23.7.0
-playwright==1.37.0
+black==23.9.1
+playwright==1.39.0
 pytest-playwright==0.4.2
-ruff===0.0.287
+ruff===0.0.292