From ca65c422208f235ed198fa662982fb4af31bea96 Mon Sep 17 00:00:00 2001 From: ngken0995 <59739226+ngken0995@users.noreply.github.com> Date: Tue, 26 Sep 2023 12:47:17 -0400 Subject: [PATCH] Replace deprecated body in the ES requests with individual parameters (#3055) * change deprecated ES search body * trim white space Add thumbnail repsonse time runbooks (#3053) * Add thumb repsonse time runbooks * Add files to index * Threshold alarms are low severity if not anomalous generate-dag-docs recipe move DAGs.md to documentation folder (#3061) Co-authored-by: Madison Swain-Bowden Upgrade psycopg to version 3 in the API (#3064) Update VSourcesTable.vue (#3026) Co-authored-by: sarayourfriend <24264157+sarayourfriend@users.noreply.github.com> Co-authored-by: Olga Bulat Co-authored-by: Krystle Salazar Transfer UUID validation inside serializer (#3068) * Transfer UUID validation inside serializer * Add test case Publish changelog for api-2023.09.28.00.26.34 (#3070) Co-authored-by: AetherUnbound Use fully qualified docker image names (#3071) Publish changelog for ingestion_server-2023.09.29.17.40.50 (#3082) Co-authored-by: stacimc Update `_AIRFLOW_DB_UPGRADE` to `_AIRFLOW_DB_MIGRATE` Increase the API sources cache TTL from 20 minutes to 4 hours (#3083) Publish changelog for api-2023.09.30.00.15.32 (#3084) Co-authored-by: AetherUnbound Bump ipython from 8.14.0 to 8.16.0 in /automations/python (#3099) Bumps [ipython](https://github.com/ipython/ipython) from 8.14.0 to 8.16.0. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/8.14.0...8.16.0) --- updated-dependencies: - dependency-name: ipython dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Remove boto3 dependency (#3073) Bump docker/login-action from 2 to 3 (#3089) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump elasticsearch from 8.8.2 to 8.10.0 in /api (#3103) Bumps [elasticsearch](https://github.com/elastic/elasticsearch-py) from 8.8.2 to 8.10.0. - [Release notes](https://github.com/elastic/elasticsearch-py/releases) - [Commits](https://github.com/elastic/elasticsearch-py/compare/v8.8.2...v8.10.0) --- updated-dependencies: - dependency-name: elasticsearch dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump psycopg2 from 2.9.7 to 2.9.8 in /ingestion_server (#3097) Bumps [psycopg2](https://github.com/psycopg/psycopg2) from 2.9.7 to 2.9.8. - [Changelog](https://github.com/psycopg/psycopg2/blob/master/NEWS) - [Commits](https://github.com/psycopg/psycopg2/compare/2.9.7...2.9.8) --- updated-dependencies: - dependency-name: psycopg2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump elasticsearch from 8.8.2 to 8.10.0 in /ingestion_server (#3095) Bumps [elasticsearch](https://github.com/elastic/elasticsearch-py) from 8.8.2 to 8.10.0. - [Release notes](https://github.com/elastic/elasticsearch-py/releases) - [Commits](https://github.com/elastic/elasticsearch-py/compare/v8.8.2...v8.10.0) --- updated-dependencies: - dependency-name: elasticsearch dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump pygithub from 1.59.1 to 2.1.1 in /automations/python (#3102) Bumps [pygithub](https://github.com/pygithub/pygithub) from 1.59.1 to 2.1.1. - [Release notes](https://github.com/pygithub/pygithub/releases) - [Changelog](https://github.com/PyGithub/PyGithub/blob/main/doc/changes.rst) - [Commits](https://github.com/pygithub/pygithub/compare/v1.59.1...v2.1.1) --- updated-dependencies: - dependency-name: pygithub dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump furo from 2023.8.19 to 2023.9.10 in /documentation (#3092) Bumps [furo](https://github.com/pradyunsg/furo) from 2023.8.19 to 2023.9.10. - [Release notes](https://github.com/pradyunsg/furo/releases) - [Changelog](https://github.com/pradyunsg/furo/blob/main/docs/changelog.md) - [Commits](https://github.com/pradyunsg/furo/compare/2023.08.19...2023.09.10) --- updated-dependencies: - dependency-name: furo dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump docker/build-push-action from 4 to 5 (#3090) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4 to 5. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v4...v5) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump actions/checkout from 3 to 4 (#3088) Bump docker/setup-buildx-action from 2 to 3 (#3087) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump fakeredis from 2.18.0 to 2.19.0 in /api (#3100) Bumps [fakeredis](https://github.com/cunla/fakeredis-py) from 2.18.0 to 2.19.0. - [Release notes](https://github.com/cunla/fakeredis-py/releases) - [Commits](https://github.com/cunla/fakeredis-py/compare/v2.18.0...v2.19.0) --- updated-dependencies: - dependency-name: fakeredis dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Show timeout errors on the frontend (#2838) * Show timeout errors on the frontend * Use FetchingError in all stores * Fix error * Show client-side errors on single result pages * Set 500 as a non-retriable error * Add changes from code review * Use local base64 image for thumbnail * Fix footer * Fix image-cell test * Extract common error checking functionality * Update unit tests Add runbooks for API Thumbnails 2XX/5XX responses and Request Count alarms (#3076) Bump jsonschema from 4.19.0 to 4.19.1 in /ingestion_server (#3098) Bumps [jsonschema](https://github.com/python-jsonschema/jsonschema) from 4.19.0 to 4.19.1. - [Release notes](https://github.com/python-jsonschema/jsonschema/releases) - [Changelog](https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst) - [Commits](https://github.com/python-jsonschema/jsonschema/compare/v4.19.0...v4.19.1) --- updated-dependencies: - dependency-name: jsonschema dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump ipython from 8.15.0 to 8.16.1 in /ingestion_server (#3110) Bumps [ipython](https://github.com/ipython/ipython) from 8.15.0 to 8.16.1. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/commits) --- updated-dependencies: - dependency-name: ipython dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump sphinx from 7.2.5 to 7.2.6 in /documentation (#3091) Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 7.2.5 to 7.2.6. - [Release notes](https://github.com/sphinx-doc/sphinx/releases) - [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES.rst) - [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.2.5...v7.2.6) --- updated-dependencies: - dependency-name: sphinx dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump tldextract from 3.4.4 to 3.6.0 in /catalog (#3093) Bumps [tldextract](https://github.com/john-kurkowski/tldextract) from 3.4.4 to 3.6.0. - [Release notes](https://github.com/john-kurkowski/tldextract/releases) - [Changelog](https://github.com/john-kurkowski/tldextract/blob/master/CHANGELOG.md) - [Commits](https://github.com/john-kurkowski/tldextract/compare/3.4.4...3.6.0) --- updated-dependencies: - dependency-name: tldextract dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump urllib3 from 2.0.5 to 2.0.6 in /documentation (#3120) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump urllib3 from 1.26.16 to 1.26.17 in /api (#3123) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump urllib3 from 2.0.5 to 2.0.6 in /automations/python (#3119) Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.0.5 to 2.0.6. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/v2.0.5...2.0.6) --- updated-dependencies: - dependency-name: urllib3 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Bump urllib3 from 1.26.16 to 1.26.17 in /ingestion_server (#3124) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> implement vale determine files mircosoft sentence format change alert level change alert level change alert level change alert level change alert level change alert level change alert level Empty-Commit --- .github/workflows/actionlint.yml | 2 +- .github/workflows/bundle_size.yml | 6 +- .github/workflows/ci_cd.yml | 48 +- .github/workflows/generate_pot.yml | 6 +- .github/workflows/label_new_pr.yml | 2 +- .github/workflows/label_pr.yml | 2 +- .github/workflows/label_sync.yml | 2 +- .../workflows/migration_safety_warning.yml | 2 +- .github/workflows/pr_closed.yml | 2 +- .github/workflows/pr_label_check.yml | 2 +- .github/workflows/pr_limit_reminders.yml | 2 +- .../project_thread_update_reminders.yml | 2 +- .github/workflows/push_docker_image.yml | 2 +- .github/workflows/release-app.yml | 4 +- .github/workflows/renovate.yml | 2 +- .github/workflows/sync_meta.yml | 2 +- .github/workflows/weekly_updates.yml | 2 +- api/Dockerfile | 11 +- api/Pipfile | 7 +- api/Pipfile.lock | 468 ++++++++---------- api/api/controllers/search_controller.py | 18 +- api/api/serializers/image_serializers.py | 11 +- api/api/views/image_views.py | 5 +- api/test/test_image_integration.py | 10 + .../commands/test_generatewaveforms.py | 4 +- automations/python/Pipfile.lock | 364 +++++++------- catalog/justfile | 8 +- catalog/requirements_prod.txt | 2 +- .../dag_doc_gen/test_dag_doc_generation.py | 12 +- .../dag_doc_gen/dag_doc_generation.py | 14 +- docker-compose.yml | 16 +- documentation/Pipfile.lock | 220 ++++---- .../catalog/reference}/DAGs.md | 160 +++--- documentation/catalog/reference/index.md | 7 + .../changelogs/api/2023.09.28.00.26.34.md | 13 + .../changelogs/api/2023.09.30.00.15.32.md | 6 + .../ingestion_server/2023.09.29.17.40.50.md | 11 + ...nails_avg_response_time_above_threshold.md | 30 ++ ...pi_thumbnails_avg_response_time_anomaly.md | 28 ++ ...api_thumbnails_http_2xx_under_threshold.md | 36 ++ ...api_thumbnails_http_5xx_above_threshold.md | 31 ++ ...nails_p99_response_time_above_threshold.md | 30 ++ ...pi_thumbnails_p99_response_time_anomaly.md | 28 ++ .../api_thumbnails_request_count_anomaly.md | 34 ++ .../meta/monitoring/runbooks/index.md | 7 + frontend/Dockerfile | 4 +- .../VErrorSection/VErrorSection.vue | 22 +- .../meta/VErrorSection.stories.mdx | 37 +- .../VExternalSearch/VExternalSearchForm.vue | 2 +- frontend/src/components/VFooter/VFooter.vue | 2 +- .../VSearchResultsGrid/VAudioCollection.vue | 4 +- .../VSearchResultsGrid/VImageGrid.vue | 10 +- frontend/src/components/VSourcesTable.vue | 5 + frontend/src/constants/errors.ts | 40 +- frontend/src/middleware/search.ts | 11 +- frontend/src/middleware/single-result.ts | 9 +- frontend/src/pages/audio/_id/index.vue | 13 +- frontend/src/pages/image/_id/index.vue | 16 +- frontend/src/pages/search.vue | 32 +- frontend/src/pages/search/audio.vue | 4 +- frontend/src/pages/search/image.vue | 4 +- frontend/src/stores/media/index.ts | 105 ++-- frontend/src/stores/media/related-media.ts | 24 +- frontend/src/stores/media/single-result.ts | 32 +- frontend/src/stores/provider.ts | 21 +- frontend/src/types/fetch-state.ts | 35 +- frontend/src/utils/errors.ts | 88 ++++ frontend/test/playwright/e2e/filters.spec.ts | 2 +- .../sources-ltr-2xl-linux.png | Bin 410687 -> 420349 bytes .../sources-ltr-lg-linux.png | Bin 404788 -> 413683 bytes .../sources-ltr-md-linux.png | Bin 405669 -> 412388 bytes .../sources-ltr-sm-linux.png | Bin 390596 -> 397563 bytes .../sources-ltr-xl-linux.png | Bin 402952 -> 412223 bytes .../sources-ltr-xs-linux.png | Bin 359325 -> 364873 bytes .../sources-rtl-2xl-linux.png | Bin 386464 -> 394303 bytes .../sources-rtl-lg-linux.png | Bin 377067 -> 384873 bytes .../sources-rtl-md-linux.png | Bin 379285 -> 385993 bytes .../sources-rtl-sm-linux.png | Bin 365372 -> 372481 bytes .../sources-rtl-xl-linux.png | Bin 378780 -> 386552 bytes .../sources-rtl-xs-linux.png | Bin 333313 -> 338935 bytes frontend/test/tapes/response-1.json5 | 16 +- .../unit/specs/stores/media-store.spec.js | 71 +-- .../test/unit/specs/stores/provider.spec.js | 36 +- .../specs/stores/single-result-store.spec.js | 29 +- ingestion_server/Dockerfile | 4 +- ingestion_server/Pipfile | 4 +- ingestion_server/Pipfile.lock | 297 ++++++----- utilities/load_testing/Dockerfile | 2 +- 88 files changed, 1557 insertions(+), 1105 deletions(-) rename {catalog => documentation/catalog/reference}/DAGs.md (95%) create mode 100644 documentation/catalog/reference/index.md create mode 100644 documentation/changelogs/api/2023.09.28.00.26.34.md create mode 100644 documentation/changelogs/api/2023.09.30.00.15.32.md create mode 100644 documentation/changelogs/ingestion_server/2023.09.29.17.40.50.md create mode 100644 documentation/meta/monitoring/runbooks/api_thumbnails_avg_response_time_above_threshold.md create mode 100644 documentation/meta/monitoring/runbooks/api_thumbnails_avg_response_time_anomaly.md create mode 100644 documentation/meta/monitoring/runbooks/api_thumbnails_http_2xx_under_threshold.md create mode 100644 documentation/meta/monitoring/runbooks/api_thumbnails_http_5xx_above_threshold.md create mode 100644 documentation/meta/monitoring/runbooks/api_thumbnails_p99_response_time_above_threshold.md create mode 100644 documentation/meta/monitoring/runbooks/api_thumbnails_p99_response_time_anomaly.md create mode 100644 documentation/meta/monitoring/runbooks/api_thumbnails_request_count_anomaly.md create mode 100644 frontend/src/utils/errors.ts diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index 85822de68ba..3309aa0ff10 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -17,7 +17,7 @@ jobs: actionlint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Check workflow files uses: docker://rhysd/actionlint:latest with: diff --git a/.github/workflows/bundle_size.yml b/.github/workflows/bundle_size.yml index 29b34ff2361..6e79d4c6574 100644 --- a/.github/workflows/bundle_size.yml +++ b/.github/workflows/bundle_size.yml @@ -22,7 +22,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get changes id: paths-filter @@ -39,7 +39,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env @@ -58,4 +58,4 @@ jobs: - name: Checkout repository # again, to enable cleaning if: always() - uses: actions/checkout@v3 + uses: actions/checkout@v4 diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 9b93ee54369..b0ff59ec849 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -40,7 +40,7 @@ jobs: packages: ${{ contains(fromJson(steps.paths-filter.outputs.changes), 'packages') }} steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get changes id: paths-filter @@ -75,7 +75,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set matrix images id: set-matrix @@ -95,7 +95,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env @@ -163,7 +163,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Sets up `just` and Node.js. # Node.js is needed for the frontend to download translations. @@ -192,12 +192,12 @@ jobs: just frontend/run i18n - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: install: true - name: Build image `${{ matrix.image }}` - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: ${{ matrix.context || matrix.image }} target: ${{ matrix.target }} @@ -239,7 +239,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env @@ -271,7 +271,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env @@ -307,7 +307,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env @@ -355,7 +355,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env @@ -419,7 +419,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env @@ -485,7 +485,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env @@ -519,7 +519,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env @@ -573,7 +573,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env @@ -632,7 +632,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env @@ -749,7 +749,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env @@ -811,13 +811,13 @@ jobs: mkdir openverse - name: Checkout repository at `gh-pages` branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: gh-pages path: gh-pages - name: Checkout automations from repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: automation-checkout @@ -867,7 +867,7 @@ jobs: body: ${{ steps.preview_diff.outputs.body }} - name: Checkout repository # again, to enable cleaning - uses: actions/checkout@v3 + uses: actions/checkout@v4 if: success() || failure() ################# @@ -904,10 +904,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Log in to GitHub Docker Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: https://ghcr.io username: ${{ github.actor }} @@ -949,7 +949,7 @@ jobs: - publish-images steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get commit message id: commit @@ -989,7 +989,7 @@ jobs: - publish-images steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get commit message id: commit @@ -1058,7 +1058,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get commit message id: commit diff --git a/.github/workflows/generate_pot.yml b/.github/workflows/generate_pot.yml index 1427aaf9d35..0d7b7e3c2e8 100644 --- a/.github/workflows/generate_pot.yml +++ b/.github/workflows/generate_pot.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env @@ -33,7 +33,7 @@ jobs: run: cp frontend/openverse.pot /tmp/openverse.pot - name: Checkout repository at `translations` branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: translations @@ -48,4 +48,4 @@ jobs: - name: Checkout repository # again, to enable cleaning. if: always() - uses: actions/checkout@v3 + uses: actions/checkout@v4 diff --git a/.github/workflows/label_new_pr.yml b/.github/workflows/label_new_pr.yml index c354d44ee55..3819104763c 100644 --- a/.github/workflows/label_new_pr.yml +++ b/.github/workflows/label_new_pr.yml @@ -26,7 +26,7 @@ jobs: wait_time: 5 # check every 5 seconds max_time: 120 # timeout after 2 minutes - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Add default label in case of failure uses: actions-ecosystem/action-add-labels@v1 if: ${{ failure() }} diff --git a/.github/workflows/label_pr.yml b/.github/workflows/label_pr.yml index d42a656798d..eeef2d3d6a1 100644 --- a/.github/workflows/label_pr.yml +++ b/.github/workflows/label_pr.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env diff --git a/.github/workflows/label_sync.yml b/.github/workflows/label_sync.yml index 29b7bdf9f04..0b1d2b13447 100644 --- a/.github/workflows/label_sync.yml +++ b/.github/workflows/label_sync.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env diff --git a/.github/workflows/migration_safety_warning.yml b/.github/workflows/migration_safety_warning.yml index a579e68a60a..6eac2cc4a9c 100644 --- a/.github/workflows/migration_safety_warning.yml +++ b/.github/workflows/migration_safety_warning.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: peter-evans/find-comment@v2 id: existing-warning diff --git a/.github/workflows/pr_closed.yml b/.github/workflows/pr_closed.yml index bb4e7057d32..391620803cc 100644 --- a/.github/workflows/pr_closed.yml +++ b/.github/workflows/pr_closed.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: gh-pages diff --git a/.github/workflows/pr_label_check.yml b/.github/workflows/pr_label_check.yml index cd05d13c79e..2ebe3cf6a5e 100644 --- a/.github/workflows/pr_label_check.yml +++ b/.github/workflows/pr_label_check.yml @@ -25,7 +25,7 @@ jobs: stack: ${{ steps.get_groups.outputs.stack }} steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get label groups id: get_groups diff --git a/.github/workflows/pr_limit_reminders.yml b/.github/workflows/pr_limit_reminders.yml index be676286f2f..fbb4084481c 100644 --- a/.github/workflows/pr_limit_reminders.yml +++ b/.github/workflows/pr_limit_reminders.yml @@ -12,7 +12,7 @@ jobs: pr_count: ${{ steps.count-prs.outputs.pr_count }} slack_id: ${{ steps.count-prs.outputs.slack_id }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/github-script@v6 id: count-prs env: diff --git a/.github/workflows/project_thread_update_reminders.yml b/.github/workflows/project_thread_update_reminders.yml index b54181ce7d4..9d3c8319b60 100644 --- a/.github/workflows/project_thread_update_reminders.yml +++ b/.github/workflows/project_thread_update_reminders.yml @@ -9,7 +9,7 @@ jobs: send-reminders: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/github-script@v6 with: github-token: ${{ secrets.ACCESS_TOKEN }} diff --git a/.github/workflows/push_docker_image.yml b/.github/workflows/push_docker_image.yml index 3e1dffcdbaa..0ce266cb4c2 100644 --- a/.github/workflows/push_docker_image.yml +++ b/.github/workflows/push_docker_image.yml @@ -20,7 +20,7 @@ jobs: contents: read steps: - name: Log in to GitHub Docker Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: https://ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/release-app.yml b/.github/workflows/release-app.yml index 28e51086efb..ade2fea66bf 100644 --- a/.github/workflows/release-app.yml +++ b/.github/workflows/release-app.yml @@ -31,7 +31,7 @@ jobs: # Needed to open the changelog PR pull-requests: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: # Creating the tag requires having the whole history of `main` fetch-depth: 0 @@ -82,7 +82,7 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Log in to GitHub Docker Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: https://ghcr.io username: ${{ github.repository_owner }} diff --git a/.github/workflows/renovate.yml b/.github/workflows/renovate.yml index b4a8beb84c0..f06caa29094 100644 --- a/.github/workflows/renovate.yml +++ b/.github/workflows/renovate.yml @@ -10,7 +10,7 @@ jobs: renovate: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: renovatebot/github-action@v39.0.5 with: configurationFile: .github/renovate.json diff --git a/.github/workflows/sync_meta.yml b/.github/workflows/sync_meta.yml index 5122005a381..a085c5f1210 100644 --- a/.github/workflows/sync_meta.yml +++ b/.github/workflows/sync_meta.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Run file sync uses: BetaHuhn/repo-file-sync-action@v1 diff --git a/.github/workflows/weekly_updates.yml b/.github/workflows/weekly_updates.yml index f9c809117f1..b39018d7c39 100644 --- a/.github/workflows/weekly_updates.yml +++ b/.github/workflows/weekly_updates.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup CI env uses: ./.github/actions/setup-env diff --git a/api/Dockerfile b/api/Dockerfile index a073e03571a..8862ba57e15 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -7,7 +7,7 @@ ARG API_PY_VERSION # Audio waveform # ################## -FROM realies/audiowaveform:latest AS awf +FROM docker.io/realies/audiowaveform:latest AS awf # Identify dependencies of the `audiowaveform` binary and move them to `/deps`, # while retaining their folder structure @@ -18,7 +18,7 @@ RUN ldd /usr/local/bin/audiowaveform | tr -s '[:blank:]' '\n' | grep '^/' | \ # Python builder # ################## -FROM python:${API_PY_VERSION} AS builder +FROM docker.io/python:${API_PY_VERSION} AS builder # Container optimizations ENV PYTHONUNBUFFERED=1 @@ -47,7 +47,7 @@ RUN pipenv install --system --deploy --dev # API # ####### -FROM python:${API_PY_VERSION}-slim AS api +FROM docker.io/python:${API_PY_VERSION}-slim AS api LABEL org.opencontainers.image.source="https://github.com/WordPress/openverse" @@ -80,10 +80,9 @@ COPY --from=awf /usr/local/bin/audiowaveform /usr/local/bin # - Install system packages needed for running Python dependencies # - libexempi8: required for watermarking -# - libpq-dev: required by `psycopg2` # - Create directory for dumping API logs RUN apt-get update \ - && apt-get install -y curl libpq-dev libexempi8 postgresql-client \ + && apt-get install -y curl libexempi8 postgresql-client \ && rm -rf /var/lib/apt/lists/* \ && mkdir -p /var/log/openverse_api/openverse_api.log @@ -122,7 +121,7 @@ CMD ["gunicorn"] # NGINX # ######### -FROM nginx:1.25.2-alpine as nginx +FROM docker.io/nginx:1.25.2-alpine as nginx LABEL org.opencontainers.image.source = "https://github.com/WordPress/openverse" diff --git a/api/Pipfile b/api/Pipfile index 87542f91e3d..5ea3e7f67c3 100644 --- a/api/Pipfile +++ b/api/Pipfile @@ -5,7 +5,7 @@ verify_ssl = true [dev-packages] factory-boy = "~=3.2" -fakeredis = "==2.18.0" +fakeredis = "==2.19.0" ipython = "~=8.13" pycodestyle = "~=2.10" pytest = "~=7.4" @@ -20,7 +20,6 @@ pytest-sugar = "~=0.9" [packages] aiohttp = "~=3.8" aws-requests-auth = "~=0.4" -boto3 = "~=1.26" deepdiff = "~=6.4" Django = "~=4.2" django-cors-headers = "~=4.2" @@ -31,18 +30,18 @@ django-tqdm = "~=1.3" django-uuslug = "~=2.0" djangorestframework = "~=3.14" drf-spectacular = "*" -elasticsearch = "==8.8.2" +elasticsearch = "==8.10.0" elasticsearch-dsl = "~=8.9" future = "~=0.18" gunicorn = "~=21.2" limit = "~=0.2" Pillow = "~=10.0.1" -psycopg2 = "~=2.9" python-decouple = "~=3.8" python-xmp-toolkit = "~=2.0" sentry-sdk = "~=1.30" django-split-settings = "*" uvloop = "~=0.17" +psycopg = "~=3.1" [requires] python_version = "3.11" diff --git a/api/Pipfile.lock b/api/Pipfile.lock index 8d23dd7eaec..63d66be7a43 100644 --- a/api/Pipfile.lock +++ b/api/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5fd8856d8457aa956832989be7bdf949923fddde484535f62327116a72e84e12" + "sha256": "9d4fca2b657151e90f65e47d1cdfde72bca4d82f9d506d6b97be20f8e8566725" }, "pipfile-spec": 6, "requires": { @@ -150,23 +150,6 @@ "index": "pypi", "version": "==0.4.3" }, - "boto3": { - "hashes": [ - "sha256:8149f17587c68e556743018f213f00ece81a0f021adb9b9b697a2ee8802583d7", - "sha256:8860ab54a26d1d596d64fc9de7e40c4d7c53c100311208cbd90d9272c3385513" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==1.28.51" - }, - "botocore": { - "hashes": [ - "sha256:8e133add22f07b55d21e14176b97b82e993d5f9aca70f5b0cd0908cb105ba53a", - "sha256:91dfb38801d45214875a892bd1e908fc7a894c2ed170bacd67e6929af72f2bd2" - ], - "markers": "python_version >= '3.7'", - "version": "==1.31.51" - }, "certifi": { "hashes": [ "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082", @@ -177,153 +160,157 @@ }, "cffi": { "hashes": [ - "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", - "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", - "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", - "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", - "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", - "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", - "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", - "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", - "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", - "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", - "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", - "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", - "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", - "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", - "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", - "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", - "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", - "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", - "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", - "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", - "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", - "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", - "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", - "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", - "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", - "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", - "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", - "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", - "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", - "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", - "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", - "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", - "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", - "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", - "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", - "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", - "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", - "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", - "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", - "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", - "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", - "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", - "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", - "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", - "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", - "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", - "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", - "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", - "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", - "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", - "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", - "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", - "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", - "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", - "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", - "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", - "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", - "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", - "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", - "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", - "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", - "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", - "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", - "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" - ], - "version": "==1.15.1" + "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc", + "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a", + "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417", + "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab", + "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520", + "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36", + "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743", + "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8", + "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed", + "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684", + "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56", + "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324", + "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d", + "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235", + "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e", + "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088", + "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000", + "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7", + "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e", + "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673", + "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c", + "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe", + "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2", + "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098", + "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8", + "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a", + "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0", + "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b", + "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896", + "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e", + "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9", + "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2", + "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b", + "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6", + "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404", + "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f", + "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0", + "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4", + "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc", + "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936", + "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba", + "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872", + "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb", + "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614", + "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1", + "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d", + "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969", + "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b", + "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4", + "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627", + "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956", + "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357" + ], + "markers": "python_version >= '3.8'", + "version": "==1.16.0" }, "charset-normalizer": { "hashes": [ - "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96", - "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c", - "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710", - "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706", - "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020", - "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252", - "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad", - "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329", - "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a", - "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f", - "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6", - "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4", - "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a", - "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46", - "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2", - "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23", - "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace", - "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd", - "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982", - "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10", - "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2", - "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea", - "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09", - "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5", - "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149", - "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489", - "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9", - "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80", - "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592", - "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3", - "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6", - "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed", - "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c", - "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200", - "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a", - "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e", - "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d", - "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6", - "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623", - "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669", - "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3", - "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa", - "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9", - "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2", - "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f", - "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1", - "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4", - "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a", - "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8", - "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3", - "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029", - "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f", - "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959", - "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22", - "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7", - "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952", - "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346", - "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e", - "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d", - "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299", - "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd", - "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a", - "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3", - "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037", - "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94", - "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c", - "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858", - "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a", - "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449", - "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c", - "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918", - "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1", - "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c", - "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac", - "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa" + "sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843", + "sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786", + "sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e", + "sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8", + "sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4", + "sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa", + "sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d", + "sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82", + "sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7", + "sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895", + "sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d", + "sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a", + "sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382", + "sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678", + "sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b", + "sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e", + "sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741", + "sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4", + "sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596", + "sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9", + "sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69", + "sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c", + "sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77", + "sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13", + "sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459", + "sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e", + "sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7", + "sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908", + "sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a", + "sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f", + "sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8", + "sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482", + "sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d", + "sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d", + "sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545", + "sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34", + "sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86", + "sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6", + "sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe", + "sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e", + "sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc", + "sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7", + "sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd", + "sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c", + "sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557", + "sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a", + "sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89", + "sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078", + "sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e", + "sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4", + "sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403", + "sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0", + "sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89", + "sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115", + "sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9", + "sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05", + "sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a", + "sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec", + "sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56", + "sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38", + "sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479", + "sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c", + "sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e", + "sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd", + "sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186", + "sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455", + "sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c", + "sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65", + "sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78", + "sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287", + "sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df", + "sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43", + "sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1", + "sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7", + "sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989", + "sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a", + "sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63", + "sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884", + "sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649", + "sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810", + "sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828", + "sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4", + "sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2", + "sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd", + "sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5", + "sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe", + "sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293", + "sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e", + "sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e", + "sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8" ], "markers": "python_full_version >= '3.7.0'", - "version": "==3.2.0" + "version": "==3.3.0" }, "cryptography": { "hashes": [ @@ -450,29 +437,29 @@ }, "drf-spectacular": { "hashes": [ - "sha256:8f5a8f87353d1bb8dcb3f3909b7109b2dcbe1d91f3e069409cf322963e140bd6", - "sha256:afeccc6533dcdb4e78afbfcc49f3c5e9c369aeb62f965e4d1a43b165449c147a" + "sha256:aee55330a774ba8a9cbdb125714d1c9ee05a8aafd3ce3be8bfd26527649aeb44", + "sha256:c0002a820b11771fdbf37853deb371947caf0159d1afeeffe7598e964bc1db94" ], "index": "pypi", "markers": "python_version >= '3.6'", - "version": "==0.26.4" + "version": "==0.26.5" }, "elastic-transport": { "hashes": [ - "sha256:19db271ab79c9f70f8c43f8f5b5111408781a6176b54ab2e54d713b6d9ceb815", - "sha256:b9ad708ceb7fcdbc6b30a96f886609a109f042c0b9d9f2e44403b3133ba7ff10" + "sha256:c718ce40e8217b6045604961463c10da69a152dda07af4e25b3feae8d7965fc0", + "sha256:e5548997113c5d9566c9a1a51ed67bce50a4871bc0e44b692166461279e4167e" ], "markers": "python_version >= '3.6'", - "version": "==8.4.0" + "version": "==8.4.1" }, "elasticsearch": { "hashes": [ - "sha256:bed8cf8fcc6c3be7c254b579de4c29afab021f373c832246f912d37aef3c6bd5", - "sha256:bffd6ce4faaacf90e6f617241773b3da8fb94e2e83554f5508e2fab92ca79643" + "sha256:1a7f1b71dda0bc24c97d3e61c3b0d8165d5ab024f9f1538f35ed5894c6831ba2", + "sha256:4a721a9ff9d669ed7140a043d5b84fb50eb7bda6aaf2eafb7bee49177dc316e8" ], "index": "pypi", "markers": "python_version >= '3.6' and python_version < '4'", - "version": "==8.8.2" + "version": "==8.10.0" }, "elasticsearch-dsl": { "hashes": [ @@ -583,21 +570,13 @@ "markers": "python_version >= '3.5'", "version": "==0.5.1" }, - "jmespath": { - "hashes": [ - "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", - "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" - ], - "markers": "python_version >= '3.7'", - "version": "==1.0.1" - }, "jsonschema": { "hashes": [ - "sha256:043dc26a3845ff09d20e4420d6012a9c91c9aa8999fa184e7efcfeccb41e32cb", - "sha256:6e1e7569ac13be8139b2dd2c21a55d350066ee3f80df06c608b398cdc6f30e8f" + "sha256:cd5f1f9ed9444e554b38ba003af06c0a8c2868131e56bfbef0550fb450c0330e", + "sha256:ec84cc37cfa703ef7cd4928db24f9cb31428a5d0fa77747b8b51a847458e0bbf" ], "markers": "python_version >= '3.8'", - "version": "==4.19.0" + "version": "==4.19.1" }, "jsonschema-specifications": { "hashes": [ @@ -719,11 +698,11 @@ }, "packaging": { "hashes": [ - "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", - "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f" + "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", + "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" ], "markers": "python_version >= '3.7'", - "version": "==23.1" + "version": "==23.2" }, "pillow": { "hashes": [ @@ -785,23 +764,14 @@ "markers": "python_version >= '3.8'", "version": "==10.0.1" }, - "psycopg2": { + "psycopg": { "hashes": [ - "sha256:1a6a2d609bce44f78af4556bea0c62a5e7f05c23e5ea9c599e07678995609084", - "sha256:44d93a0109dfdf22fe399b419bcd7fa589d86895d3931b01fb321d74dadc68f1", - "sha256:8275abf628c6dc7ec834ea63f6f3846bf33518907a2b9b693d41fd063767a866", - "sha256:91e81a8333a0037babfc9fe6d11e997a9d4dac0f38c43074886b0d9dead94fe9", - "sha256:b22ed9c66da2589a664e0f1ca2465c29b75aaab36fa209d4fb916025fb9119e5", - "sha256:b6bd7d9d3a7a63faae6edf365f0ed0e9b0a1aaf1da3ca146e6b043fb3eb5d723", - "sha256:c7949770cafbd2f12cecc97dea410c514368908a103acf519f2a346134caa4d5", - "sha256:d1210fcf99aae6f728812d1d2240afc1dc44b9e6cba526a06fb8134f969957c2", - "sha256:d5c5297e2fbc8068d4255f1e606bfc9291f06f91ec31b2a0d4c536210ac5c0a2", - "sha256:e9b04cbef584310a1ac0f0d55bb623ca3244c87c51187645432e342de9ae81a8", - "sha256:f00cc35bd7119f1fed17b85bd1007855194dde2cbd8de01ab8ebb17487440ad8" + "sha256:8ec5230d6a7eb654b4fb3cf2d3eda8871d68f24807b934790504467f1deee9f8", + "sha256:cec7ad2bc6a8510e56c45746c631cf9394148bdc8a9a11fd8cf8554ce129ae78" ], "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==2.9.7" + "markers": "python_version >= '3.7'", + "version": "==3.1.12" }, "pycparser": { "hashes": [ @@ -906,11 +876,11 @@ }, "redis": { "hashes": [ - "sha256:06570d0b2d84d46c21defc550afbaada381af82f5b83e5b3777600e05d8e2ed0", - "sha256:5cea6c0d335c9a7332a460ed8729ceabb4d0c489c7285b0a86dbbf8a017bd120" + "sha256:0dab495cd5753069d3bc650a0dde8a8f9edde16fc5691b689a566eda58100d0f", + "sha256:ed4802971884ae19d640775ba3b03aa2e7bd5e8fb8dfaed2decce4d0fc48391f" ], "markers": "python_version >= '3.7'", - "version": "==5.0.0" + "version": "==5.0.1" }, "referencing": { "hashes": [ @@ -1031,14 +1001,6 @@ "markers": "python_version >= '3.8'", "version": "==0.10.3" }, - "s3transfer": { - "hashes": [ - "sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084", - "sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861" - ], - "markers": "python_version >= '3.7'", - "version": "==0.6.2" - }, "sentry-sdk": { "hashes": [ "sha256:64a7141005fb775b9db298a30de93e3b83e0ddd1232dc6f36eb38aebc1553291", @@ -1078,6 +1040,14 @@ "markers": "python_version >= '3.7'", "version": "==4.66.1" }, + "typing-extensions": { + "hashes": [ + "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0", + "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef" + ], + "markers": "python_version >= '3.8'", + "version": "==4.8.0" + }, "uritemplate": { "hashes": [ "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0", @@ -1088,11 +1058,12 @@ }, "urllib3": { "hashes": [ - "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f", - "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14" + "sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21", + "sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b" ], - "markers": "python_version >= '3.6'", - "version": "==1.26.16" + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.26.17" }, "uvloop": { "hashes": [ @@ -1354,10 +1325,10 @@ }, "executing": { "hashes": [ - "sha256:0314a69e37426e3608aada02473b4161d4caf5a4b244d1d0c48072b8fee7bacc", - "sha256:19da64c18d2d851112f09c287f8d3dbbdf725ab0e569077efb6cdcbd3497c107" + "sha256:06df6183df67389625f4e763921c6cf978944721abf3e714000200aab95b0657", + "sha256:0ff053696fdeef426cda5bd18eacd94f82c91f49823a2e9090124212ceea9b08" ], - "version": "==1.2.0" + "version": "==2.0.0" }, "factory-boy": { "hashes": [ @@ -1370,20 +1341,20 @@ }, "faker": { "hashes": [ - "sha256:5d6b7880b3bea708075ddf91938424453f07053a59f8fa0453c1870df6ff3292", - "sha256:64c8513c53c3a809075ee527b323a0ba61517814123f3137e4912f5d43350139" + "sha256:8fba91068dc26e3159c1ac9f22444a2338704b0991d86605322e454bda420092", + "sha256:d5d5953556b0fb428a46019e03fc2d40eab2980135ddef5a9eb3d054947fdf83" ], "markers": "python_version >= '3.8'", - "version": "==19.6.1" + "version": "==19.6.2" }, "fakeredis": { "hashes": [ - "sha256:bc7a82e9caec80096659fd9a810b72d21f833574be26ff7b97955db626d60bac", - "sha256:f9c18d3dba81a470953cc042868b411e334109e065cde53a7a82beef6702a1de" + "sha256:83dbf922d07244d114103843d8d4e760fa25119a5fc88cfc21b4548ea2d6353c", + "sha256:e136cd13bddd7e7f71270cc7c4693d14f975d929135eb0f8625ef55f204eee25" ], "index": "pypi", "markers": "python_version >= '3.7' and python_version < '4.0'", - "version": "==2.18.0" + "version": "==2.19.0" }, "freezegun": { "hashes": [ @@ -1411,28 +1382,28 @@ }, "ipython": { "hashes": [ - "sha256:2baeb5be6949eeebf532150f81746f8333e2ccce02de1c7eedde3f23ed5e9f1e", - "sha256:45a2c3a529296870a97b7de34eda4a31bee16bc7bf954e07d39abe49caf8f887" + "sha256:7a1b2e1e6a3ec5baa466163c451335081f31859883889ff0289c6b21f7a095e2", + "sha256:dba644376826a532e362da945a672865be7efda76ecf999219e6021bda85d702" ], "index": "pypi", "markers": "python_version >= '3.9'", - "version": "==8.15.0" + "version": "==8.16.0" }, "jedi": { "hashes": [ - "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4", - "sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e" + "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd", + "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0" ], "markers": "python_version >= '3.6'", - "version": "==0.19.0" + "version": "==0.19.1" }, "jsonschema": { "hashes": [ - "sha256:043dc26a3845ff09d20e4420d6012a9c91c9aa8999fa184e7efcfeccb41e32cb", - "sha256:6e1e7569ac13be8139b2dd2c21a55d350066ee3f80df06c608b398cdc6f30e8f" + "sha256:cd5f1f9ed9444e554b38ba003af06c0a8c2868131e56bfbef0550fb450c0330e", + "sha256:ec84cc37cfa703ef7cd4928db24f9cb31428a5d0fa77747b8b51a847458e0bbf" ], "markers": "python_version >= '3.8'", - "version": "==4.19.0" + "version": "==4.19.1" }, "jsonschema-specifications": { "hashes": [ @@ -1459,11 +1430,11 @@ }, "packaging": { "hashes": [ - "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", - "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f" + "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", + "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" ], "markers": "python_version >= '3.7'", - "version": "==23.1" + "version": "==23.2" }, "parso": { "hashes": [ @@ -1557,11 +1528,12 @@ }, "psycopg": { "hashes": [ - "sha256:15b25741494344c24066dc2479b0f383dd1b82fa5e75612fa4fa5bb30726e9b6", - "sha256:8bbeddae5075c7890b2fa3e3553440376d3c5e28418335dee3c3656b06fa2b52" + "sha256:8ec5230d6a7eb654b4fb3cf2d3eda8871d68f24807b934790504467f1deee9f8", + "sha256:cec7ad2bc6a8510e56c45746c631cf9394148bdc8a9a11fd8cf8554ce129ae78" ], + "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==3.1.10" + "version": "==3.1.12" }, "ptyprocess": { "hashes": [ @@ -1646,11 +1618,11 @@ }, "redis": { "hashes": [ - "sha256:06570d0b2d84d46c21defc550afbaada381af82f5b83e5b3777600e05d8e2ed0", - "sha256:5cea6c0d335c9a7332a460ed8729ceabb4d0c489c7285b0a86dbbf8a017bd120" + "sha256:0dab495cd5753069d3bc650a0dde8a8f9edde16fc5691b689a566eda58100d0f", + "sha256:ed4802971884ae19d640775ba3b03aa2e7bd5e8fb8dfaed2decce4d0fc48391f" ], "markers": "python_version >= '3.7'", - "version": "==5.0.0" + "version": "==5.0.1" }, "referencing": { "hashes": [ @@ -1875,10 +1847,10 @@ }, "stack-data": { "hashes": [ - "sha256:32d2dd0376772d01b6cb9fc996f3c8b57a357089dec328ed4b6553d037eaf815", - "sha256:cbb2a53eb64e5785878201a97ed7c7b94883f48b87bfb0bbe8b623c74679e4a8" + "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", + "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695" ], - "version": "==0.6.2" + "version": "==0.6.3" }, "tabulate": { "extras": [ @@ -1901,11 +1873,11 @@ }, "traitlets": { "hashes": [ - "sha256:417745a96681fbb358e723d5346a547521f36e9bd0d50ba7ab368fff5d67aa54", - "sha256:f584ea209240466e66e91f3c81aa7d004ba4cf794990b0c775938a1544217cd1" + "sha256:07ab9c5bf8a0499fd7b088ba51be899c90ffc936ffc797d7b6907fc516bcd116", + "sha256:db9c4aa58139c3ba850101913915c042bdba86f7c8a0dda1c6f7f92c5da8e542" ], "markers": "python_version >= '3.8'", - "version": "==5.10.0" + "version": "==5.10.1" }, "typing-extensions": { "hashes": [ @@ -1917,10 +1889,10 @@ }, "wcwidth": { "hashes": [ - "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e", - "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0" + "sha256:77f719e01648ed600dfa5402c347481c0992263b81a027344f3e1ba25493a704", + "sha256:8705c569999ffbb4f6a87c6d1b80f324bd6db952f5eb0b95bc07517f4c1813d4" ], - "version": "==0.2.6" + "version": "==0.2.8" }, "xmltodict": { "hashes": [ diff --git a/api/api/controllers/search_controller.py b/api/api/controllers/search_controller.py index e90061c9b7e..f0cf402e105 100644 --- a/api/api/controllers/search_controller.py +++ b/api/api/controllers/search_controller.py @@ -25,7 +25,7 @@ ELASTICSEARCH_MAX_RESULT_WINDOW = 10000 -SOURCE_CACHE_TIMEOUT = 60 * 20 +SOURCE_CACHE_TIMEOUT = 60 * 60 * 4 # 4 hours FILTER_CACHE_TIMEOUT = 30 DEAD_LINK_RATIO = 1 / 2 THUMBNAIL = "thumbnail" @@ -556,19 +556,17 @@ def get_sources(index): # Don't increase `size` without reading this issue first: # https://github.com/elastic/elasticsearch/issues/18838 size = 100 - agg_body = { - "aggs": { - "unique_sources": { - "terms": { - "field": "source.keyword", - "size": size, - "order": {"_key": "desc"}, - } + aggs = { + "unique_sources": { + "terms": { + "field": "source.keyword", + "size": size, + "order": {"_key": "desc"}, } } } try: - results = settings.ES.search(index=index, body=agg_body, request_cache=True) + results = settings.ES.search(index=index, aggs=aggs, request_cache=True) buckets = results["aggregations"]["unique_sources"]["buckets"] except NotFoundError: buckets = [{"key": "none_found", "doc_count": 0}] diff --git a/api/api/serializers/image_serializers.py b/api/api/serializers/image_serializers.py index 4f0af390d26..5824198936c 100644 --- a/api/api/serializers/image_serializers.py +++ b/api/api/serializers/image_serializers.py @@ -1,4 +1,5 @@ from typing import Literal +from uuid import UUID from rest_framework import serializers @@ -124,7 +125,15 @@ class OembedRequestSerializer(serializers.Serializer): @staticmethod def validate_url(value): - return add_protocol(value) + url = add_protocol(value) + if url.endswith("/"): + url = url[:-1] + identifier = url.rsplit("/", 1)[1] + try: + uuid = UUID(identifier) + except ValueError: + raise serializers.ValidationError("Could not parse identifier from URL.") + return uuid class OembedSerializer(BaseModelSerializer): diff --git a/api/api/views/image_views.py b/api/api/views/image_views.py index 95844ffbf97..6f138908db8 100644 --- a/api/api/views/image_views.py +++ b/api/api/views/image_views.py @@ -83,10 +83,7 @@ def oembed(self, request, *_, **__): context = self.get_serializer_context() - url = params.validated_data["url"] - if url.endswith("/"): - url = url[:-1] - identifier = url.rsplit("/", 1)[1] + identifier = params.validated_data["url"] image = get_object_or_404(Image, identifier=identifier) if not (image.height and image.width): image_file = requests.get(image.url, headers=self.OEMBED_HEADERS) diff --git a/api/test/test_image_integration.py b/api/test/test_image_integration.py index 2ed9d9b0bc3..e8b7d479913 100644 --- a/api/test/test_image_integration.py +++ b/api/test/test_image_integration.py @@ -92,6 +92,16 @@ def test_oembed_endpoint_with_non_existent_image(): assert response.status_code == 404 +def test_oembed_endpoint_with_bad_identifier(): + params = { + "url": "https://any.domain/any/path/not-a-valid-uuid", + } + response = requests.get( + f"{API_URL}/v1/images/oembed?{urlencode(params)}", verify=False + ) + assert response.status_code == 400 + + @pytest.mark.parametrize( "url", [ diff --git a/api/test/unit/management/commands/test_generatewaveforms.py b/api/test/unit/management/commands/test_generatewaveforms.py index 20d9a9bcd1a..bc12a9669d0 100644 --- a/api/test/unit/management/commands/test_generatewaveforms.py +++ b/api/test/unit/management/commands/test_generatewaveforms.py @@ -8,8 +8,8 @@ from django.db import connections from django.test.utils import CaptureQueriesContext -import psycopg2 import pytest +from psycopg.errors import NotNullViolation from api.models.audio import Audio, AudioAddOn @@ -111,7 +111,7 @@ def test_paginates_audio_waveforms_to_generate( {"stderr": b"This is an error string"}, ), ( - psycopg2.errors.NotNullViolation, + NotNullViolation, tuple(), dict(), ), diff --git a/automations/python/Pipfile.lock b/automations/python/Pipfile.lock index 3dcd9db58d1..b4169c4d4c7 100644 --- a/automations/python/Pipfile.lock +++ b/automations/python/Pipfile.lock @@ -22,6 +22,7 @@ "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a" ], "index": "pypi", + "markers": "python_full_version >= '3.6.0'", "version": "==4.12.2" }, "certifi": { @@ -34,153 +35,157 @@ }, "cffi": { "hashes": [ - "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", - "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", - "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", - "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", - "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", - "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", - "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", - "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", - "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", - "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", - "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", - "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", - "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", - "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", - "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", - "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", - "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", - "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", - "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", - "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", - "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", - "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", - "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", - "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", - "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", - "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", - "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", - "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", - "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", - "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", - "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", - "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", - "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", - "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", - "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", - "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", - "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", - "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", - "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", - "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", - "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", - "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", - "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", - "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", - "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", - "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", - "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", - "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", - "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", - "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", - "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", - "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", - "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", - "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", - "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", - "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", - "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", - "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", - "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", - "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", - "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", - "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", - "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", - "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" - ], - "version": "==1.15.1" + "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc", + "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a", + "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417", + "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab", + "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520", + "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36", + "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743", + "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8", + "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed", + "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684", + "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56", + "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324", + "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d", + "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235", + "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e", + "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088", + "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000", + "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7", + "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e", + "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673", + "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c", + "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe", + "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2", + "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098", + "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8", + "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a", + "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0", + "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b", + "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896", + "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e", + "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9", + "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2", + "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b", + "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6", + "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404", + "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f", + "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0", + "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4", + "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc", + "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936", + "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba", + "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872", + "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb", + "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614", + "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1", + "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d", + "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969", + "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b", + "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4", + "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627", + "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956", + "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357" + ], + "markers": "python_version >= '3.8'", + "version": "==1.16.0" }, "charset-normalizer": { "hashes": [ - "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96", - "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c", - "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710", - "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706", - "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020", - "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252", - "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad", - "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329", - "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a", - "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f", - "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6", - "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4", - "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a", - "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46", - "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2", - "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23", - "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace", - "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd", - "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982", - "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10", - "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2", - "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea", - "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09", - "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5", - "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149", - "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489", - "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9", - "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80", - "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592", - "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3", - "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6", - "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed", - "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c", - "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200", - "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a", - "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e", - "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d", - "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6", - "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623", - "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669", - "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3", - "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa", - "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9", - "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2", - "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f", - "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1", - "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4", - "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a", - "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8", - "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3", - "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029", - "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f", - "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959", - "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22", - "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7", - "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952", - "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346", - "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e", - "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d", - "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299", - "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd", - "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a", - "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3", - "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037", - "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94", - "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c", - "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858", - "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a", - "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449", - "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c", - "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918", - "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1", - "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c", - "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac", - "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa" + "sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843", + "sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786", + "sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e", + "sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8", + "sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4", + "sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa", + "sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d", + "sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82", + "sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7", + "sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895", + "sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d", + "sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a", + "sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382", + "sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678", + "sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b", + "sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e", + "sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741", + "sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4", + "sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596", + "sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9", + "sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69", + "sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c", + "sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77", + "sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13", + "sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459", + "sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e", + "sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7", + "sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908", + "sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a", + "sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f", + "sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8", + "sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482", + "sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d", + "sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d", + "sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545", + "sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34", + "sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86", + "sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6", + "sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe", + "sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e", + "sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc", + "sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7", + "sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd", + "sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c", + "sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557", + "sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a", + "sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89", + "sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078", + "sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e", + "sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4", + "sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403", + "sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0", + "sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89", + "sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115", + "sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9", + "sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05", + "sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a", + "sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec", + "sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56", + "sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38", + "sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479", + "sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c", + "sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e", + "sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd", + "sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186", + "sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455", + "sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c", + "sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65", + "sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78", + "sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287", + "sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df", + "sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43", + "sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1", + "sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7", + "sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989", + "sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a", + "sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63", + "sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884", + "sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649", + "sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810", + "sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828", + "sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4", + "sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2", + "sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd", + "sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5", + "sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe", + "sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293", + "sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e", + "sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e", + "sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8" ], - "markers": "python_version >= '3.7'", - "version": "==3.2.0" + "markers": "python_full_version >= '3.7.0'", + "version": "==3.3.0" }, "cryptography": { "hashes": [ @@ -208,7 +213,6 @@ "sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb", "sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f" ], - "index": "pypi", "version": "==41.0.4" }, "deprecated": { @@ -252,11 +256,12 @@ }, "pygithub": { "hashes": [ - "sha256:3d87a822e6c868142f0c2c4bf16cce4696b5a7a4d142a7bd160e1bdf75bc54a9", - "sha256:c44e3a121c15bf9d3a5cc98d94c9a047a5132a9b01d22264627f58ade9ddc217" + "sha256:4b528d5d6f35e991ea5fd3f942f58748f24938805cb7fcf24486546637917337", + "sha256:ecf12c2809c44147bce63b047b3d2e9dac8a41b63e90fcb263c703f64936b97c" ], "index": "pypi", - "version": "==1.59.1" + "markers": "python_version >= '3.7'", + "version": "==2.1.1" }, "pyjwt": { "extras": [ @@ -291,8 +296,17 @@ "sha256:81c2e5865b8ac55e825b0358e496e1d9387c811e85bb40e71a3b29b288963612" ], "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==2.9.0" }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" + }, "pyyaml": { "hashes": [ "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5", @@ -347,6 +361,7 @@ "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f" ], "index": "pypi", + "markers": "python_version >= '3.6'", "version": "==6.0.1" }, "requests": { @@ -355,6 +370,7 @@ "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1" ], "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==2.31.0" }, "six": { @@ -373,13 +389,22 @@ "markers": "python_version >= '3.8'", "version": "==2.5" }, + "typing-extensions": { + "hashes": [ + "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0", + "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef" + ], + "markers": "python_version >= '3.8'", + "version": "==4.8.0" + }, "urllib3": { "hashes": [ - "sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594", - "sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e" + "sha256:7a7c7003b000adf9e7ca2a377c9688bbc54ed41b985789ed576570342a375cd2", + "sha256:b19e1a85d206b56d7df1d5e683df4a7725252a964e3993648dd0fb5a1c157564" ], + "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==2.0.5" + "version": "==2.0.6" }, "webencodings": { "hashes": [ @@ -495,26 +520,27 @@ }, "executing": { "hashes": [ - "sha256:0314a69e37426e3608aada02473b4161d4caf5a4b244d1d0c48072b8fee7bacc", - "sha256:19da64c18d2d851112f09c287f8d3dbbdf725ab0e569077efb6cdcbd3497c107" + "sha256:06df6183df67389625f4e763921c6cf978944721abf3e714000200aab95b0657", + "sha256:0ff053696fdeef426cda5bd18eacd94f82c91f49823a2e9090124212ceea9b08" ], - "version": "==1.2.0" + "version": "==2.0.0" }, "ipython": { "hashes": [ - "sha256:1d197b907b6ba441b692c48cf2a3a2de280dc0ac91a3405b39349a50272ca0a1", - "sha256:248aca623f5c99a6635bc3857677b7320b9b8039f99f070ee0d20a5ca5a8e6bf" + "sha256:7a1b2e1e6a3ec5baa466163c451335081f31859883889ff0289c6b21f7a095e2", + "sha256:dba644376826a532e362da945a672865be7efda76ecf999219e6021bda85d702" ], "index": "pypi", - "version": "==8.14.0" + "markers": "python_version >= '3.9'", + "version": "==8.16.0" }, "jedi": { "hashes": [ - "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4", - "sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e" + "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd", + "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0" ], "markers": "python_version >= '3.6'", - "version": "==0.19.0" + "version": "==0.19.1" }, "matplotlib-inline": { "hashes": [ @@ -574,7 +600,7 @@ "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692", "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29" ], - "markers": "python_full_version >= '3.7.0'", + "markers": "python_version >= '3.7'", "version": "==2.16.1" }, "six": { @@ -587,25 +613,25 @@ }, "stack-data": { "hashes": [ - "sha256:32d2dd0376772d01b6cb9fc996f3c8b57a357089dec328ed4b6553d037eaf815", - "sha256:cbb2a53eb64e5785878201a97ed7c7b94883f48b87bfb0bbe8b623c74679e4a8" + "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", + "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695" ], - "version": "==0.6.2" + "version": "==0.6.3" }, "traitlets": { "hashes": [ - "sha256:417745a96681fbb358e723d5346a547521f36e9bd0d50ba7ab368fff5d67aa54", - "sha256:f584ea209240466e66e91f3c81aa7d004ba4cf794990b0c775938a1544217cd1" + "sha256:07ab9c5bf8a0499fd7b088ba51be899c90ffc936ffc797d7b6907fc516bcd116", + "sha256:db9c4aa58139c3ba850101913915c042bdba86f7c8a0dda1c6f7f92c5da8e542" ], "markers": "python_version >= '3.8'", - "version": "==5.10.0" + "version": "==5.10.1" }, "wcwidth": { "hashes": [ - "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e", - "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0" + "sha256:77f719e01648ed600dfa5402c347481c0992263b81a027344f3e1ba25493a704", + "sha256:8705c569999ffbb4f6a87c6d1b80f324bd6db952f5eb0b95bc07517f4c1813d4" ], - "version": "==0.2.6" + "version": "==0.2.8" } } } diff --git a/catalog/justfile b/catalog/justfile index 8d9f0c333d6..fd34546cd23 100644 --- a/catalog/justfile +++ b/catalog/justfile @@ -126,15 +126,15 @@ generate-dag-docs fail_on_diff="false": just ../run \ {{ SERVICE }} \ "bash -c 'python catalog/utilities/dag_doc_gen/dag_doc_generation.py && chmod 666 /opt/airflow/catalog/utilities/dag_doc_gen/DAGs.md'" - # Move the file to the top level, since that level is not mounted into the container - mv utilities/dag_doc_gen/DAGs.md DAGs.md + # Move the file to the top level on the documentation folder, since that level is not mounted into the container + mv utilities/dag_doc_gen/DAGs.md ../documentation/catalog/reference/DAGs.md echo -n "Running linting..." # Linting step afterwards is necessary since the generated output differs greatly from what prettier expects - just ../lint prettier catalog/DAGs.md &>/dev/null || true + just ../lint prettier documentation/catalog/reference/DAGs.md &>/dev/null || true echo "Done!" if {{ fail_on_diff }}; then set +e - git diff --exit-code DAGs.md + git diff --exit-code -- documentation/catalog/reference/DAGs.md if [ $? -ne 0 ]; then printf "\n\n\e[31m!! Changes found in DAG documentation, please run 'just generate-dag-docs' locally and commit difference !!\n\n" exit 1 diff --git a/catalog/requirements_prod.txt b/catalog/requirements_prod.txt index 4ec4eb6651c..e41653e007b 100644 --- a/catalog/requirements_prod.txt +++ b/catalog/requirements_prod.txt @@ -8,4 +8,4 @@ psycopg2-binary requests-file==1.5.1 requests-oauthlib retry==0.9.2 -tldextract==3.4.4 +tldextract==3.6.0 diff --git a/catalog/tests/utilities/dag_doc_gen/test_dag_doc_generation.py b/catalog/tests/utilities/dag_doc_gen/test_dag_doc_generation.py index b7442a90f02..f65fac2fec3 100644 --- a/catalog/tests/utilities/dag_doc_gen/test_dag_doc_generation.py +++ b/catalog/tests/utilities/dag_doc_gen/test_dag_doc_generation.py @@ -33,7 +33,7 @@ class DagMock(NamedTuple): [ (None, None), ("Sample simple doc", "Sample simple doc"), - ("# Big header", "### Big header"), + ("# Big header", "#### Big header"), ], ) @pytest.mark.parametrize( @@ -107,7 +107,7 @@ def test_get_dags_info( ), False, """ -## Special Name +### Special Name | DAG ID | Schedule Interval | | --- | --- | @@ -126,7 +126,7 @@ def test_get_dags_info( ), False, """ -## Special Name +### Special Name | DAG ID | Schedule Interval | | --- | --- | @@ -145,7 +145,7 @@ def test_get_dags_info( ), True, """ -## Special Name +### Special Name | DAG ID | Schedule Interval | Dated | Media Type(s) | | --- | --- | --- | --- | @@ -167,7 +167,7 @@ def test_generate_dag_doc(): + """\ 1. [T1](#t1) -## T1 +### T1 | DAG ID | Schedule Interval | | --- | --- | @@ -180,7 +180,7 @@ def test_generate_dag_doc(): 1. [`b`](#b) -## `b` +### `b` this one has a doc """ diff --git a/catalog/utilities/dag_doc_gen/dag_doc_generation.py b/catalog/utilities/dag_doc_gen/dag_doc_generation.py index b3c71734f4e..436bb709607 100644 --- a/catalog/utilities/dag_doc_gen/dag_doc_generation.py +++ b/catalog/utilities/dag_doc_gen/dag_doc_generation.py @@ -45,13 +45,13 @@ - [DAGs by Type](#dags-by-type) - [Individual DAG documentation](#dag-documentation) -# DAGs by Type +## DAGs by Type The following are DAGs grouped by their primary tag: """ MIDAMBLE = """ -# DAG documentation +## DAG documentation The following is documentation associated with each DAG (where available): @@ -94,7 +94,7 @@ def get_provider_workflows() -> dict[str, ProviderWorkflow]: def fix_headings(doc: str) -> str: """ - Increase all heading levels by 2. + Increase all heading levels by 3. This is necessary to accommodate the embedded setting of the DAG docs in the final Markdown output. @@ -105,7 +105,7 @@ def fix_headings(doc: str) -> str: for match in reversed(list(HEADING_PROG.finditer(doc))): start, end = match.span() original_heading = match.string[start:end] - new_heading = f"##{original_heading}" + new_heading = f"###{original_heading}" doc = f"{doc[:start]}{new_heading}{doc[end:]}" return doc @@ -149,7 +149,7 @@ def generate_type_subsection( ) -> str: """Generate the documentation for a "DAGs by type" subsection.""" log.info(f"Building subsection for '{name}'") - text = f"## {name}\n\n" + text = f"### {name}\n\n" # Columns for all DAGs header = "| DAG ID | Schedule Interval |" # Conditionally add the other columns for the provider-specific DAGs @@ -183,7 +183,7 @@ def generate_type_subsection( def generate_single_documentation(dag: DagInfo) -> str: """Generate the documentation for a single DAG.""" return f""" -## `{dag.dag_id}` +### `{dag.dag_id}` {dag.doc} @@ -217,7 +217,7 @@ def generate_dag_doc(dag_folder: Path = DAG_FOLDER) -> str: # For each type we generate a sub-list of DAGs. We add a link to each generated # sub-list as part of a table of contents, but defer adding the sub-lists until # all are generated. - text += f" 1. [{name}](#{type_})\n" + text += f" 1. [{name}](#{type_.replace('_', '-')})\n" dag_types.append(generate_type_subsection(name, dags, is_provider)) text += "\n" + "\n\n".join(dag_types) diff --git a/docker-compose.yml b/docker-compose.yml index 1c226b6cf40..7ff70da8bf7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,7 +30,7 @@ services: profiles: - ingestion_server - api - image: postgres:13.10-alpine + image: docker.io/postgres:13.10-alpine ports: - "50254:5432" volumes: @@ -65,7 +65,7 @@ services: profiles: - catalog_dependencies - catalog - image: minio/minio:latest + image: docker.io/minio/minio:latest ports: - "5010:5000" - "5011:5001" @@ -88,7 +88,7 @@ services: profiles: - catalog_dependencies - catalog - image: minio/mc:latest + image: docker.io/minio/mc:latest env_file: - .env - docker/minio/.env @@ -112,7 +112,7 @@ services: - "8793" # Used for fetching logs environment: # Upgrade the DB on startup - _AIRFLOW_DB_UPGRADE: "true" + _AIRFLOW_DB_MIGRATE: "true" _AIRFLOW_WWW_USER_CREATE: "true" _AIRFLOW_WWW_USER_USERNAME: airflow _AIRFLOW_WWW_USER_PASSWORD: airflow @@ -145,7 +145,7 @@ services: plausible_db: profiles: - frontend - image: postgres:13.10-alpine + image: docker.io/postgres:13.10-alpine expose: - "5432" volumes: @@ -158,7 +158,7 @@ services: plausible_ch: profiles: - frontend - image: clickhouse/clickhouse-server:23.4-alpine + image: docker.io/clickhouse/clickhouse-server:23.4-alpine volumes: - plausible-clickhouse:/var/lib/clickhouse - ./docker/clickhouse/clickhouse-config.xml:/etc/clickhouse-server/config.d/logging.xml:ro @@ -184,7 +184,7 @@ services: cache: profiles: - api - image: redis:4.0.14 + image: docker.io/redis:4.0.14 ports: - "50263:6379" @@ -306,7 +306,7 @@ services: proxy: profiles: - api - image: nginx:alpine + image: docker.io/nginx:alpine ports: - "50200:9080" - "50243:9443" diff --git a/documentation/Pipfile.lock b/documentation/Pipfile.lock index d64ace76ca1..e8ac0ceb768 100644 --- a/documentation/Pipfile.lock +++ b/documentation/Pipfile.lock @@ -51,84 +51,99 @@ }, "charset-normalizer": { "hashes": [ - "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96", - "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c", - "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710", - "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706", - "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020", - "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252", - "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad", - "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329", - "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a", - "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f", - "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6", - "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4", - "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a", - "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46", - "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2", - "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23", - "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace", - "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd", - "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982", - "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10", - "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2", - "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea", - "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09", - "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5", - "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149", - "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489", - "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9", - "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80", - "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592", - "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3", - "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6", - "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed", - "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c", - "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200", - "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a", - "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e", - "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d", - "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6", - "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623", - "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669", - "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3", - "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa", - "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9", - "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2", - "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f", - "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1", - "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4", - "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a", - "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8", - "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3", - "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029", - "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f", - "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959", - "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22", - "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7", - "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952", - "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346", - "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e", - "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d", - "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299", - "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd", - "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a", - "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3", - "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037", - "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94", - "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c", - "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858", - "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a", - "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449", - "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c", - "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918", - "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1", - "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c", - "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac", - "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa" + "sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843", + "sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786", + "sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e", + "sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8", + "sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4", + "sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa", + "sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d", + "sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82", + "sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7", + "sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895", + "sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d", + "sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a", + "sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382", + "sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678", + "sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b", + "sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e", + "sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741", + "sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4", + "sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596", + "sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9", + "sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69", + "sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c", + "sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77", + "sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13", + "sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459", + "sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e", + "sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7", + "sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908", + "sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a", + "sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f", + "sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8", + "sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482", + "sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d", + "sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d", + "sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545", + "sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34", + "sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86", + "sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6", + "sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe", + "sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e", + "sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc", + "sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7", + "sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd", + "sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c", + "sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557", + "sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a", + "sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89", + "sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078", + "sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e", + "sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4", + "sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403", + "sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0", + "sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89", + "sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115", + "sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9", + "sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05", + "sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a", + "sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec", + "sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56", + "sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38", + "sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479", + "sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c", + "sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e", + "sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd", + "sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186", + "sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455", + "sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c", + "sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65", + "sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78", + "sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287", + "sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df", + "sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43", + "sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1", + "sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7", + "sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989", + "sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a", + "sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63", + "sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884", + "sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649", + "sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810", + "sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828", + "sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4", + "sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2", + "sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd", + "sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5", + "sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe", + "sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293", + "sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e", + "sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e", + "sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8" ], "markers": "python_full_version >= '3.7.0'", - "version": "==3.2.0" + "version": "==3.3.0" }, "colorama": { "hashes": [ @@ -148,11 +163,12 @@ }, "furo": { "hashes": [ - "sha256:12f99f87a1873b6746228cfde18f77244e6c1ffb85d7fed95e638aae70d80590", - "sha256:e671ee638ab3f1b472f4033b0167f502ab407830e0db0f843b1c1028119c9cd1" + "sha256:513092538537dc5c596691da06e3c370714ec99bc438680edc1debffb73e5bfc", + "sha256:5707530a476d2a63b8cad83b4f961f3739a69f4b058bcf38a03a39fa537195b2" ], "index": "pypi", - "version": "==2023.8.19" + "markers": "python_version >= '3.8'", + "version": "==2023.9.10" }, "idna": { "hashes": [ @@ -199,8 +215,11 @@ "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e", "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431", "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686", + "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c", "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559", "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc", + "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb", + "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939", "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c", "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0", "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4", @@ -208,6 +227,7 @@ "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575", "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba", "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d", + "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd", "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3", "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00", "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155", @@ -216,6 +236,7 @@ "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f", "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8", "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b", + "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007", "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24", "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea", "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198", @@ -223,9 +244,12 @@ "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee", "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be", "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2", + "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1", "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707", "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6", + "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c", "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58", + "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823", "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779", "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636", "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c", @@ -244,7 +268,9 @@ "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9", "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57", "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc", - "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2" + "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc", + "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2", + "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11" ], "markers": "python_version >= '3.7'", "version": "==2.1.3" @@ -271,15 +297,16 @@ "sha256:ea929a67a6a0b1683cdbe19b8d2e724cd7643f8aa3e7bb18dd65beac3483bead" ], "index": "pypi", + "markers": "python_version >= '3.8'", "version": "==2.0.0" }, "packaging": { "hashes": [ - "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", - "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f" + "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5", + "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7" ], "markers": "python_version >= '3.7'", - "version": "==23.1" + "version": "==23.2" }, "pygments": { "hashes": [ @@ -370,19 +397,20 @@ }, "soupsieve": { "hashes": [ - "sha256:1c1bfee6819544a3447586c889157365a27e10d88cde3ad3da0cf0ddf646feb8", - "sha256:89d12b2d5dfcd2c9e8c22326da9d9aa9cb3dfab0a83a024f05704076ee8d35ea" + "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690", + "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7" ], - "markers": "python_version >= '3.7'", - "version": "==2.4.1" + "markers": "python_version >= '3.8'", + "version": "==2.5" }, "sphinx": { "hashes": [ - "sha256:1a9290001b75c497fd087e92b0334f1bbfa1a1ae7fddc084990c4b7bd1130b88", - "sha256:9269f9ed2821c9ebd30e4204f5c2339f5d4980e377bc89cb2cb6f9b17409c20a" + "sha256:1e09160a40b956dc623c910118fa636da93bd3ca0b9876a7b3df90f07d691560", + "sha256:9a5160e1ea90688d5963ba09a2dcd8bdd526620edbb65c328728f1b2228d5ab5" ], "index": "pypi", - "version": "==7.2.5" + "markers": "python_version >= '3.9'", + "version": "==7.2.6" }, "sphinx-autobuild": { "hashes": [ @@ -390,6 +418,7 @@ "sha256:de1ca3b66e271d2b5b5140c35034c89e47f263f2cd5db302c9217065f7443f05" ], "index": "pypi", + "markers": "python_version >= '3.6'", "version": "==2021.3.14" }, "sphinx-basic-ng": { @@ -406,6 +435,7 @@ "sha256:fb543fd386d917746c9a2c50360c7905b605726b9355cd26e9974857afeae06e" ], "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==0.5.2" }, "sphinx-notfound-page": { @@ -414,6 +444,7 @@ "sha256:40a5741a6b07245a08fe55dbbd603ad6719e191b1419ab2e5337c706ebd16554" ], "index": "pypi", + "markers": "python_version >= '3.8'", "version": "==1.0.0" }, "sphinx-reredirects": { @@ -422,6 +453,7 @@ "sha256:a0e7213304759b01edc22f032f1715a1c61176fc8f167164e7a52b9feec9ac64" ], "index": "pypi", + "markers": "python_version >= '3.5'", "version": "==0.1.2" }, "sphinxcontrib-applehelp": { @@ -462,6 +494,7 @@ "sha256:6795a72037ca55e65663d2a2c1a043d636dc3d30d418e56dd6087d1459d98a5d" ], "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==0.9.2" }, "sphinxcontrib-qthelp": { @@ -499,11 +532,12 @@ }, "urllib3": { "hashes": [ - "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11", - "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4" + "sha256:7a7c7003b000adf9e7ca2a377c9688bbc54ed41b985789ed576570342a375cd2", + "sha256:b19e1a85d206b56d7df1d5e683df4a7725252a964e3993648dd0fb5a1c157564" ], + "index": "pypi", "markers": "python_version >= '3.7'", - "version": "==2.0.4" + "version": "==2.0.6" } } } diff --git a/catalog/DAGs.md b/documentation/catalog/reference/DAGs.md similarity index 95% rename from catalog/DAGs.md rename to documentation/catalog/reference/DAGs.md index ac3212fae71..72a3b8f14b6 100644 --- a/catalog/DAGs.md +++ b/documentation/catalog/reference/DAGs.md @@ -10,27 +10,27 @@ The DAGs are shown in two forms: - [DAGs by Type](#dags-by-type) - [Individual DAG documentation](#dag-documentation) -# DAGs by Type +## DAGs by Type The following are DAGs grouped by their primary tag: -1. [Data Normalization](#data_normalization) -1. [Data Refresh](#data_refresh) +1. [Data Normalization](#data-normalization) +1. [Data Refresh](#data-refresh) 1. [Database](#database) 1. [Maintenance](#maintenance) 1. [Oauth](#oauth) 1. [Other](#other) -1. [Popularity Refresh](#popularity_refresh) +1. [Popularity Refresh](#popularity-refresh) 1. [Provider](#provider) 1. [Provider Reingestion](#provider-reingestion) -## Data Normalization +### Data Normalization | DAG ID | Schedule Interval | | ------------------------------------- | ----------------- | | [`add_license_url`](#add_license_url) | `None` | -## Data Refresh +### Data Refresh | DAG ID | Schedule Interval | | ------------------------------------------------------------- | ----------------- | @@ -39,7 +39,7 @@ The following are DAGs grouped by their primary tag: | [`create_filtered_image_index`](#create_filtered_image_index) | `None` | | [`image_data_refresh`](#image_data_refresh) | `0 0 * * 1` | -## Database +### Database | DAG ID | Schedule Interval | | --------------------------------------------------------------------------------- | ----------------- | @@ -49,7 +49,7 @@ The following are DAGs grouped by their primary tag: | [`report_pending_reported_media`](#report_pending_reported_media) | `@weekly` | | [`staging_database_restore`](#staging_database_restore) | `@monthly` | -## Maintenance +### Maintenance | DAG ID | Schedule Interval | | --------------------------------------------------------------------------- | ----------------- | @@ -59,27 +59,27 @@ The following are DAGs grouped by their primary tag: | [`pr_review_reminders`](#pr_review_reminders) | `0 0 * * 1-5` | | [`rotate_db_snapshots`](#rotate_db_snapshots) | `0 0 * * 6` | -## Oauth +### Oauth | DAG ID | Schedule Interval | | ----------------------------------------------- | ----------------- | | [`oauth2_authorization`](#oauth2_authorization) | `None` | | [`oauth2_token_refresh`](#oauth2_token_refresh) | `0 */12 * * *` | -## Other +### Other | DAG ID | Schedule Interval | | --------------------------------------------------------- | ----------------- | | [`flickr_thumbnails_removal`](#flickr_thumbnails_removal) | `None` | -## Popularity Refresh +### Popularity Refresh | DAG ID | Schedule Interval | | ------------------------------------------------------- | ----------------- | | [`audio_popularity_refresh`](#audio_popularity_refresh) | `@monthly` | | [`image_popularity_refresh`](#image_popularity_refresh) | `@monthly` | -## Provider +### Provider | DAG ID | Schedule Interval | Dated | Media Type(s) | | --------------------------------------------------------------- | ----------------- | ------- | ------------- | @@ -105,7 +105,7 @@ The following are DAGs grouped by their primary tag: | [`wikimedia_commons_workflow`](#wikimedia_commons_workflow) | `@daily` | `True` | image, audio | | [`wordpress_workflow`](#wordpress_workflow) | `@monthly` | `False` | image | -## Provider Reingestion +### Provider Reingestion | DAG ID | Schedule Interval | | --------------------------------------------------------------------------------------- | ----------------- | @@ -114,7 +114,7 @@ The following are DAGs grouped by their primary tag: | [`phylopic_reingestion_workflow`](#phylopic_reingestion_workflow) | `@weekly` | | [`wikimedia_reingestion_workflow`](#wikimedia_reingestion_workflow) | `@weekly` | -# DAG documentation +## DAG documentation The following is documentation associated with each DAG (where available): @@ -160,9 +160,9 @@ The following is documentation associated with each DAG (where available): 1. [`wikimedia_reingestion_workflow`](#wikimedia_reingestion_workflow) 1. [`wordpress_workflow`](#wordpress_workflow) -## `add_license_url` +### `add_license_url` -### Add license URL +#### Add license URL Add `license_url` to all rows that have `NULL` in their `meta_data` fields. This PR sets the meta_data value to "{license_url: https://... }", where the url is @@ -172,9 +172,9 @@ This is a maintenance DAG that should be run once. If all the null values in the `meta_data` column are updated, the DAG will only run the first and the last step, logging the statistics. -## `airflow_log_cleanup` +### `airflow_log_cleanup` -### Clean up airflow logs +#### Clean up airflow logs A maintenance workflow that you can deploy into Airflow to periodically clean out the task logs to avoid those getting too big. By default, this will also @@ -197,9 +197,9 @@ airflow dags trigger --conf - maxLogAgeInDays: - Optional - enableDelete: - Optional -## `audio_data_refresh` +### `audio_data_refresh` -### Data Refresh DAG Factory +#### Data Refresh DAG Factory This file generates our data refresh DAGs using a factory function. For the given media type these DAGs will initiate a data refresh on the ingestion server @@ -221,9 +221,9 @@ and related PRs: - [[Feature] Data refresh orchestration DAG](https://github.com/WordPress/openverse-catalog/issues/353) - [[Feature] Merge popularity calculations and data refresh into a single DAG](https://github.com/WordPress/openverse-catalog/issues/453) -## `audio_popularity_refresh` +### `audio_popularity_refresh` -### Popularity Refresh DAG Factory +#### Popularity Refresh DAG Factory This file generates our popularity refresh DAGs using a factory function. @@ -244,7 +244,7 @@ implementation plan: - [[Implementation Plan] Decoupling Popularity Calculations from the Data Refresh](https://docs.openverse.org/projects/proposals/popularity_optimizations/20230420-implementation_plan_popularity_optimizations.html) -## `batched_update` +### `batched_update` Batched Update DAG @@ -311,9 +311,9 @@ used when the DagRun configuration needs to be changed after the table was already created: for example, if there was a problem with the `update_query` which caused DAG failures during the `update_batches` step. -## `check_silenced_dags` +### `check_silenced_dags` -### Silenced DAGs check +#### Silenced DAGs check Check for DAGs that have silenced Slack alerts or skipped errors which may need to be turned back on. @@ -336,9 +336,9 @@ issue has been resolved. The DAG runs weekly. -## `create_filtered_audio_index` +### `create_filtered_audio_index` -### Create filtered index DAG factory +#### Create filtered index DAG factory This module creates the filtered index creation DAGs for each media type using a factory function. @@ -350,7 +350,7 @@ index for a given media type. The DAG awaits the completion of the filtered index creation and then points the filtered index alias for the media type to the newly created index. -#### When this DAG runs +##### When this DAG runs The DAGs generated in this module are triggered by the data refresh DAGs. Maintaining this process separate from the data refresh DAGs, while still @@ -360,7 +360,7 @@ changes to the filtered index creation; and for re-running filtered index creation if an urgent change to the sensitive terms calls for an immediate recreation of the filtered indexes. -#### Race conditions +##### Race conditions Because filtered index creation employs the `reindex` Elasticsearch API to derive the filtered index from an existing index, we need to be mindful of the @@ -389,9 +389,9 @@ parameter is only for use by the data refresh DAG and should not be used when manually triggering the DAG unless you are absolutely certain of what you are doing. -## `create_filtered_image_index` +### `create_filtered_image_index` -### Create filtered index DAG factory +#### Create filtered index DAG factory This module creates the filtered index creation DAGs for each media type using a factory function. @@ -403,7 +403,7 @@ index for a given media type. The DAG awaits the completion of the filtered index creation and then points the filtered index alias for the media type to the newly created index. -#### When this DAG runs +##### When this DAG runs The DAGs generated in this module are triggered by the data refresh DAGs. Maintaining this process separate from the data refresh DAGs, while still @@ -413,7 +413,7 @@ changes to the filtered index creation; and for re-running filtered index creation if an urgent change to the sensitive terms calls for an immediate recreation of the filtered indexes. -#### Race conditions +##### Race conditions Because filtered index creation employs the `reindex` Elasticsearch API to derive the filtered index from an existing index, we need to be mindful of the @@ -442,7 +442,7 @@ parameter is only for use by the data refresh DAG and should not be used when manually triggering the DAG unless you are absolutely certain of what you are doing. -## `europeana_workflow` +### `europeana_workflow` Content Provider: Europeana @@ -452,7 +452,7 @@ Output: TSV file containing the images and the respective meta-data. Notes: https://pro.europeana.eu/page/search -## `finnish_museums_workflow` +### `finnish_museums_workflow` Content Provider: Finnish Museums @@ -466,15 +466,15 @@ script is a dated DAG that ingests all records that were last updated in the previous day. Because of this, it is not necessary to run a separate reingestion DAG, as updated data will be processed during regular ingestion. -## `flickr_audit_sub_provider_workflow` +### `flickr_audit_sub_provider_workflow` -### Flickr Sub Provider Audit +#### Flickr Sub Provider Audit Check the list of member institutions of the Flickr Commons for institutions that have cc-licensed images and are not already configured as sub-providers for the Flickr DAG. Report suggestions for new sub-providers to Slack. -## `flickr_reingestion_workflow` +### `flickr_reingestion_workflow` Content Provider: Flickr @@ -484,12 +484,12 @@ Output: TSV file containing the images and the respective meta-data. Notes: https://www.flickr.com/help/terms/api Rate limit: 3600 requests per hour. -## `flickr_thumbnails_removal` +### `flickr_thumbnails_removal` One-time run DAG to remove progressively all the old Flickr thumbnails, as they were determined to be unsuitable for the Openverse UI requirements. -## `flickr_workflow` +### `flickr_workflow` Content Provider: Flickr @@ -499,7 +499,7 @@ Output: TSV file containing the images and the respective meta-data. Notes: https://www.flickr.com/help/terms/api Rate limit: 3600 requests per hour. -## `freesound_workflow` +### `freesound_workflow` Content Provider: Freesound @@ -510,9 +510,9 @@ Output: TSV file containing the image, the respective meta-data. Notes: https://freesound.org/docs/api/ Rate limit: No limit for our API key. This script can be run either to ingest the full dataset or as a dated DAG. -## `image_data_refresh` +### `image_data_refresh` -### Data Refresh DAG Factory +#### Data Refresh DAG Factory This file generates our data refresh DAGs using a factory function. For the given media type these DAGs will initiate a data refresh on the ingestion server @@ -534,9 +534,9 @@ and related PRs: - [[Feature] Data refresh orchestration DAG](https://github.com/WordPress/openverse-catalog/issues/353) - [[Feature] Merge popularity calculations and data refresh into a single DAG](https://github.com/WordPress/openverse-catalog/issues/453) -## `image_popularity_refresh` +### `image_popularity_refresh` -### Popularity Refresh DAG Factory +#### Popularity Refresh DAG Factory This file generates our popularity refresh DAGs using a factory function. @@ -557,7 +557,7 @@ implementation plan: - [[Implementation Plan] Decoupling Popularity Calculations from the Data Refresh](https://docs.openverse.org/projects/proposals/popularity_optimizations/20230420-implementation_plan_popularity_optimizations.html) -## `inaturalist_workflow` +### `inaturalist_workflow` Provider: iNaturalist @@ -575,7 +575,7 @@ We use the table structure defined here, https://github.com/inaturalist/inaturalist-open-data/blob/main/Metadata/structure.sql except for adding ancestry tags to the taxa table. -## `jamendo_workflow` +### `jamendo_workflow` Content Provider: Jamendo @@ -588,7 +588,7 @@ non-commercial apps Jamendo Music has more than 500,000 tracks shared by 40,000 artists from over 150 countries all over the world. Audio quality: uploaded as WAV/ FLAC/ AIFF bit depth: 16/24 sample rate: 44.1 or 48 kHz channels: 1/2 -## `justtakeitfree_workflow` +### `justtakeitfree_workflow` Content Provider: Justtakeitfree @@ -599,7 +599,7 @@ Output: TSV file containing the media and the respective meta-data. Notes: https://justtakeitfree.com/api/api.php This API requires an API key. For more details, see https://github.com/WordPress/openverse/pull/2793 -## `metropolitan_museum_reingestion_workflow` +### `metropolitan_museum_reingestion_workflow` Content Provider: Metropolitan Museum of Art @@ -623,7 +623,7 @@ blocking during local development testing. connect with just date and license. https://collectionapi.metmuseum.org/public/collection/v1/search?isPublicDomain=true&metadataDate=2022-08-07 -## `metropolitan_museum_workflow` +### `metropolitan_museum_workflow` Content Provider: Metropolitan Museum of Art @@ -647,7 +647,7 @@ blocking during local development testing. connect with just date and license. https://collectionapi.metmuseum.org/public/collection/v1/search?isPublicDomain=true&metadataDate=2022-08-07 -## `nappy_workflow` +### `nappy_workflow` Content Provider: Nappy @@ -658,9 +658,9 @@ Output: TSV file containing the image meta-data. Notes: This api was written specially for Openverse. There are no known limits or restrictions. https://nappy.co/ -## `oauth2_authorization` +### `oauth2_authorization` -### OAuth Provider Authorization +#### OAuth Provider Authorization Iterates through all the OAuth2 providers and attempts to authorize them using tokens found in the in the `OAUTH2_AUTH_KEYS` Variable. Once authorization has @@ -672,9 +672,9 @@ authorization will create an access/refresh token pair in the - Freesound -## `oauth2_token_refresh` +### `oauth2_token_refresh` -### OAuth Provider Token Refresh +#### OAuth Provider Token Refresh Iterates through all OAuth2 providers and attempts to refresh the access token using the refresh token stored in the `OAUTH2_ACCESS_TOKENS` Variable. This DAG @@ -684,7 +684,7 @@ will update the tokens stored in the Variable upon successful refresh. - Freesound -## `phylopic_reingestion_workflow` +### `phylopic_reingestion_workflow` Content Provider: PhyloPic @@ -694,7 +694,7 @@ Output: TSV file containing the image, their respective meta-data. Notes: http://api-docs.phylopic.org/v2/ No rate limit specified. -## `phylopic_workflow` +### `phylopic_workflow` Content Provider: PhyloPic @@ -704,9 +704,9 @@ Output: TSV file containing the image, their respective meta-data. Notes: http://api-docs.phylopic.org/v2/ No rate limit specified. -## `pr_review_reminders` +### `pr_review_reminders` -### PR Review Reminders +#### PR Review Reminders Iterates through open PRs in our repositories and pings assigned reviewers who have not yet approved the PR or explicitly requested changes. @@ -727,7 +727,7 @@ Unfortunately the DAG does not know when someone is on vacation. It is up to the author of the PR to re-assign review if one of the randomly selected reviewers is unavailable for the time period during which the PR should be reviewed. -## `rawpixel_workflow` +### `rawpixel_workflow` Content Provider: Rawpixel @@ -741,7 +741,7 @@ issues. The public API max results range is limited to 100,000 results, although the API key we've been given can circumvent this limit. https://www.rawpixel.com/api/v1/search?tags=$publicdomain&page=1&pagesize=100 -## `recreate_audio_popularity_calculation` +### `recreate_audio_popularity_calculation` This file generates Apache Airflow DAGs that, for the given media type, completely wipes out and recreates the PostgreSQL functions involved in @@ -755,7 +755,7 @@ popularity constants and standardized popularity scores using the new functions. These DAGs are not on a schedule, and should only be run manually when new SQL code is deployed for the calculation. -## `recreate_image_popularity_calculation` +### `recreate_image_popularity_calculation` This file generates Apache Airflow DAGs that, for the given media type, completely wipes out and recreates the PostgreSQL functions involved in @@ -769,9 +769,9 @@ popularity constants and standardized popularity scores using the new functions. These DAGs are not on a schedule, and should only be run manually when new SQL code is deployed for the calculation. -## `report_pending_reported_media` +### `report_pending_reported_media` -### Report Pending Reported Media DAG +#### Report Pending Reported Media DAG This DAG checks for any user-reported media pending manual review, and alerts via Slack. @@ -782,7 +782,7 @@ whether further action (such as deindexing the record) needs to be taken. If a record has been reported multiple times, it only needs to be reviewed once and so is only counted once in the reporting by this DAG. -## `rotate_db_snapshots` +### `rotate_db_snapshots` Manages weekly database snapshots. @@ -799,7 +799,7 @@ Requires two variables: `CATALOG_RDS_DB_IDENTIFIER`: The "DBIdentifier" of the RDS DB instance. `CATALOG_RDS_SNAPSHOTS_TO_RETAIN`: How many historical snapshots to retain. -## `science_museum_workflow` +### `science_museum_workflow` Content Provider: Science Museum @@ -811,7 +811,7 @@ Notes: https://github.com/TheScienceMuseum/collectionsonline/wiki/Collections-Online-API Rate limited, no specific rate given. -## `smithsonian_workflow` +### `smithsonian_workflow` Content Provider: Smithsonian @@ -821,7 +821,7 @@ Output: TSV file containing the images and the respective meta-data. Notes: https://api.si.edu/openaccess/api/v1.0/search -## `smk_workflow` +### `smk_workflow` Content Provider: Statens Museum for Kunst (National Gallery of Denmark) @@ -831,9 +831,9 @@ Output: TSV file containing the media metadata. Notes: https://www.smk.dk/en/article/smk-api/ -## `staging_database_restore` +### `staging_database_restore` -### Update the staging database +#### Update the staging database This DAG is responsible for updating the staging database using the most recent snapshot of the production database. @@ -855,7 +855,7 @@ the RDS operations run using a different hook: - `AIRFLOW_CONN_`: The connection string to use for RDS operations (per the above example, it might be `AIRFLOW_CONN_AWS_RDS`) -## `stocksnap_workflow` +### `stocksnap_workflow` Content Provider: StockSnap @@ -867,7 +867,7 @@ Notes: https://stocksnap.io/api/load-photos/date/desc/1 https://stocksnap.io/faq All images are licensed under CC0. No rate limits or authorization required. API is undocumented. -## `wikimedia_commons_workflow` +### `wikimedia_commons_workflow` **Content Provider:** Wikimedia Commons @@ -875,7 +875,7 @@ is undocumented. **Output:** TSV file containing the image, the respective meta-data. -### Notes +#### Notes Rate limit of no more than 200 requests/second, and we are required to set a unique User-Agent field @@ -967,7 +967,7 @@ cases where they might exceed the limit. Technically, it's feasible for almost any property to exceed the limit, but these are the ones that we've seen in practice. -#### `imageinfo` +##### `imageinfo` [Docs](https://commons.wikimedia.org/w/api.php?action=help&modules=query%2Bimageinfo) @@ -977,7 +977,7 @@ practice. For these requests, we can remove the `metadata` property from the `iiprops` parameter to avoid this issue on subsequent iterations. -#### `globalusage` +##### `globalusage` [Docs](https://commons.wikimedia.org/w/api.php?action=help&modules=query%2Bglobalusage) @@ -986,7 +986,7 @@ parameter to avoid this issue on subsequent iterations. For these requests, we can remove the `globalusage` property from the `prop` parameter entirely and eschew the popularity data for these items. -## `wikimedia_reingestion_workflow` +### `wikimedia_reingestion_workflow` **Content Provider:** Wikimedia Commons @@ -994,7 +994,7 @@ parameter entirely and eschew the popularity data for these items. **Output:** TSV file containing the image, the respective meta-data. -### Notes +#### Notes Rate limit of no more than 200 requests/second, and we are required to set a unique User-Agent field @@ -1086,7 +1086,7 @@ cases where they might exceed the limit. Technically, it's feasible for almost any property to exceed the limit, but these are the ones that we've seen in practice. -#### `imageinfo` +##### `imageinfo` [Docs](https://commons.wikimedia.org/w/api.php?action=help&modules=query%2Bimageinfo) @@ -1096,7 +1096,7 @@ practice. For these requests, we can remove the `metadata` property from the `iiprops` parameter to avoid this issue on subsequent iterations. -#### `globalusage` +##### `globalusage` [Docs](https://commons.wikimedia.org/w/api.php?action=help&modules=query%2Bglobalusage) @@ -1105,7 +1105,7 @@ parameter to avoid this issue on subsequent iterations. For these requests, we can remove the `globalusage` property from the `prop` parameter entirely and eschew the popularity data for these items. -## `wordpress_workflow` +### `wordpress_workflow` Content Provider: WordPress Photo Directory diff --git a/documentation/catalog/reference/index.md b/documentation/catalog/reference/index.md new file mode 100644 index 00000000000..b98ba63ec03 --- /dev/null +++ b/documentation/catalog/reference/index.md @@ -0,0 +1,7 @@ +# Reference + +```{toctree} +:titlesonly: + +DAGs +``` diff --git a/documentation/changelogs/api/2023.09.28.00.26.34.md b/documentation/changelogs/api/2023.09.28.00.26.34.md new file mode 100644 index 00000000000..05cea37ff47 --- /dev/null +++ b/documentation/changelogs/api/2023.09.28.00.26.34.md @@ -0,0 +1,13 @@ +# 2023.09.28.00.26.34 + +## Internal Improvements + +- Upgrade psycopg to version 3 in the API + ([#3064](https://github.com/WordPress/openverse/pull/3064)) by @krysal + +## Bug Fixes + +- Transfer UUID validation inside serializer + ([#3068](https://github.com/WordPress/openverse/pull/3068)) by @dhruvkb +- Replace deprecated body in the ES requests with individual parameters + ([#3055](https://github.com/WordPress/openverse/pull/3055)) by @ngken0995 diff --git a/documentation/changelogs/api/2023.09.30.00.15.32.md b/documentation/changelogs/api/2023.09.30.00.15.32.md new file mode 100644 index 00000000000..df13d9af92c --- /dev/null +++ b/documentation/changelogs/api/2023.09.30.00.15.32.md @@ -0,0 +1,6 @@ +# 2023.09.30.00.15.32 + +## Improvements + +- Increase the API sources cache TTL from 20 minutes to 4 hours + ([#3083](https://github.com/WordPress/openverse/pull/3083)) by @AetherUnbound diff --git a/documentation/changelogs/ingestion_server/2023.09.29.17.40.50.md b/documentation/changelogs/ingestion_server/2023.09.29.17.40.50.md new file mode 100644 index 00000000000..ab27d9a61bc --- /dev/null +++ b/documentation/changelogs/ingestion_server/2023.09.29.17.40.50.md @@ -0,0 +1,11 @@ +# 2023.09.29.17.40.50 + +## Improvements + +- Refactor popularity SQL + ([#2964](https://github.com/WordPress/openverse/pull/2964)) by @stacimc + +## Bug Fixes + +- Upgrade ES dependencies to match cluster version + ([#3029](https://github.com/WordPress/openverse/pull/3029)) by @sarayourfriend diff --git a/documentation/meta/monitoring/runbooks/api_thumbnails_avg_response_time_above_threshold.md b/documentation/meta/monitoring/runbooks/api_thumbnails_avg_response_time_above_threshold.md new file mode 100644 index 00000000000..b48e6d9fafe --- /dev/null +++ b/documentation/meta/monitoring/runbooks/api_thumbnails_avg_response_time_above_threshold.md @@ -0,0 +1,30 @@ +# Run Book: API Thumbnails Production Average Response Time above threshold + +```{admonition} Metadata +Status: **Unstable** +Maintainer: @stacimc +Alarm link: +- +``` + +## Severity Guide + +If the avg response time is not [anomalously high][anomaly_alarm], the severity +is likely low. Check for a recent deployment that may have introduced the +problem, and rollback to the previous version. If not, check the request count +and general network activity. If abnormally high, refer to the [traffic analysis +run book][traffic_runbook] to identify and block any malicious traffic. + +[anomaly_alarm]: + https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#alarmsV2:alarm/API+Thumbnails+Production+Average+Response+Time+anomalously+high +[traffic_runbook]: + /meta/monitoring/traffic/runbooks/identifying-and-blocking-traffic-anomalies.md + +## Historical false positives + +Nothing registered to date. + +## Related incident reports + +- 2023-09-05 at 22:15 UTC: Unhealthy thumbnail tasks restarted +- 2023-07-27 at 19:14 UTC: API Thumbnails unhealthy hosts diff --git a/documentation/meta/monitoring/runbooks/api_thumbnails_avg_response_time_anomaly.md b/documentation/meta/monitoring/runbooks/api_thumbnails_avg_response_time_anomaly.md new file mode 100644 index 00000000000..d556118fb5a --- /dev/null +++ b/documentation/meta/monitoring/runbooks/api_thumbnails_avg_response_time_anomaly.md @@ -0,0 +1,28 @@ +# Run Book: API Thumbnails Production Average Response Time anomalously high + +```{admonition} Metadata +Status: **Unstable** +Maintainer: @stacimc +Alarm link: +- +``` + +## Severity Guide + +Confirm that there is not a total outage of the service. If not, the severity is +likely low. Check for a recent deployment that may have introduced the problem, +and rollback to the previous version. If not, check the request count and +general network activity. If abnormally high, refer to the [traffic analysis run +book][traffic_runbook] to identify and block any malicious traffic. + +[traffic_runbook]: + /meta/monitoring/traffic/runbooks/identifying-and-blocking-traffic-anomalies.md + +## Historical false positives + +Nothing registered to date. + +## Related incident reports + +- 2023-09-05 at 22:15 UTC: Unhealthy thumbnail tasks restarted +- 2023-07-27 at 19:14 UTC: API Thumbnails unhealthy hosts diff --git a/documentation/meta/monitoring/runbooks/api_thumbnails_http_2xx_under_threshold.md b/documentation/meta/monitoring/runbooks/api_thumbnails_http_2xx_under_threshold.md new file mode 100644 index 00000000000..719c5e58b3b --- /dev/null +++ b/documentation/meta/monitoring/runbooks/api_thumbnails_http_2xx_under_threshold.md @@ -0,0 +1,36 @@ +# Run Book: API Thumbnails Production HTTP 2XX responses count under threshold + +```{admonition} Metadata +Status: **Unstable** + +Maintainer: @krysaldb + +Alarm link: +- +``` + +## Severity Guide + +After confirming there is not a total outage, check if the overall request count +has decreased as well (go to the [CloudWatch dashboard][cloudwatch] or +alternatively check in CloudFlare). If the overall requests are lower then the +severity is low, and you should continue searching for the cause of the general +decrease. + +If the lower number is only in 2XX responses the severity is likely high, so +also check the dashboard to look for other anomalies. Verify if any of the +thumbnail providers are experiencing an outage or are rate-limiting Openverse. +Go to the [API logs][api_logs] to check for errors or data that yield clues. + +[cloudwatch]: + https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#dashboards/dashboard/ECS-Production-Dashboard +[api_logs]: + https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#logsV2:log-groups/log-group/$252Fecs$252Fproduction$252Fapi + +## Historical false positives + +Nothing registered to date. + +## Related incident reports + +Nothing registered to date. diff --git a/documentation/meta/monitoring/runbooks/api_thumbnails_http_5xx_above_threshold.md b/documentation/meta/monitoring/runbooks/api_thumbnails_http_5xx_above_threshold.md new file mode 100644 index 00000000000..83acf7a0bf7 --- /dev/null +++ b/documentation/meta/monitoring/runbooks/api_thumbnails_http_5xx_above_threshold.md @@ -0,0 +1,31 @@ +# Run Book: API Thumbnails Production HTTP 5XX responses count above threshold + +```{admonition} Metadata +Status: **Unstable** + +Maintainer: @krysaldb + +Alarm link: +- +``` + +## Severity Guide + +After confirming there is not a total outage, check if the increase of 5XX HTTP +errors is related to a regular time where resources are expected to be +constrained like a recent deployment, a data refresh, DB maintenance, etc. If +the spike is related to one of these events and the alarms stabilizes in the +short time then the severity is low. + +If the issue is not related to known recurrent events and persists, the severity +is critical. Check if dependent services –DB, Redis, Elasticsearch– are +available to the API or if the problem is intrinsic to itself. + +## Historical false positives + +Nothing registered to date. + +## Related incident reports + +- 2023-09-27 from 16:15 to 18:00 UTC: 5XX responses spike between 983 and 1434 + due to unstable tasks. diff --git a/documentation/meta/monitoring/runbooks/api_thumbnails_p99_response_time_above_threshold.md b/documentation/meta/monitoring/runbooks/api_thumbnails_p99_response_time_above_threshold.md new file mode 100644 index 00000000000..557140e524d --- /dev/null +++ b/documentation/meta/monitoring/runbooks/api_thumbnails_p99_response_time_above_threshold.md @@ -0,0 +1,30 @@ +# Run Book: API Thumbnails Production P99 Response Time above threshold + +```{admonition} Metadata +Status: **Unstable** +Maintainer: @stacimc +Alarm link: +- +``` + +## Severity Guide + +If the P99 response time is not [anomalously high][anomaly_alarm], the severity +is likely low. Check for a recent deployment that may have introduced the +problem, and rollback to the previous version. If not, check the request count +and general network activity. If abnormally high, refer to the [traffic analysis +run book][traffic_runbook] to identify and block any malicious traffic. + +[anomaly_alarm]: + https://us-east-1.console.aws.amazon.com/cloudwatch/home?region=us-east-1#alarmsV2:alarm/API+Thumbnails+Production+P99+Response+Time+anomalously+high +[traffic_runbook]: + /meta/monitoring/traffic/runbooks/identifying-and-blocking-traffic-anomalies.md + +## Historical false positives + +Nothing registered to date. + +## Related incident reports + +- 2023-09-05 at 22:15 UTC: Unhealthy thumbnail tasks restarted +- 2023-07-27 at 19:14 UTC: API Thumbnails unhealthy hosts diff --git a/documentation/meta/monitoring/runbooks/api_thumbnails_p99_response_time_anomaly.md b/documentation/meta/monitoring/runbooks/api_thumbnails_p99_response_time_anomaly.md new file mode 100644 index 00000000000..a3379025039 --- /dev/null +++ b/documentation/meta/monitoring/runbooks/api_thumbnails_p99_response_time_anomaly.md @@ -0,0 +1,28 @@ +# Run Book: API Thumbnails Production P99 Response Time anomalously high + +```{admonition} Metadata +Status: **Unstable** +Maintainer: @stacimc +Alarm link: +- +``` + +## Severity Guide + +Confirm that there is not a total outage of the service. If not, the severity is +likely low. Check for a recent deployment that may have introduced the problem, +and rollback to the previous version. If not, check the request count and +general network activity. If abnormally high, refer to the [traffic analysis run +book][traffic_runbook] to identify and block any malicious traffic. + +[traffic_runbook]: + /meta/monitoring/traffic/runbooks/identifying-and-blocking-traffic-anomalies.md + +## Historical false positives + +Nothing registered to date. + +## Related incident reports + +- 2023-09-05 at 22:15 UTC: Unhealthy thumbnail tasks restarted +- 2023-07-27 at 19:14 UTC: API Thumbnails unhealthy hosts diff --git a/documentation/meta/monitoring/runbooks/api_thumbnails_request_count_anomaly.md b/documentation/meta/monitoring/runbooks/api_thumbnails_request_count_anomaly.md new file mode 100644 index 00000000000..a80595f87c6 --- /dev/null +++ b/documentation/meta/monitoring/runbooks/api_thumbnails_request_count_anomaly.md @@ -0,0 +1,34 @@ +# Run Book: API Thumbnails Production Request Count anomalously high + +```{admonition} Metadata +Status: **Unstable** +Maintainer: @krysaldb +Alarm link: +- +``` + +## Severity Guide + +When a sudden increase in request count is noticed, verify that the services are +supporting the load by looking at metrics like response time or ES CPU usage for +example. If the API is doing fine, then severity is low and may only require +future resource scaling depending on the kind of traffic. + +If the services are strained then the severity is critical, search for the root +cause to prevent more serious outages. If there are no recent obvious +integrations (like the Gutenberg plugin) then follow the run book to [identify +traffic anomalies in Cloudflare][runbook_traffic], to determine whether the +recent traffic is organic or if it comes from a botnet. Find the origin of +requests and evaluate whether it needs to be blocked or if Openverse services +need to adapt to the new demand. + +[traffic_runbook]: + /meta/monitoring/traffic/runbooks/identifying-and-blocking-traffic-anomalies.md + +## Historical false positives + +Nothing registered to date. + +## Related incident reports + +Nothing registered to date. diff --git a/documentation/meta/monitoring/runbooks/index.md b/documentation/meta/monitoring/runbooks/index.md index 3c4745f0a52..2145a9dd494 100644 --- a/documentation/meta/monitoring/runbooks/index.md +++ b/documentation/meta/monitoring/runbooks/index.md @@ -19,6 +19,13 @@ api_avg_response_time_above_threshold api_avg_response_time_anomaly api_p99_response_time_above_threshold api_p99_response_time_anomaly +api_thumbnails_http_2xx_under_threshold +api_thumbnails_http_5xx_above_threshold +api_thumbnails_request_count_anomaly +api_thumbnails_avg_response_time_above_threshold +api_thumbnails_avg_response_time_anomaly +api_thumbnails_p99_response_time_above_threshold +api_thumbnails_p99_response_time_anomaly nuxt_request_count nuxt_2xx_under_threshold nuxt_5xx_above_threshold diff --git a/frontend/Dockerfile b/frontend/Dockerfile index fa7f9e6a45e..692af03504c 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -7,7 +7,7 @@ ARG FRONTEND_NODE_VERSION # Node.js builder # ################### -FROM node:${FRONTEND_NODE_VERSION}-alpine as builder +FROM docker.io/node:${FRONTEND_NODE_VERSION}-alpine as builder # Automatically use the right version of pnpm. ARG FRONTEND_PNPM_VERSION @@ -58,7 +58,7 @@ RUN pnpm build:only # Nuxt app # ############ -FROM node:${FRONTEND_NODE_VERSION}-alpine as app +FROM docker.io/node:${FRONTEND_NODE_VERSION}-alpine as app LABEL org.opencontainers.image.source="https://github.com/WordPress/openverse" diff --git a/frontend/src/components/VErrorSection/VErrorSection.vue b/frontend/src/components/VErrorSection/VErrorSection.vue index 42363862e6a..51edfcf524e 100644 --- a/frontend/src/components/VErrorSection/VErrorSection.vue +++ b/frontend/src/components/VErrorSection/VErrorSection.vue @@ -18,10 +18,9 @@ diff --git a/frontend/src/components/VSourcesTable.vue b/frontend/src/components/VSourcesTable.vue index 4fd59510c2d..719488bf31a 100644 --- a/frontend/src/components/VSourcesTable.vue +++ b/frontend/src/components/VSourcesTable.vue @@ -111,6 +111,11 @@ export default defineComponent({ function compareProviders(prov1: MediaProvider, prov2: MediaProvider) { let field1 = prov1[sorting.field] let field2 = prov2[sorting.field] + if (sorting.field === "display_name") { + field1 = prov1[sorting.field].toLowerCase() + field2 = prov2[sorting.field].toLowerCase() + } + if (sorting.field === "source_url") { field1 = cleanSourceUrlForPresentation(field1 as string) field2 = cleanSourceUrlForPresentation(field2 as string) diff --git a/frontend/src/constants/errors.ts b/frontend/src/constants/errors.ts index 9b48069ac10..7cf4057954e 100644 --- a/frontend/src/constants/errors.ts +++ b/frontend/src/constants/errors.ts @@ -5,7 +5,45 @@ export const NO_RESULT = "NO_RESULT" export const SERVER_TIMEOUT = "SERVER_TIMEOUT" +export const ECONNABORTED = "ECONNABORTED" -export const errorCodes = [NO_RESULT, SERVER_TIMEOUT] as const +export const ERR_UNKNOWN = "ERR_UNKNOWN" + +export const customErrorCodes = [ + NO_RESULT, + SERVER_TIMEOUT, + ECONNABORTED, + ERR_UNKNOWN, +] as const + +/** + * The error codes Axios uses. + * @see https://github.com/axios/axios/blob/9588fcdec8aca45c3ba2f7968988a5d03f23168c/lib/core/AxiosError.js#L57C2-L71 + */ +const axiosErrorCodes = [ + "ERR_BAD_OPTION_VALUE", + "ERR_BAD_OPTION", + "ECONNABORTED", + "ETIMEDOUT", + "ERR_NETWORK", + "ERR_FR_TOO_MANY_REDIRECTS", + "ERR_DEPRECATED", + "ERR_BAD_RESPONSE", + "ERR_BAD_REQUEST", + "ERR_CANCELED", + "ERR_NOT_SUPPORT", + "ERR_INVALID_URL", +] as const + +export const errorCodes = [...customErrorCodes, ...axiosErrorCodes] as const + +export const clientSideErrorCodes: readonly ErrorCode[] = [ + ECONNABORTED, + SERVER_TIMEOUT, + NO_RESULT, + ERR_UNKNOWN, + "ERR_NETWORK", + "ETIMEDOUT", +] as const export type ErrorCode = (typeof errorCodes)[number] diff --git a/frontend/src/middleware/search.ts b/frontend/src/middleware/search.ts index f0f2cd78c87..93a19c545b6 100644 --- a/frontend/src/middleware/search.ts +++ b/frontend/src/middleware/search.ts @@ -1,6 +1,7 @@ import { useSearchStore } from "~/stores/search" import { useMediaStore } from "~/stores/media" -import { NO_RESULT } from "~/constants/errors" + +import { handledClientSide } from "~/utils/errors" import type { Middleware } from "@nuxt/types" @@ -44,13 +45,7 @@ export const searchMiddleware: Middleware = async ({ const results = await mediaStore.fetchMedia() const fetchingError = mediaStore.fetchState.fetchingError - // NO_RESULTS and timeout are handled client-side, for other errors show server error page - if ( - !results && - fetchingError && - !fetchingError?.message?.includes(NO_RESULT) && - !fetchingError?.message?.includes("timeout") - ) { + if (!results && fetchingError && !handledClientSide(fetchingError)) { nuxtError(fetchingError) } } diff --git a/frontend/src/middleware/single-result.ts b/frontend/src/middleware/single-result.ts index ca9f2f81b65..83babf0218e 100644 --- a/frontend/src/middleware/single-result.ts +++ b/frontend/src/middleware/single-result.ts @@ -1,6 +1,7 @@ import { useSingleResultStore } from "~/stores/media/single-result" import { useSearchStore } from "~/stores/search" import { useRelatedMediaStore } from "~/stores/media/related-media" +import { isRetriable } from "~/utils/errors" import { AUDIO, IMAGE } from "~/constants/media" @@ -17,11 +18,15 @@ export const singleResultMiddleware: Middleware = async ({ if (process.server) { const media = await singleResultStore.fetch(mediaType, route.params.id) - await useRelatedMediaStore($pinia).fetchMedia(mediaType, route.params.id) if (!media) { - error(singleResultStore.fetchState.fetchingError ?? {}) + const fetchingError = singleResultStore.fetchState.fetchingError + + if (fetchingError && !isRetriable(fetchingError)) { + error(fetchingError ?? {}) + } } + await useRelatedMediaStore($pinia).fetchMedia(mediaType, route.params.id) } else { // Client-side rendering singleResultStore.setMediaById(mediaType, route.params.id) diff --git a/frontend/src/pages/audio/_id/index.vue b/frontend/src/pages/audio/_id/index.vue index 28afd835bd7..8091a2b9dfc 100644 --- a/frontend/src/pages/audio/_id/index.vue +++ b/frontend/src/pages/audio/_id/index.vue @@ -1,6 +1,11 @@