From e894621d4f09ae9c39273a5ecbf650ba54a4a632 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 12:16:57 -0400 Subject: [PATCH 01/17] publish calitp map utils to pypi --- .github/workflows/build-calitp-map-utils.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-calitp-map-utils.yml b/.github/workflows/build-calitp-map-utils.yml index 8c75c0c588..a2de5c18f5 100644 --- a/.github/workflows/build-calitp-map-utils.yml +++ b/.github/workflows/build-calitp-map-utils.yml @@ -27,9 +27,16 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.9' - - run: curl -sSL https://install.python-poetry.org | python - - - run: poetry install - - run: poetry run mypy calitp_map_utils - - run: poetry run pytest --spec - - run: poetry run build - # TODO: should we actually publish to pypi? + - run: | + curl -sSL https://install.python-poetry.org | python - + poetry install + poetry run mypy calitp_map_utils + poetry run pytest --spec + poetry run build + # TODO: only run on main/version changes + - run: | + echo $PYPI_TOKEN > token + poetry config pypi-token.pypi token + poetry run publish + env: + PYPI_TOKEN: ${{ secrets.PYPI_CALITP_MAP_UTILS_TOKEN }} From e864ef635862955b4bb116c20a2c8c08c08728cf Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 12:20:12 -0400 Subject: [PATCH 02/17] fix working directory --- .github/workflows/build-calitp-map-utils.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-calitp-map-utils.yml b/.github/workflows/build-calitp-map-utils.yml index a2de5c18f5..1949de9aac 100644 --- a/.github/workflows/build-calitp-map-utils.yml +++ b/.github/workflows/build-calitp-map-utils.yml @@ -20,7 +20,6 @@ jobs: check_and_build: name: check python runs-on: ubuntu-latest - working-directory: apps/maps steps: - run: sudo apt-get install -y libgraphviz-dev graphviz-dev - uses: actions/checkout@v3 @@ -33,10 +32,12 @@ jobs: poetry run mypy calitp_map_utils poetry run pytest --spec poetry run build + working-directory: apps/maps # TODO: only run on main/version changes - run: | echo $PYPI_TOKEN > token poetry config pypi-token.pypi token poetry run publish + working-directory: apps/maps env: PYPI_TOKEN: ${{ secrets.PYPI_CALITP_MAP_UTILS_TOKEN }} From c694baa05da1b826b6f8e20fc4f6d629ddaeef2c Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 12:27:35 -0400 Subject: [PATCH 03/17] add other pypi publishing --- .github/workflows/build-calitp-data-analysis.yml | 9 ++++++++- .github/workflows/build-calitp-data-infra.yml | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-calitp-data-analysis.yml b/.github/workflows/build-calitp-data-analysis.yml index 6d2d4070d6..fdbb5197f8 100644 --- a/.github/workflows/build-calitp-data-analysis.yml +++ b/.github/workflows/build-calitp-data-analysis.yml @@ -37,4 +37,11 @@ jobs: poetry run mypy . poetry run pytest poetry build -# TODO: add publishing + # TODO: only run on main/version changes + - run: | + echo $PYPI_TOKEN > token + poetry config pypi-token.pypi token + poetry run publish + working-directory: packages/calitp-data-analysis + env: + PYPI_TOKEN: ${{ secrets.PYPI_CALITP_DATA_ANALYSIS_TOKEN }} diff --git a/.github/workflows/build-calitp-data-infra.yml b/.github/workflows/build-calitp-data-infra.yml index 61f314a5ec..1d6fe40c79 100644 --- a/.github/workflows/build-calitp-data-infra.yml +++ b/.github/workflows/build-calitp-data-infra.yml @@ -38,4 +38,11 @@ jobs: CALITP_BUCKET__GTFS_DOWNLOAD_CONFIG: gs://this-does-not-exist CALITP_BUCKET__GTFS_SCHEDULE_RAW: gs://this-does-not-exist CALITP_BUCKET__GTFS_RT_RAW: gs://this-does-not-exist -# TODO: add publishing + # TODO: only run on main/version changes + - run: | + echo $PYPI_TOKEN > token + poetry config pypi-token.pypi token + poetry run publish + working-directory: packages/calitp-data-infra + env: + PYPI_TOKEN: ${{ secrets.PYPI_CALITP_DATA_INFRA_TOKEN }} From b3c693b5da0ea61720f1b2156fd2840305053363 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 12:32:04 -0400 Subject: [PATCH 04/17] pass pypi token as env --- .github/workflows/build-calitp-data-analysis.yml | 7 ++----- .github/workflows/build-calitp-data-infra.yml | 7 ++----- .github/workflows/build-calitp-map-utils.yml | 7 ++----- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-calitp-data-analysis.yml b/.github/workflows/build-calitp-data-analysis.yml index fdbb5197f8..7d9c35a181 100644 --- a/.github/workflows/build-calitp-data-analysis.yml +++ b/.github/workflows/build-calitp-data-analysis.yml @@ -38,10 +38,7 @@ jobs: poetry run pytest poetry build # TODO: only run on main/version changes - - run: | - echo $PYPI_TOKEN > token - poetry config pypi-token.pypi token - poetry run publish + - run: poetry publish working-directory: packages/calitp-data-analysis env: - PYPI_TOKEN: ${{ secrets.PYPI_CALITP_DATA_ANALYSIS_TOKEN }} + POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_DATA_ANALYSIS_TOKEN }} diff --git a/.github/workflows/build-calitp-data-infra.yml b/.github/workflows/build-calitp-data-infra.yml index 1d6fe40c79..8f28d68621 100644 --- a/.github/workflows/build-calitp-data-infra.yml +++ b/.github/workflows/build-calitp-data-infra.yml @@ -39,10 +39,7 @@ jobs: CALITP_BUCKET__GTFS_SCHEDULE_RAW: gs://this-does-not-exist CALITP_BUCKET__GTFS_RT_RAW: gs://this-does-not-exist # TODO: only run on main/version changes - - run: | - echo $PYPI_TOKEN > token - poetry config pypi-token.pypi token - poetry run publish + - run: poetry publish working-directory: packages/calitp-data-infra env: - PYPI_TOKEN: ${{ secrets.PYPI_CALITP_DATA_INFRA_TOKEN }} + POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_DATA_INFRA_TOKEN }} diff --git a/.github/workflows/build-calitp-map-utils.yml b/.github/workflows/build-calitp-map-utils.yml index 1949de9aac..ad6199b3e6 100644 --- a/.github/workflows/build-calitp-map-utils.yml +++ b/.github/workflows/build-calitp-map-utils.yml @@ -34,10 +34,7 @@ jobs: poetry run build working-directory: apps/maps # TODO: only run on main/version changes - - run: | - echo $PYPI_TOKEN > token - poetry config pypi-token.pypi token - poetry run publish + - run: poetry publish working-directory: apps/maps env: - PYPI_TOKEN: ${{ secrets.PYPI_CALITP_MAP_UTILS_TOKEN }} + POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_MAP_UTILS_TOKEN }} From bedb196324d1469dbebe3648a3b0c820e0a219d0 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 12:37:06 -0400 Subject: [PATCH 05/17] bump calitp map utils to test ci --- apps/maps/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/maps/pyproject.toml b/apps/maps/pyproject.toml index ec4dad4aea..d24a588d0b 100644 --- a/apps/maps/pyproject.toml +++ b/apps/maps/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "calitp_map_utils" -version = "2023.8.1" +version = "2023.8.2" description = "" authors = ["Andrew Vaccaro "] readme = "README.md" From 47542a6ef8f7b645016fd43f80db4f9baa62c725 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 12:42:55 -0400 Subject: [PATCH 06/17] try test pypi --- .github/workflows/build-calitp-map-utils.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-calitp-map-utils.yml b/.github/workflows/build-calitp-map-utils.yml index ad6199b3e6..fd54339ee1 100644 --- a/.github/workflows/build-calitp-map-utils.yml +++ b/.github/workflows/build-calitp-map-utils.yml @@ -34,7 +34,11 @@ jobs: poetry run build working-directory: apps/maps # TODO: only run on main/version changes - - run: poetry publish + - if: ${{ github.ref != 'refs/heads/main' }} + run: | + poetry config repositories.test-pypi https://test.pypi.org/legacy/ + poetry config pypi-token.test-pypi $POETRY_TEST_PYPI_TOKEN_PYPI + poetry publish -r test-pypi working-directory: apps/maps env: - POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_MAP_UTILS_TOKEN }} + POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.TEST_PYPI_CALITP_MAP_UTILS_TOKEN }} From 353c75f2372db4c1e41b617d166b3b0a395d6e4d Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 12:45:35 -0400 Subject: [PATCH 07/17] publish others to test, fix names --- .github/workflows/build-calitp-data-analysis.yml | 9 ++++++--- .github/workflows/build-calitp-data-infra.yml | 9 ++++++--- .github/workflows/build-calitp-map-utils.yml | 1 - 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-calitp-data-analysis.yml b/.github/workflows/build-calitp-data-analysis.yml index 7d9c35a181..0fd0a9b453 100644 --- a/.github/workflows/build-calitp-data-analysis.yml +++ b/.github/workflows/build-calitp-data-analysis.yml @@ -18,7 +18,6 @@ concurrency: jobs: check_and_build: - name: check python runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -38,7 +37,11 @@ jobs: poetry run pytest poetry build # TODO: only run on main/version changes - - run: poetry publish + - if: ${{ github.ref != 'refs/heads/main' }} + run: | + poetry config repositories.test-pypi https://test.pypi.org/legacy/ + poetry config pypi-token.test-pypi $POETRY_TEST_PYPI_TOKEN_PYPI + poetry publish -r test-pypi working-directory: packages/calitp-data-analysis env: - POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_DATA_ANALYSIS_TOKEN }} + POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.TEST_PYPI_CALITP_DATA_ANALYSIS_TOKEN }} diff --git a/.github/workflows/build-calitp-data-infra.yml b/.github/workflows/build-calitp-data-infra.yml index 8f28d68621..8e9d9e038a 100644 --- a/.github/workflows/build-calitp-data-infra.yml +++ b/.github/workflows/build-calitp-data-infra.yml @@ -18,7 +18,6 @@ concurrency: jobs: check_and_build: - name: check python runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -39,7 +38,11 @@ jobs: CALITP_BUCKET__GTFS_SCHEDULE_RAW: gs://this-does-not-exist CALITP_BUCKET__GTFS_RT_RAW: gs://this-does-not-exist # TODO: only run on main/version changes - - run: poetry publish + - if: ${{ github.ref != 'refs/heads/main' }} + run: | + poetry config repositories.test-pypi https://test.pypi.org/legacy/ + poetry config pypi-token.test-pypi $POETRY_TEST_PYPI_TOKEN_PYPI + poetry publish -r test-pypi working-directory: packages/calitp-data-infra env: - POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_DATA_INFRA_TOKEN }} + POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.TEST_PYPI_CALITP_DATA_INFRA_TOKEN }} diff --git a/.github/workflows/build-calitp-map-utils.yml b/.github/workflows/build-calitp-map-utils.yml index fd54339ee1..eb3fd420cc 100644 --- a/.github/workflows/build-calitp-map-utils.yml +++ b/.github/workflows/build-calitp-map-utils.yml @@ -18,7 +18,6 @@ concurrency: jobs: check_and_build: - name: check python runs-on: ubuntu-latest steps: - run: sudo apt-get install -y libgraphviz-dev graphviz-dev From 57c294026339a5d01e8408dc2bb14c90edda21bd Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 15:40:57 -0400 Subject: [PATCH 08/17] set up test and prod pypi publishing --- .github/workflows/build-calitp-data-analysis.yml | 9 +++++++-- .github/workflows/build-calitp-data-infra.yml | 11 ++++++++--- .github/workflows/build-calitp-map-utils.yml | 9 +++++++-- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-calitp-data-analysis.yml b/.github/workflows/build-calitp-data-analysis.yml index 0fd0a9b453..70683a57ac 100644 --- a/.github/workflows/build-calitp-data-analysis.yml +++ b/.github/workflows/build-calitp-data-analysis.yml @@ -36,12 +36,17 @@ jobs: poetry run mypy . poetry run pytest poetry build - # TODO: only run on main/version changes - if: ${{ github.ref != 'refs/heads/main' }} run: | poetry config repositories.test-pypi https://test.pypi.org/legacy/ poetry config pypi-token.test-pypi $POETRY_TEST_PYPI_TOKEN_PYPI - poetry publish -r test-pypi + poetry publish -r test-pypi --skip-existing working-directory: packages/calitp-data-analysis env: POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.TEST_PYPI_CALITP_DATA_ANALYSIS_TOKEN }} + - if: ${{ github.ref == 'refs/heads/main' }} + run: | + poetry publish --skip-existing + working-directory: packages/calitp-data-analysis + env: + POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_DATA_ANALYSIS_TOKEN }} diff --git a/.github/workflows/build-calitp-data-infra.yml b/.github/workflows/build-calitp-data-infra.yml index 8e9d9e038a..23f53ac466 100644 --- a/.github/workflows/build-calitp-data-infra.yml +++ b/.github/workflows/build-calitp-data-infra.yml @@ -37,12 +37,17 @@ jobs: CALITP_BUCKET__GTFS_DOWNLOAD_CONFIG: gs://this-does-not-exist CALITP_BUCKET__GTFS_SCHEDULE_RAW: gs://this-does-not-exist CALITP_BUCKET__GTFS_RT_RAW: gs://this-does-not-exist - # TODO: only run on main/version changes - if: ${{ github.ref != 'refs/heads/main' }} run: | poetry config repositories.test-pypi https://test.pypi.org/legacy/ poetry config pypi-token.test-pypi $POETRY_TEST_PYPI_TOKEN_PYPI - poetry publish -r test-pypi - working-directory: packages/calitp-data-infra + poetry publish -r test-pypi --skip-existing + working-directory: packages/calitp-data-analysis + env: + POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_DATA_INFRA_TOKEN }} + - if: ${{ github.ref == 'refs/heads/main' }} + run: | + poetry publish --skip-existing + working-directory: packages/calitp-data-analysis env: POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.TEST_PYPI_CALITP_DATA_INFRA_TOKEN }} diff --git a/.github/workflows/build-calitp-map-utils.yml b/.github/workflows/build-calitp-map-utils.yml index eb3fd420cc..577d930236 100644 --- a/.github/workflows/build-calitp-map-utils.yml +++ b/.github/workflows/build-calitp-map-utils.yml @@ -32,12 +32,17 @@ jobs: poetry run pytest --spec poetry run build working-directory: apps/maps - # TODO: only run on main/version changes - if: ${{ github.ref != 'refs/heads/main' }} run: | poetry config repositories.test-pypi https://test.pypi.org/legacy/ poetry config pypi-token.test-pypi $POETRY_TEST_PYPI_TOKEN_PYPI - poetry publish -r test-pypi + poetry publish -r test-pypi --skip-existing working-directory: apps/maps env: POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.TEST_PYPI_CALITP_MAP_UTILS_TOKEN }} + - if: ${{ github.ref == 'refs/heads/main' }} + run: | + poetry publish --skip-existing + working-directory: apps/maps + env: + POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_MAP_UTILS_TOKEN }} From b37783ec348b9283dcc03a7d6d712baacbf72ea7 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 15:54:26 -0400 Subject: [PATCH 09/17] use run defaults --- .github/workflows/build-calitp-data-analysis.yml | 8 ++++---- .github/workflows/build-calitp-data-infra.yml | 8 ++++---- .github/workflows/build-calitp-map-utils.yml | 9 ++++----- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-calitp-data-analysis.yml b/.github/workflows/build-calitp-data-analysis.yml index 70683a57ac..8c67c5ada2 100644 --- a/.github/workflows/build-calitp-data-analysis.yml +++ b/.github/workflows/build-calitp-data-analysis.yml @@ -19,6 +19,9 @@ concurrency: jobs: check_and_build: runs-on: ubuntu-latest + defaults: + run: + working-directory: packages/calitp-data-analysis steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -41,12 +44,9 @@ jobs: poetry config repositories.test-pypi https://test.pypi.org/legacy/ poetry config pypi-token.test-pypi $POETRY_TEST_PYPI_TOKEN_PYPI poetry publish -r test-pypi --skip-existing - working-directory: packages/calitp-data-analysis env: POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.TEST_PYPI_CALITP_DATA_ANALYSIS_TOKEN }} - if: ${{ github.ref == 'refs/heads/main' }} - run: | - poetry publish --skip-existing - working-directory: packages/calitp-data-analysis + run: poetry publish --skip-existing env: POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_DATA_ANALYSIS_TOKEN }} diff --git a/.github/workflows/build-calitp-data-infra.yml b/.github/workflows/build-calitp-data-infra.yml index 23f53ac466..2eb8971fb3 100644 --- a/.github/workflows/build-calitp-data-infra.yml +++ b/.github/workflows/build-calitp-data-infra.yml @@ -19,6 +19,9 @@ concurrency: jobs: check_and_build: runs-on: ubuntu-latest + defaults: + run: + working-directory: packages/calitp-data-analysis steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -42,12 +45,9 @@ jobs: poetry config repositories.test-pypi https://test.pypi.org/legacy/ poetry config pypi-token.test-pypi $POETRY_TEST_PYPI_TOKEN_PYPI poetry publish -r test-pypi --skip-existing - working-directory: packages/calitp-data-analysis env: POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_DATA_INFRA_TOKEN }} - if: ${{ github.ref == 'refs/heads/main' }} - run: | - poetry publish --skip-existing - working-directory: packages/calitp-data-analysis + run: poetry publish --skip-existing env: POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.TEST_PYPI_CALITP_DATA_INFRA_TOKEN }} diff --git a/.github/workflows/build-calitp-map-utils.yml b/.github/workflows/build-calitp-map-utils.yml index 577d930236..d6161b6748 100644 --- a/.github/workflows/build-calitp-map-utils.yml +++ b/.github/workflows/build-calitp-map-utils.yml @@ -19,6 +19,9 @@ concurrency: jobs: check_and_build: runs-on: ubuntu-latest + defaults: + run: + working-directory: apps/maps steps: - run: sudo apt-get install -y libgraphviz-dev graphviz-dev - uses: actions/checkout@v3 @@ -31,18 +34,14 @@ jobs: poetry run mypy calitp_map_utils poetry run pytest --spec poetry run build - working-directory: apps/maps - if: ${{ github.ref != 'refs/heads/main' }} run: | poetry config repositories.test-pypi https://test.pypi.org/legacy/ poetry config pypi-token.test-pypi $POETRY_TEST_PYPI_TOKEN_PYPI poetry publish -r test-pypi --skip-existing - working-directory: apps/maps env: POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.TEST_PYPI_CALITP_MAP_UTILS_TOKEN }} - if: ${{ github.ref == 'refs/heads/main' }} - run: | - poetry publish --skip-existing - working-directory: apps/maps + run: poetry publish --skip-existing env: POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_MAP_UTILS_TOKEN }} From 94d8c2534f19a8e86ea12bed392330860bdb9aab Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 15:56:50 -0400 Subject: [PATCH 10/17] remove old wds --- .github/workflows/build-calitp-data-analysis.yml | 1 - .github/workflows/build-calitp-data-infra.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/build-calitp-data-analysis.yml b/.github/workflows/build-calitp-data-analysis.yml index 8c67c5ada2..96d59a7575 100644 --- a/.github/workflows/build-calitp-data-analysis.yml +++ b/.github/workflows/build-calitp-data-analysis.yml @@ -32,7 +32,6 @@ jobs: service_account_key: ${{ secrets.GCP_SA_KEY }} export_default_credentials: true - name: Run checks - working-directory: packages/calitp-data-analysis run: | curl -sSL https://install.python-poetry.org | python - poetry install diff --git a/.github/workflows/build-calitp-data-infra.yml b/.github/workflows/build-calitp-data-infra.yml index 2eb8971fb3..cf8934f728 100644 --- a/.github/workflows/build-calitp-data-infra.yml +++ b/.github/workflows/build-calitp-data-infra.yml @@ -28,7 +28,6 @@ jobs: with: python-version: '3.9' - name: Run checks - working-directory: packages/calitp-data-infra run: | curl -sSL https://install.python-poetry.org | python - poetry install From 1fc82dda6945963bf8a855b903060b921e8571c5 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 16:02:48 -0400 Subject: [PATCH 11/17] checkout first so working directory exists --- .github/workflows/build-calitp-map-utils.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-calitp-map-utils.yml b/.github/workflows/build-calitp-map-utils.yml index d6161b6748..75804d908a 100644 --- a/.github/workflows/build-calitp-map-utils.yml +++ b/.github/workflows/build-calitp-map-utils.yml @@ -23,8 +23,8 @@ jobs: run: working-directory: apps/maps steps: - - run: sudo apt-get install -y libgraphviz-dev graphviz-dev - uses: actions/checkout@v3 + - run: sudo apt-get install -y libgraphviz-dev graphviz-dev - uses: actions/setup-python@v4 with: python-version: '3.9' From 48f0ecfe2ac65f02ebf0bf825e0bc6ab42b89e79 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 16:14:53 -0400 Subject: [PATCH 12/17] make mypy pass, remove unnecessary action install --- .github/workflows/build-calitp-map-utils.yml | 1 - apps/maps/calitp_map_utils/__init__.py | 37 ++++++++++++++------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-calitp-map-utils.yml b/.github/workflows/build-calitp-map-utils.yml index 75804d908a..690afbca96 100644 --- a/.github/workflows/build-calitp-map-utils.yml +++ b/.github/workflows/build-calitp-map-utils.yml @@ -24,7 +24,6 @@ jobs: working-directory: apps/maps steps: - uses: actions/checkout@v3 - - run: sudo apt-get install -y libgraphviz-dev graphviz-dev - uses: actions/setup-python@v4 with: python-version: '3.9' diff --git a/apps/maps/calitp_map_utils/__init__.py b/apps/maps/calitp_map_utils/__init__.py index ea8817d437..9ad3320dc3 100644 --- a/apps/maps/calitp_map_utils/__init__.py +++ b/apps/maps/calitp_map_utils/__init__.py @@ -3,12 +3,12 @@ import json import os from enum import Enum -from typing import Any, Dict, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union -import gcsfs +import gcsfs # type: ignore[import] import requests import typer -from furl import furl +from furl import furl # type: ignore[import] from geojson_pydantic import Feature, FeatureCollection, MultiPolygon, Point, Polygon from geojson_pydantic.geometries import Geometry from geojson_pydantic.types import Position @@ -32,6 +32,14 @@ class Tooltip(BaseModel): style: Optional[Dict[str, Any]] +# https://github.com/pydantic/pydantic/discussions/5528#discussioncomment-5663312 +if TYPE_CHECKING: + Color = Optional[list[int]] +else: + # we add alpha in the JS if only 3 colors are passed + Color = Optional[conlist(int, min_items=3, max_items=4)] + + class Speedmap(BaseModel): stop_id: Optional[str] stop_name: Optional[str] @@ -39,9 +47,8 @@ class Speedmap(BaseModel): tooltip: Optional[Tooltip] avg_mph: Optional[float] p20_mph: Optional[float] = Field(alias="_20p_mph") - # we add alpha in the JS if only 3 colors are passed - color: Optional[conlist(int, min_items=3, max_items=4)] - highlight_color: Optional[conlist(int, min_items=3, max_items=4)] + color: Color + highlight_color: Color @root_validator def some_identifier_exists(cls, values): @@ -89,7 +96,10 @@ def validate_geojson( collection = FeatureCollection(**d) - layer_type_class = LAYER_FEATURE_TYPES.get(layer_type, Feature[Geometry, Dict]) + if layer_type: + layer_type_class = LAYER_FEATURE_TYPES.get(layer_type, Feature[Geometry, Dict]) + else: + layer_type_class = Feature[Geometry, Dict] if verbose: typer.secho( @@ -117,13 +127,18 @@ class BasemapConfig(BaseModel): options: Dict[str, Any] +if TYPE_CHECKING: + LayerList = list[Layer] +else: + # this will not render in erdantic; needs to be List[Layer] but then pydantic2ts will not set min_items + LayerList = conlist(Layer, min_items=1) + + # Any positions in this are flipped from typical geojson # leaflet wants lat/lon class State(BaseModel): name: Optional[str] - layers: conlist( - Layer, min_items=1 - ) # this will not render in erdantic; needs to be List[Layer] but then pydantic2ts will not set min_items + layers: LayerList lat_lon: Optional[Position] zoom: Optional[int] bbox: Optional[Tuple[Position, Position]] @@ -151,7 +166,7 @@ def validate_layers( if data: validate_geojson(layer.url, layer.typ, verbose=verbose) - def iframe_url(self, host: str = None) -> str: + def iframe_url(self, host: Optional[str] = None) -> str: host = host or MAP_APP_URL if not host: raise RuntimeError( From b07608fd02838243efd8b36837ce241b22c4e872 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 16:16:48 -0400 Subject: [PATCH 13/17] actually we do need these for erdantic --- .github/workflows/build-calitp-map-utils.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-calitp-map-utils.yml b/.github/workflows/build-calitp-map-utils.yml index 690afbca96..75804d908a 100644 --- a/.github/workflows/build-calitp-map-utils.yml +++ b/.github/workflows/build-calitp-map-utils.yml @@ -24,6 +24,7 @@ jobs: working-directory: apps/maps steps: - uses: actions/checkout@v3 + - run: sudo apt-get install -y libgraphviz-dev graphviz-dev - uses: actions/setup-python@v4 with: python-version: '3.9' From 860dbef241a23397f6cb4f283809be8374adfabb Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 20:27:55 -0400 Subject: [PATCH 14/17] make calitp map utils pass tests --- apps/maps/tests/test_calitp_map_utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/maps/tests/test_calitp_map_utils.py b/apps/maps/tests/test_calitp_map_utils.py index 855c4751cf..7745a4055a 100644 --- a/apps/maps/tests/test_calitp_map_utils.py +++ b/apps/maps/tests/test_calitp_map_utils.py @@ -46,7 +46,7 @@ { "name": "LA Metro Bus Speed Maps AM Peak", "url": "https://storage.googleapis.com/calitp-map-tiles/metro_am.geojson.gz", - "type": "speedmaps", + "type": "speedmap", }, ], "bbox": [[34.1, -118.5], [33.9, -118]], @@ -86,9 +86,12 @@ def test_iframe_url_works_with_env_var(): with pytest.MonkeyPatch.context() as mp: mp.setattr("calitp_map_utils.MAP_APP_URL", "https://some.domain") - assert "some.domain" in State(**TEST_STATES[0]).iframe_url + assert "some.domain" in State(**TEST_STATES[0]).iframe_url() def test_iframe_url_missing_env_throws_exception(monkeypatch): - with pytest.raises(RuntimeError): - _ = State(**TEST_STATES[0]).iframe_url + with pytest.MonkeyPatch.context() as mp: + mp.setattr("calitp_map_utils.MAP_APP_URL", None) + + with pytest.raises(RuntimeError): + _ = State(**TEST_STATES[0]).iframe_url() From 6c8524f09f1bfaf8959f601b5b0da59ab0e7849b Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 20:40:15 -0400 Subject: [PATCH 15/17] more ci fixes --- .github/workflows/build-calitp-data-infra.yml | 4 ++++ .github/workflows/build-calitp-map-utils.yml | 2 ++ .github/workflows/build-dbt.yml | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-calitp-data-infra.yml b/.github/workflows/build-calitp-data-infra.yml index cf8934f728..999ee22199 100644 --- a/.github/workflows/build-calitp-data-infra.yml +++ b/.github/workflows/build-calitp-data-infra.yml @@ -27,6 +27,10 @@ jobs: - uses: actions/setup-python@v4 with: python-version: '3.9' + - uses: google-github-actions/setup-gcloud@v0 + with: + export_default_credentials: true + service_account_key: ${{ secrets.GCP_SA_KEY }} - name: Run checks run: | curl -sSL https://install.python-poetry.org | python - diff --git a/.github/workflows/build-calitp-map-utils.yml b/.github/workflows/build-calitp-map-utils.yml index 75804d908a..117196de66 100644 --- a/.github/workflows/build-calitp-map-utils.yml +++ b/.github/workflows/build-calitp-map-utils.yml @@ -34,6 +34,8 @@ jobs: poetry run mypy calitp_map_utils poetry run pytest --spec poetry run build + env: + CALITP_MAP_APP_URL: https://embeddable-maps.calitp.org/ - if: ${{ github.ref != 'refs/heads/main' }} run: | poetry config repositories.test-pypi https://test.pypi.org/legacy/ diff --git a/.github/workflows/build-dbt.yml b/.github/workflows/build-dbt.yml index a1f6168e49..3c2e678e15 100644 --- a/.github/workflows/build-dbt.yml +++ b/.github/workflows/build-dbt.yml @@ -44,8 +44,8 @@ jobs: export_default_credentials: true service_account_key: ${{ secrets.GCP_SA_KEY }} - name: Compile dbt project + working-directory: warehouse run: | - cd warehouse poetry install poetry run dbt deps poetry run dbt compile --target prod --full-refresh From 6733ff8f27f07e6a216a8deca2a938a9fd0e6266 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Wed, 2 Aug 2023 20:46:13 -0400 Subject: [PATCH 16/17] couple more --- .github/workflows/build-calitp-data-infra.yml | 4 ++-- .github/workflows/build-calitp-map-utils.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-calitp-data-infra.yml b/.github/workflows/build-calitp-data-infra.yml index 999ee22199..f3d809c4f5 100644 --- a/.github/workflows/build-calitp-data-infra.yml +++ b/.github/workflows/build-calitp-data-infra.yml @@ -49,8 +49,8 @@ jobs: poetry config pypi-token.test-pypi $POETRY_TEST_PYPI_TOKEN_PYPI poetry publish -r test-pypi --skip-existing env: - POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_DATA_INFRA_TOKEN }} + POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.TEST_PYPI_CALITP_DATA_INFRA_TOKEN }} - if: ${{ github.ref == 'refs/heads/main' }} run: poetry publish --skip-existing env: - POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.TEST_PYPI_CALITP_DATA_INFRA_TOKEN }} + POETRY_TEST_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_CALITP_DATA_INFRA_TOKEN }} diff --git a/.github/workflows/build-calitp-map-utils.yml b/.github/workflows/build-calitp-map-utils.yml index 117196de66..7a7716e06a 100644 --- a/.github/workflows/build-calitp-map-utils.yml +++ b/.github/workflows/build-calitp-map-utils.yml @@ -33,7 +33,7 @@ jobs: poetry install poetry run mypy calitp_map_utils poetry run pytest --spec - poetry run build + poetry build env: CALITP_MAP_APP_URL: https://embeddable-maps.calitp.org/ - if: ${{ github.ref != 'refs/heads/main' }} From 0743f7438a056de2cc6a9fc40202486546a7ec82 Mon Sep 17 00:00:00 2001 From: Andrew Vaccaro Date: Fri, 4 Aug 2023 14:40:16 -0400 Subject: [PATCH 17/17] fix calitp-data-infra ci wd --- .github/workflows/build-calitp-data-infra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-calitp-data-infra.yml b/.github/workflows/build-calitp-data-infra.yml index f3d809c4f5..d62994e2b1 100644 --- a/.github/workflows/build-calitp-data-infra.yml +++ b/.github/workflows/build-calitp-data-infra.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest defaults: run: - working-directory: packages/calitp-data-analysis + working-directory: packages/calitp-data-infra steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4