diff --git a/api/src/pcapi/scripts/install.py b/api/src/pcapi/scripts/install.py index 5ca39137515..177f89a0ca5 100644 --- a/api/src/pcapi/scripts/install.py +++ b/api/src/pcapi/scripts/install.py @@ -36,6 +36,7 @@ def install_commands(app: flask.Flask) -> None: "pcapi.scripts.ubble_archive_past_identifications", "pcapi.scripts.offer.fix_offer_data_titelive", "pcapi.scripts.offer.fix_product_gtl_id_titelive", + "pcapi.scripts.provider_migration.commands", "pcapi.utils.db", "pcapi.utils.human_ids", "pcapi.utils.secrets", diff --git a/api/src/pcapi/scripts/provider_migration/commands.py b/api/src/pcapi/scripts/provider_migration/commands.py new file mode 100644 index 00000000000..4ae6fe9b5e4 --- /dev/null +++ b/api/src/pcapi/scripts/provider_migration/commands.py @@ -0,0 +1,36 @@ +import logging + +import click + +from pcapi.scripts.provider_migration import migrate_venue_provider +from pcapi.utils.blueprint import Blueprint + + +logger = logging.getLogger(__name__) +blueprint = Blueprint(__name__, __name__) + + +@blueprint.cli.command("execute_scheduled_venue_provider_migration") +@click.option( + "-d", + "--date_and_hour", + help="Target date and hour in following format `%d/%m/%y_%HH` (for instance: `07/10/24_10H`)", +) +def execute_scheduled_venue_provider_migration(date_and_hour: str | None) -> None: + """ + Normally, `date_and_hour` param should not be used as `target_day` and `target_hour` + should be based on the actual time the command is called. + `date_and_hour` is here in case we need to rerun a migration we missed. + """ + if date_and_hour: + date_and_hour_tuple = date_and_hour.split("_") + if len(date_and_hour_tuple) != 2: + logger.error( + "Incorrect `date_and_hour` argument. Expected format:`%d/%m/%y_%HH` (for instance: `07/10/24_10H`)" + ) + return + target_day, targe_hour = date_and_hour_tuple + else: + target_day, targe_hour = migrate_venue_provider.get_migration_date_and_hour_keys() + + migrate_venue_provider.execute_scheduled_venue_provider_migration(target_day=target_day, target_hour=targe_hour) diff --git a/api/src/pcapi/scripts/provider_migration/main.py b/api/src/pcapi/scripts/provider_migration/main.py deleted file mode 100644 index 9c74976cb1d..00000000000 --- a/api/src/pcapi/scripts/provider_migration/main.py +++ /dev/null @@ -1,119 +0,0 @@ -import datetime -import logging - -from sqlalchemy.orm import joinedload - -from pcapi.core.history import api as history_api -from pcapi.core.history import models as history_models -from pcapi.core.offerers import models as offerers_models -from pcapi.core.providers import api as providers_api -from pcapi.core.providers import models as providers_models -from pcapi.core.users import models as users_models -from pcapi.flask_app import app -from pcapi.models import db -from pcapi.repository import transaction -from pcapi.scripts.provider_migration.data import VENUES_TO_MIGRATE_BY_DATE_AND_HOUR - - -logger = logging.getLogger(__name__) - - -def _delete_venue_provider( - venue_provider: providers_models.VenueProvider, - author: users_models.User, - comment: str, -) -> None: - venue_id = venue_provider.venueId - provider_id = venue_provider.provider.id - history_api.add_action( - history_models.ActionType.LINK_VENUE_PROVIDER_DELETED, - author=author, - venue=venue_provider.venue, - provider_id=venue_provider.providerId, - provider_name=venue_provider.provider.name, - comment=comment, - ) - db.session.delete(venue_provider) - logger.info( - "Deleted VenueProvider for venue %d", - venue_id, - extra={"venue_id": venue_id, "provider_id": provider_id}, - technical_message_id="offer.sync.deleted", - ) - - -def _migrate_venue_providers( - provider_id: int, - venues_ids: list[int], - migration_author_id: int, - comment: str, -) -> None: - """ - For each venue : - 1. Delete existing VenueProviders - 2. Insert new VenueProvider (for given provider) - """ - provider = providers_models.Provider.query.get(provider_id) - - if not provider: - logger.info(f"No provider was found for id {provider_id}") # pylint: disable=logging-fstring-interpolation - return - - venues = ( - offerers_models.Venue.query.filter(offerers_models.Venue.id.in_(venues_ids)) - .options(joinedload(offerers_models.Venue.venueProviders)) - .all() - ) - - if len(venues) != len(venues_ids): - found_venues_ids = [venue.id for venue in venues] - logger.info( # pylint: disable=logging-fstring-interpolation - f"Some venues don't exist {[id for id in venues_ids if id not in found_venues_ids]}" - ) - return - - user = users_models.User.query.get(migration_author_id) - - with transaction(): - for venue in venues: - logger.info(f"Handling venue <#{venue.id} - {venue.name}>") # pylint: disable=logging-fstring-interpolation - if len(venue.venueProviders): - for venue_provider in venue.venueProviders: - _delete_venue_provider(venue_provider, author=user, comment=comment) - providers_api.create_venue_provider(provider.id, venue.id, current_user=user) - - -def execute_scheduled_venue_provider_migration() -> None: - now = datetime.datetime.utcnow() - today = now.strftime("%d/%m/%y") - current_hour = now.hour - venues_to_migrate_today = VENUES_TO_MIGRATE_BY_DATE_AND_HOUR.get(today) - - if not venues_to_migrate_today: - logger.info("No venues to migrate today") - return - - if current_hour < 8: # i.e. < 10h in Paris Tz - migration_data = venues_to_migrate_today.get("10H") - else: - migration_data = venues_to_migrate_today.get("14H") - - if not migration_data: - logger.info("No venues to migrate at this time of day") - return - - provider_id = migration_data["target_provider_id"] - comment = migration_data["comment"] - venues_ids = migration_data["venues_ids"] - - _migrate_venue_providers( - provider_id=provider_id, # type: ignore[arg-type] - venues_ids=venues_ids, # type: ignore[arg-type] - migration_author_id=2568200, # id of our backend tech lead user - comment=comment, # type: ignore[arg-type] - ) - - -if __name__ == "__main__": - app.app_context().push() - execute_scheduled_venue_provider_migration() diff --git a/api/src/pcapi/scripts/provider_migration/migrate_venue_provider.py b/api/src/pcapi/scripts/provider_migration/migrate_venue_provider.py new file mode 100644 index 00000000000..3f0b355613d --- /dev/null +++ b/api/src/pcapi/scripts/provider_migration/migrate_venue_provider.py @@ -0,0 +1,192 @@ +import datetime +import logging + +import pydantic.v1 as pydantic_v1 +from sqlalchemy.orm import joinedload + +from pcapi.core.history import api as history_api +from pcapi.core.history import models as history_models +from pcapi.core.offerers import models as offerers_models +from pcapi.core.providers import api as providers_api +from pcapi.core.providers import models as providers_models +from pcapi.core.users import models as users_models +from pcapi.models import db +from pcapi.repository import transaction +from pcapi.scripts.provider_migration.data import VENUES_TO_MIGRATE_BY_DATE_AND_HOUR +from pcapi.utils.date import utc_datetime_to_department_timezone + + +logger = logging.getLogger(__name__) + + +BACKEND_USER_ID = 2568200 # id of our backend tech lead user + + +class MigrationData(pydantic_v1.BaseModel): + target_provider_id: int + venues_ids: list[int] + comment: str + + +def get_migration_date_and_hour_keys() -> tuple[str, str]: + """ + Format current datetime to tuple `target_day`, `target_hour`. + """ + now_utc = datetime.datetime.utcnow() + now_in_paris_tz = utc_datetime_to_department_timezone(now_utc, departement_code="75") + today = now_in_paris_tz.strftime("%d/%m/%y") + one_hour_from_now = now_in_paris_tz + datetime.timedelta(hours=1) + target_hour = one_hour_from_now.strftime("%HH") + return today, target_hour + + +def execute_scheduled_venue_provider_migration(target_day: str, target_hour: str) -> None: + """ + Execute VenueProvider migration scheduled for `target_day` at `target_hour` + + :target_day : Expected format `%d/%m/%y`, for instance: `07/10/24` + :target_hour : Expected format `%HH`, for instance: `10H` + """ + logging_message, migration_data = _retrieve_migration_data( + VENUES_TO_MIGRATE_BY_DATE_AND_HOUR, + target_day=target_day, + target_hour=target_hour, + ) + logger.info(logging_message) + + if migration_data: + # Check provider exists + provider = providers_models.Provider.query.get(migration_data.target_provider_id) + if not provider: + logger.error("[❌ MIGRATION ABORTED] No provider was found for id %s", migration_data.target_provider_id) + return + + # Check all venues exist + missing_venues_ids = _look_for_missing_venues(migration_data.venues_ids) + if missing_venues_ids: + logger.error( + "[❌ MIGRATION ABORTED] Some venues don't exist %s", ", ".join([str(id) for id in missing_venues_ids]) + ) + return + + _migrate_venue_providers( + provider_id=provider.id, + venues_ids=migration_data.venues_ids, + migration_author=users_models.User.query.get(BACKEND_USER_ID), + comment=migration_data.comment, + ) + + +def _retrieve_migration_data( + data: dict, target_day: str, target_hour: str +) -> tuple[str, None] | tuple[str, MigrationData]: + """ + Return a logging message and `MigrationData` object if there is data set fo given `target_day` + at given `target_hour` + + :data : A dict with the following structure + { + "%d/%m/%y": { + "%HH": { + "target_provider_id": int, + "venues_ids": list[int], + "comment": str, + } + } + # etc... + } + For instance : + { + "07/10/24": { + "10H": { + "target_provider_id": 42, + "venues_ids": [1, 2, 4], + "comment": "Migration to provider X", + }, + # etc... + } + } + :target_day : Expected format `%d/%m/%y`, for instance: `07/10/24` + :target_hour : Expected format `%HH`, for instance: `10H` + """ + venues_to_migrate_today = data.get(target_day) + if not venues_to_migrate_today: + return f"({target_day}) No venues to migrate today", None + + migration_data = venues_to_migrate_today.get(target_hour) + if not migration_data: + return f"({target_day} - {target_hour}) No venues to migrate at this time of day", None + + migration_data = MigrationData(**migration_data) + return ( + f"({target_day} - {target_hour}) {len(migration_data.venues_ids)} venues to migrate to provider #{migration_data.target_provider_id}", + migration_data, + ) + + +def _look_for_missing_venues(venues_ids: list[int]) -> list[int]: + """ + Return a list of venues ids not found in DB + """ + venues = offerers_models.Venue.query.filter(offerers_models.Venue.id.in_(venues_ids)).all() + + if len(venues) != len(venues_ids): + found_venues_ids = [venue.id for venue in venues] + return [id for id in venues_ids if id not in found_venues_ids] + + return [] + + +def _delete_venue_provider( + venue_provider: providers_models.VenueProvider, + author: users_models.User, + comment: str, +) -> None: + """ + Delete existing VenueProvider and add corresponding action in the ActionHistory table + """ + venue_id = venue_provider.venueId + provider_id = venue_provider.provider.id + history_api.add_action( + history_models.ActionType.LINK_VENUE_PROVIDER_DELETED, + author=author, + venue=venue_provider.venue, + provider_id=venue_provider.providerId, + provider_name=venue_provider.provider.name, + comment=comment, + ) + db.session.delete(venue_provider) + logger.info( + "Deleted VenueProvider for venue %d", + venue_id, + extra={"venue_id": venue_id, "provider_id": provider_id}, + technical_message_id="offer.sync.deleted", + ) + + +def _migrate_venue_providers( + provider_id: int, + venues_ids: list[int], + migration_author: users_models.User, + comment: str, +) -> None: + """ + Migrate a list of venues to a new provider. + + Migrating a venue is a two steps process: + 1. Delete existing VenueProviders (but do not deactivate existing offers) + 2. Insert new VenueProvider (for given provider) + """ + venues = ( + offerers_models.Venue.query.filter(offerers_models.Venue.id.in_(venues_ids)) + .options(joinedload(offerers_models.Venue.venueProviders)) + .all() + ) + + with transaction(): + for venue in venues: + logger.info("Handling venue <#%s - %s>", venue.id, venue.name) + if len(venue.venueProviders): + for venue_provider in venue.venueProviders: + _delete_venue_provider(venue_provider, author=migration_author, comment=comment) + providers_api.create_venue_provider(provider_id, venue.id, current_user=migration_author) diff --git a/api/tests/scripts/provider_migration/migrate_venue_provider_test.py b/api/tests/scripts/provider_migration/migrate_venue_provider_test.py new file mode 100644 index 00000000000..4e95152ac6f --- /dev/null +++ b/api/tests/scripts/provider_migration/migrate_venue_provider_test.py @@ -0,0 +1,202 @@ +import logging +from unittest import mock + +import pydantic.v1 as pydantic_v1 +import pytest +import time_machine + +from pcapi.core.offerers import factories as offerers_factories +from pcapi.core.providers import factories as providers_factories +from pcapi.core.providers import repository as providers_repository +from pcapi.core.users import factories as users_factories +from pcapi.scripts.provider_migration import migrate_venue_provider + + +FAKE_VENUES_TO_MIGRATE_BY_DATE_AND_HOUR = { + "07/10/24": { + "10H": { + "target_provider_id": 12, + "venues_ids": [1, 3, 4], + "comment": "Smart comment such as `Y-a pas à dire, les cotelettes c'est savoureux`", + }, + "12H": { + "target_provider_id": 14, + "venues_ids": [2, 5], + "comment": "Dumb comment such as `J'adore le salsifi`", + }, + } +} + + +class GetMigrationDateAndHourKeysTest: + @pytest.mark.parametrize( + "utc_datetime,expected_target_date,expected_target_hour", + [ + ("2024-10-07 18:29", "07/10/24", "21H"), + ("2024-10-10 12:00", "10/10/24", "15H"), + ("2024-10-10 11:59", "10/10/24", "14H"), + ], + ) + def test_should_return_tuple(self, utc_datetime, expected_target_date, expected_target_hour): + with time_machine.travel(utc_datetime): + target_day, target_hour = migrate_venue_provider.get_migration_date_and_hour_keys() + assert target_day == expected_target_date + assert target_hour == expected_target_hour + + +class RetrieveMigrationDataTest: + @pytest.mark.parametrize( + "data,target_day,target_hour,expected_logging_message,expected_migration_data", + [ + ({}, "07/10/24", "10H", "(07/10/24) No venues to migrate today", None), + ( + {"07/10/24": {"12H": {}}}, + "07/10/24", + "10H", + "(07/10/24 - 10H) No venues to migrate at this time of day", + None, + ), + ( + FAKE_VENUES_TO_MIGRATE_BY_DATE_AND_HOUR, + "07/10/24", + "10H", + "(07/10/24 - 10H) 3 venues to migrate to provider #12", + migrate_venue_provider.MigrationData( + target_provider_id=12, + venues_ids=[1, 3, 4], + comment="Smart comment such as `Y-a pas à dire, les cotelettes c'est savoureux`", + ), + ), + ], + ) + def test_should_return_logging_message_and_migration_data( + self, data, target_day, target_hour, expected_logging_message, expected_migration_data + ): + logging_message, migration_data = migrate_venue_provider._retrieve_migration_data( + data, + target_day=target_day, + target_hour=target_hour, + ) + assert logging_message == expected_logging_message + assert migration_data == expected_migration_data + + def test_should_raise_an_error(self): + with pytest.raises(pydantic_v1.ValidationError): + migrate_venue_provider._retrieve_migration_data( + { + "07/10/24": { + "10H": { + "target_provider_id": "heyyy!", + "venues_ids": ["coucou"], + "comment": None, + }, + } + }, + target_day="07/10/24", + target_hour="10H", + ) + + +@pytest.mark.usefixtures("db_session") +@mock.patch( + "pcapi.scripts.provider_migration.migrate_venue_provider.VENUES_TO_MIGRATE_BY_DATE_AND_HOUR", + FAKE_VENUES_TO_MIGRATE_BY_DATE_AND_HOUR, +) +class ExecuteScheduledVenueProviderMigrationTest: + @mock.patch("pcapi.scripts.provider_migration.migrate_venue_provider._migrate_venue_providers") + @pytest.mark.parametrize( + "target_day,target_hour,expected_logging_message", + [ + ("06/10/24", "10H", "(06/10/24) No venues to migrate today"), + ("07/10/24", "11H", "(07/10/24 - 11H) No venues to migrate at this time of day"), + ], + ) + def test_should_exit_because_not_data( + self, + _mock_migrate_venue_providers, + caplog, + target_day, + target_hour, + expected_logging_message, + ): + with caplog.at_level(logging.INFO): + migrate_venue_provider.execute_scheduled_venue_provider_migration(target_day, target_hour) + assert caplog.messages[0] == expected_logging_message + _mock_migrate_venue_providers.assert_not_called() + + @mock.patch("pcapi.scripts.provider_migration.migrate_venue_provider._migrate_venue_providers") + def test_should_exit_because_provider_not_found(self, _mock_migrate_venue_providers, caplog): + with caplog.at_level(logging.ERROR): + migrate_venue_provider.execute_scheduled_venue_provider_migration("07/10/24", "10H") + assert caplog.messages[0] == "[❌ MIGRATION ABORTED] No provider was found for id 12" + _mock_migrate_venue_providers.assert_not_called() + + @mock.patch("pcapi.scripts.provider_migration.migrate_venue_provider._migrate_venue_providers") + def test_should_exit_because_missing_venues(self, _mock_migrate_venue_providers, caplog): + providers_factories.PublicApiProviderFactory(id=12) + offerers_factories.VenueFactory(id=3) + with caplog.at_level(logging.ERROR): + migrate_venue_provider.execute_scheduled_venue_provider_migration("07/10/24", "10H") + assert caplog.messages[0] == "[❌ MIGRATION ABORTED] Some venues don't exist 1, 4" + _mock_migrate_venue_providers.assert_not_called() + + @mock.patch("pcapi.scripts.provider_migration.migrate_venue_provider._migrate_venue_providers") + def test_should_call_migrate_venue_provider(self, _mock_migrate_venue_providers, caplog): + providers_factories.PublicApiProviderFactory(id=12) + offerers_factories.VenueFactory(id=1) + offerers_factories.VenueFactory(id=3) + offerers_factories.VenueFactory(id=4) + user = users_factories.UserFactory(id=2568200) + with caplog.at_level(logging.INFO): + migrate_venue_provider.execute_scheduled_venue_provider_migration("07/10/24", "10H") + assert caplog.messages[0] == "(07/10/24 - 10H) 3 venues to migrate to provider #12" + _mock_migrate_venue_providers.assert_called_with( + provider_id=12, + venues_ids=[1, 3, 4], + migration_author=user, + comment="Smart comment such as `Y-a pas à dire, les cotelettes c'est savoureux`", + ) + + def test_full_process(self, caplog): + old_provider = providers_factories.ProviderFactory() + very_old_provider = providers_factories.ProviderFactory() + provider = providers_factories.PublicApiProviderFactory(id=12) + providers_factories.OffererProviderFactory(provider=provider) + venue_1 = offerers_factories.VenueFactory(id=1) + providers_factories.VenueProviderFactory(venue=venue_1, provider=old_provider) + providers_factories.VenueProviderFactory(venue=venue_1, provider=very_old_provider) + venue_3 = offerers_factories.VenueFactory(id=3) + providers_factories.VenueProviderFactory(venue=venue_3, provider=old_provider) + venue_4 = offerers_factories.VenueFactory(id=4) + users_factories.UserFactory(id=2568200) + with caplog.at_level(logging.INFO): + migrate_venue_provider.execute_scheduled_venue_provider_migration("07/10/24", "10H") + # Test on logging messages + assert caplog.messages[0] == "(07/10/24 - 10H) 3 venues to migrate to provider #12" + # Venue #1 + assert caplog.messages[1] == f"Handling venue <#1 - {venue_1.name}>" + assert caplog.messages[2] == f"Deleted VenueProvider for venue {venue_1.id}" + assert caplog.records[2].extra == {"venue_id": venue_1.id, "provider_id": old_provider.id} + assert caplog.messages[3] == f"Deleted VenueProvider for venue {venue_1.id}" + assert caplog.records[3].extra == {"venue_id": venue_1.id, "provider_id": very_old_provider.id} + assert caplog.messages[4] == f"La synchronisation d'offre a été activée" + # Venue #3 + assert caplog.messages[5] == f"Handling venue <#3 - {venue_3.name}>" + assert caplog.records[6].extra == {"venue_id": venue_3.id, "provider_id": old_provider.id} + assert caplog.messages[6] == f"Deleted VenueProvider for venue {venue_3.id}" + assert caplog.messages[7] == f"La synchronisation d'offre a été activée" + # Venue #4 + assert caplog.messages[8] == f"Handling venue <#4 - {venue_4.name}>" + assert caplog.messages[9] == f"La synchronisation d'offre a été activée" + + # Test on data + # Old venue_provider should have been removed + assert not providers_repository.get_venue_provider_by_venue_and_provider_ids(venue_1.id, old_provider.id) + assert not providers_repository.get_venue_provider_by_venue_and_provider_ids( + venue_1.id, very_old_provider.id + ) + assert not providers_repository.get_venue_provider_by_venue_and_provider_ids(venue_3.id, old_provider.id) + # New venue_provider should have been added + assert providers_repository.get_venue_provider_by_venue_and_provider_ids(venue_1.id, provider.id).isActive + assert providers_repository.get_venue_provider_by_venue_and_provider_ids(venue_1.id, provider.id).isActive + assert providers_repository.get_venue_provider_by_venue_and_provider_ids(venue_3.id, provider.id).isActive