From 767b607fa656a8e78705b4dfae2420b8eef9b2d0 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 6 May 2024 19:46:28 +0530 Subject: [PATCH 1/9] fix: migration executor logic on failure --- .../compose_project/compose_project.py | 3 +- .../migration_manager/migration_base.py | 6 +- .../migration_manager/migration_executor.py | 56 ++++++++++++------- .../migrations/migrate_0_13_0.py | 5 ++ frappe_manager/utils/helpers.py | 6 +- 5 files changed, 51 insertions(+), 25 deletions(-) diff --git a/frappe_manager/compose_project/compose_project.py b/frappe_manager/compose_project/compose_project.py index 17a8bcee..dad59477 100644 --- a/frappe_manager/compose_project/compose_project.py +++ b/frappe_manager/compose_project/compose_project.py @@ -3,6 +3,7 @@ from rich.text import Text from frappe_manager.compose_manager.ComposeFile import ComposeFile from frappe_manager.compose_project.exceptions import ( + DockerComposeProjectFailedToPullImagesError, DockerComposeProjectFailedToRemoveError, DockerComposeProjectFailedToRestartError, DockerComposeProjectFailedToStartError, @@ -77,7 +78,7 @@ def pull_images(self): if self.quiet: richprint.live_lines(output, padding=(0, 0, 0, 2)) except DockerException as e: - raise DockerComposeProjectFailedToRemoveError( + raise DockerComposeProjectFailedToPullImagesError( self.compose_file_manager.compose_path, self.compose_file_manager.get_services_list() ) diff --git a/frappe_manager/migration_manager/migration_base.py b/frappe_manager/migration_manager/migration_base.py index 093f3b43..d79d54a7 100644 --- a/frappe_manager/migration_manager/migration_base.py +++ b/frappe_manager/migration_manager/migration_base.py @@ -4,23 +4,23 @@ from frappe_manager.migration_manager.version import Version from frappe_manager.logger import log + @runtime_checkable class MigrationBase(Protocol): version: Version = Version("0.0.0") skip: bool = False migration_executor = None backup_manager: BackupManager # Declare the backup_manager variable - logger : Logger + logger: Logger def init(self): self.backup_manager = BackupManager(str(self.version)) # Assign the value to backup_manager - self.logger = log.get_logger() + self.logger = log.get_logger() def set_migration_executor(self, migration_executor): self.migration_executor = migration_executor def get_rollback_version(self): - return self.version def up(self): diff --git a/frappe_manager/migration_manager/migration_executor.py b/frappe_manager/migration_manager/migration_executor.py index 9048a548..de73d186 100644 --- a/frappe_manager/migration_manager/migration_executor.py +++ b/frappe_manager/migration_manager/migration_executor.py @@ -78,7 +78,7 @@ def execute(self): richprint.print("Pending Migrations...", emoji_code=':counterclockwise_arrows_button:') for migration in self.migrations: - richprint.print(f"[bold]v{migration.version}[/bold]",emoji_code=':package:') + richprint.print(f"[bold]v{migration.version}[/bold]", emoji_code=':package:') richprint.print("This process may take a while.", emoji_code="\n:hourglass_not_done:") @@ -105,8 +105,10 @@ def execute(self): rollback = False archive = False + exception_migration_in_bench_occured = False try: # run all the migrations + prev_migration = None for migration in self.migrations: richprint.change_head(f"Running migration introduced in v{migration.version}") self.logger.info(f"[{migration.version}] : Migration starting") @@ -115,13 +117,16 @@ def execute(self): migration.up() - if not self.rollback_version > migration.version: - self.rollback_version = migration.get_rollback_version() + prev_migration = migration + if not exception_migration_in_bench_occured: + self.rollback_version = prev_migration.get_rollback_version() except MigrationExceptionInBench as e: - captured_output = capture_and_format_exception() + captured_output = capture_and_format_exception(traceback_max_frames=0) self.logger.error(f"[{migration.version}] : Migration Failed\n{captured_output}") + if migration.version < self.migrations[-1].version: + exception_migration_in_bench_occured = True continue raise e @@ -130,37 +135,46 @@ def execute(self): self.logger.error(f"[{migration.version}] : Migration Failed\n{captured_output}") raise e + if exception_migration_in_bench_occured: + raise MigrationExceptionInBench('') + except MigrationExceptionInBench as e: if self.migrate_benches: print_head = True for bench, bench_status in self.migrate_benches.items(): if not bench_status["exception"]: if print_head: - richprint.stdout.rule('[bold][red]Migration Passed Benches[/red][bold]',style='green') - print_head = False + richprint.stdout.rule('[bold][red]Migration Passed Benches[/red][bold]', style='green') + print_head = False - richprint.print(f"[red]Bench[/red]: {bench}",emoji_code=':construction:') + richprint.print(f"[green]Bench[/green]: {bench}", emoji_code=':construction:') if not print_head: richprint.stdout.rule(style='red') print_head = True + for bench, bench_status in self.migrate_benches.items(): if bench_status["exception"]: if print_head: - richprint.stdout.rule(':police_car_light: [bold][red]Migration Failed Benches[/red][bold] :police_car_light:',style='red') - print_head = False + richprint.stdout.rule( + ':police_car_light: [bold][red]Migration Failed Benches[/red][bold] :police_car_light:', + style='red', + ) + print_head = False - richprint.error(f"[red]Bench[/red]: {bench}",emoji_code=':construction:') + richprint.error(f"[red]Bench[/red]: {bench}", emoji_code=':construction:') richprint.error( - f"[red]Failed Migration Version[/red]: {bench_status['last_migration_version']}",emoji_code=':package:' + f"[red]Failed Migration Version[/red]: {bench_status['last_migration_version']}", + emoji_code=':package:', ) richprint.error( - f"[red]Exception[/red]: {type(bench_status['exception']).__name__}",emoji_code=':stop_sign:' + f"[red]Exception[/red]: {type(bench_status['exception']).__name__}", + emoji_code=':stop_sign:', ) - richprint.stdout.print(Padding(Text(text=str(bench_status['exception'])),(0,0,0,3))) + richprint.stdout.print(Padding(Text(text=str(bench_status['exception'])), (0, 0, 0, 3))) richprint.print(f"For error specifics, refer to {CLI_DIR}/logs/fm.log", emoji_code=':page_facing_up:') @@ -168,9 +182,9 @@ def execute(self): richprint.stdout.rule(style='red') archive_msg = [ - 'Please select one of the available options after migrations failure :\n', - f"[blue]yes[/blue] Archive failed benches : Benches that have failed will be rolled back and stored in '{CLI_SITES_ARCHIVE}'.", - '[blue]no[/blue] Revert migration : Revert the entire fm enrionment to the previous fm version before migration.', + 'Available options after migrations failure :', + f"[blue]\[yes][/blue] Archive failed benches : Benches that have failed will be rolled back to there previous state and stored in '{CLI_SITES_ARCHIVE}'.", + '[blue]\[no][/blue] Revert migration : Revert the entire FM environment to the previous FM version before migration.', '\nDo you wish to archive all benches that failed during migration ?', ] archive = richprint.prompt_ask(prompt="\n".join(archive_msg), choices=["yes", "no"]) @@ -179,7 +193,7 @@ def execute(self): rollback = True except Exception as e: - richprint.print(f"Migration failed: {e}") + richprint.error(f"[red]Migration failed[red] : {e}") rollback = True if archive == "yes": @@ -199,14 +213,18 @@ def execute(self): f"Installing [bold][blue]Frappe-Manager[/blue][/bold] version: v{str(self.rollback_version.version)}" ) install_package("frappe-manager", str(self.rollback_version.version)) - richprint.exit("Rollback complete.",emoji_code=':back:') + richprint.exit("Rollback complete.", emoji_code=':back:') self.fm_config_manager.version = self.current_version self.fm_config_manager.export_to_toml() return True def set_bench_data( - self, bench: MigrationBench, exception=None, migration_version: Optional[Version] = None, traceback_str: Optional[str] = None + self, + bench: MigrationBench, + exception=None, + migration_version: Optional[Version] = None, + traceback_str: Optional[str] = None, ): self.migrate_benches[bench.name] = { "object": bench, diff --git a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py index 02e75a51..6e9b34da 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py @@ -165,6 +165,11 @@ def undo_bench_migrate(self, bench: MigrationBench): if backup.bench == bench.name: self.backup_manager.restore(backup, force=True) + admin_tools_compose_path = bench.path / 'docker-compose.admin-tools.yml' + + if admin_tools_compose_path.exists(): + admin_tools_compose_path.unlink() + self.logger.info(f"Undo successfull for bench: {bench.name}") def migrate_bench_compose(self, bench: MigrationBench): diff --git a/frappe_manager/utils/helpers.py b/frappe_manager/utils/helpers.py index 6f3e658d..28da2adb 100644 --- a/frappe_manager/utils/helpers.py +++ b/frappe_manager/utils/helpers.py @@ -405,13 +405,15 @@ def rich_traceback_to_string(traceback: Traceback) -> str: return captured_str -def capture_and_format_exception() -> str: +def capture_and_format_exception(traceback_max_frames: int = 100) -> str: """Capture the current exception and return a formatted traceback string.""" exc_type, exc_value, exc_traceback = sys.exc_info() # Capture current exception info # Create a Traceback object with rich formatting # - traceback = Traceback.from_exception(exc_type, exc_value, exc_traceback, show_locals=True) + traceback = Traceback.from_exception( + exc_type, exc_value, exc_traceback, show_locals=True, max_frames=traceback_max_frames + ) # Convert the Traceback object to a formatted string formatted_traceback = rich_traceback_to_string(traceback) From fbba1c433d0dc178d9747766a8f6d20e34b6c82d Mon Sep 17 00:00:00 2001 From: Xieyt Date: Mon, 6 May 2024 20:02:05 +0530 Subject: [PATCH 2/9] fix: fm update run only when site running --- frappe_manager/commands.py | 4 ++++ frappe_manager/site_manager/site_exceptions.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index 0de09cfc..1943713f 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -1,4 +1,5 @@ from pathlib import Path +from frappe_manager.site_manager.site_exceptions import BenchNotRunning from frappe_manager.utils.site import pull_docker_images import typer import os @@ -499,6 +500,9 @@ def update( restart_required = False bench_config_save = False + if not bench.compose_project.running: + raise BenchNotRunning(bench_name=bench.name) + if developer_mode: if developer_mode == EnableDisableOptionsEnum.enable: bench.bench_config.developer_mode = True diff --git a/frappe_manager/site_manager/site_exceptions.py b/frappe_manager/site_manager/site_exceptions.py index 098aa76c..75f4de36 100644 --- a/frappe_manager/site_manager/site_exceptions.py +++ b/frappe_manager/site_manager/site_exceptions.py @@ -159,7 +159,7 @@ def __init__(self, bench_name, service_name, message="Attach to {} service conta class BenchNotRunning(BenchException): - def __init__(self, bench_name, message="Services not running."): + def __init__(self, bench_name, message="Bench services not running."): self.bench_name = bench_name self.message = message super().__init__(self.bench_name, self.message) From 893ee67969d1499a7ff4d245706ad074092a5a58 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 7 May 2024 19:40:41 +0530 Subject: [PATCH 3/9] Update options msg when migration fails --- frappe_manager/migration_manager/migration_executor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frappe_manager/migration_manager/migration_executor.py b/frappe_manager/migration_manager/migration_executor.py index de73d186..9ddc8533 100644 --- a/frappe_manager/migration_manager/migration_executor.py +++ b/frappe_manager/migration_manager/migration_executor.py @@ -183,8 +183,8 @@ def execute(self): archive_msg = [ 'Available options after migrations failure :', - f"[blue]\[yes][/blue] Archive failed benches : Benches that have failed will be rolled back to there previous state and stored in '{CLI_SITES_ARCHIVE}'.", - '[blue]\[no][/blue] Revert migration : Revert the entire FM environment to the previous FM version before migration.', + f"[blue]\[yes][/blue] Archive failed benches : Benches that have failed will be rolled back to there last successfully completed migration version and stored in '{CLI_SITES_ARCHIVE}'.", + '[blue]\[no][/blue] Revert migration : Restore the FM CLI and FM environment to the last successfully completed migration version for all benches.', '\nDo you wish to archive all benches that failed during migration ?', ] archive = richprint.prompt_ask(prompt="\n".join(archive_msg), choices=["yes", "no"]) From 3ebc5c14105dd0b9f52742d004d8a0b6775611fa Mon Sep 17 00:00:00 2001 From: Xieyt Date: Tue, 7 May 2024 19:52:53 +0530 Subject: [PATCH 4/9] bump: v0.13.3 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1f2dc15d..73795de9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "frappe-manager" -version = "0.13.2" +version = "0.13.3" license = "MIT" repository = "https://github.com/rtcamp/frappe-manager" description = "A CLI tool based on Docker Compose to easily manage Frappe based projects. As of now, only suitable for development in local machines running on Mac and Linux based OS." From e00fc6bb7fc927adfa1f8f7544b6f1d2bc39416a Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 10 May 2024 04:34:35 +0530 Subject: [PATCH 5/9] refactor migrations --- .../migration_manager/backup_manager.py | 53 +++--- .../migration_manager/migration_base.py | 180 +++++++++++++++++- .../migration_manager/migration_executor.py | 23 +-- .../migrations/migrate_0_10_0.py | 107 +++++------ .../migrations/migrate_0_11_0.py | 116 ++--------- .../migrations/migrate_0_12_0.py | 119 +++--------- .../migrations/migrate_0_13_0.py | 126 +++--------- .../migrations/migrate_0_13_1.py | 40 ++-- .../migrations/migrate_0_9_0.py | 43 ++--- .../database_service_manager.py | 120 ++++++++---- 10 files changed, 449 insertions(+), 478 deletions(-) diff --git a/frappe_manager/migration_manager/backup_manager.py b/frappe_manager/migration_manager/backup_manager.py index 0c3d109e..e0301c3e 100644 --- a/frappe_manager/migration_manager/backup_manager.py +++ b/frappe_manager/migration_manager/backup_manager.py @@ -1,21 +1,23 @@ import shutil from datetime import datetime from pathlib import Path -from frappe_manager import CLI_DIR +from frappe_manager import CLI_BENCHES_DIRECTORY, CLI_DIR from dataclasses import dataclass from typing import Optional from frappe_manager.logger import log random_strings = [] + @dataclass -class BackupData(): +class BackupData: src: Path dest: Path bench: Optional[str] = None + bench_path: Optional[Path] = None + prefix_timestamp: bool = False allow_restore: bool = True _is_restored: bool = False - prefix_timestamp = False _prefix_length: int = 5 @property @@ -27,7 +29,6 @@ def is_restored(self, v: bool) -> None: self._is_restored = v def __post_init__(self): - file_name = self.dest.name if self.prefix_timestamp: @@ -43,43 +44,49 @@ def __post_init__(self): break self.real_dest = self.dest.parent - - if self.bench: - self.real_dest = self.real_dest / self.bench - self.real_dest: Path = self.real_dest / file_name def exists(self): return self.dest.exists() -CLI_MIGARATIONS_DIR = CLI_DIR / 'migrations'/ f"{datetime.now().strftime('%d-%b-%y--%H-%M-%S')}" / 'backups' +current_migration_timestamp = f"{datetime.now().strftime('%d-%b-%y--%H-%M-%S')}" +CLI_MIGARATIONS_DIR = CLI_DIR / 'backups' / 'migrations' / current_migration_timestamp -class BackupManager(): - def __init__(self,name, base_dir: Path = CLI_MIGARATIONS_DIR): - self.root_backup_dir = base_dir - self.backup_dir = self.root_backup_dir / name +class BackupManager: + def __init__(self, name, benches_dir=CLI_BENCHES_DIRECTORY, backup_dir: Path = CLI_MIGARATIONS_DIR): + self.name = name + self.root_backup_dir: Path = backup_dir + self.benches_dir: Path = benches_dir + self.backup_dir: Path = self.root_backup_dir / self.name + self.bench_backup_dir: Path = Path('backups') / 'migrations' / current_migration_timestamp self.backups = [] self.logger = log.get_logger() # create backup dir if not exists - self.backup_dir.mkdir(parents=True,exist_ok=True) - - def backup(self, src: Path, dest: Optional[Path] = None, bench_name: Optional[str] = None, allow_restore: bool = True ): - + self.backup_dir.mkdir(parents=True, exist_ok=True) + + def backup( + self, + src: Path, + dest: Optional[Path] = None, + bench_name: Optional[str] = None, + allow_restore: bool = True, + ): if not src.exists(): return None if not dest: dest = self.backup_dir / src.name - backup_data = BackupData(src, dest, allow_restore=allow_restore) + if bench_name: + dest: Path = self.benches_dir / bench_name / self.bench_backup_dir / self.name / src.name + + backup_data = BackupData(src, dest, bench=bench_name, allow_restore=allow_restore) - if bench_name: - backup_data = BackupData(src, dest,bench=bench_name) - if not backup_data.real_dest.parent.exists(): - backup_data.real_dest.parent.mkdir(parents=True,exist_ok=True) + if not backup_data.real_dest.parent.exists(): + backup_data.real_dest.parent.mkdir(parents=True, exist_ok=True) self.logger.debug(f"Backup: {backup_data.src} => {backup_data.real_dest} ") @@ -94,7 +101,7 @@ def backup(self, src: Path, dest: Optional[Path] = None, bench_name: Optional[st return backup_data - def restore(self, backup_data, force = False): + def restore(self, backup_data, force=False): """ Restore a file from a backup. """ diff --git a/frappe_manager/migration_manager/migration_base.py b/frappe_manager/migration_manager/migration_base.py index d79d54a7..a4fdf73d 100644 --- a/frappe_manager/migration_manager/migration_base.py +++ b/frappe_manager/migration_manager/migration_base.py @@ -1,21 +1,33 @@ +from abc import ABC from logging import Logger -from typing import Protocol, runtime_checkable +from pathlib import Path +from frappe_manager import CLI_DIR from frappe_manager.migration_manager.backup_manager import BackupManager +from frappe_manager.migration_manager.migration_exections import MigrationExceptionInBench +from frappe_manager.migration_manager.migration_helpers import ( + MigrationBench, + MigrationBenches, + MigrationServicesManager, +) from frappe_manager.migration_manager.version import Version from frappe_manager.logger import log +from frappe_manager.services_manager.database_service_manager import DatabaseServerServiceInfo, MariaDBManager +from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.utils.helpers import capture_and_format_exception -@runtime_checkable -class MigrationBase(Protocol): +# @runtime_checkable +class MigrationBase(ABC): version: Version = Version("0.0.0") + benches_dir: Path = CLI_DIR / "sites" skip: bool = False migration_executor = None - backup_manager: BackupManager # Declare the backup_manager variable - logger: Logger + logger: Logger = log.get_logger() def init(self): - self.backup_manager = BackupManager(str(self.version)) # Assign the value to backup_manager - self.logger = log.get_logger() + self.backup_manager = BackupManager(str(self.version), self.benches_dir) + self.benches_manager = MigrationBenches(self.benches_dir) + self.services_manager: MigrationServicesManager = MigrationServicesManager(services_path=CLI_DIR / 'services') def set_migration_executor(self, migration_executor): self.migration_executor = migration_executor @@ -24,7 +36,159 @@ def get_rollback_version(self): return self.version def up(self): - pass + if self.skip: + return True + + richprint.stdout.rule(f':package: [bold][blue]v{str(self.version)}[/blue][bold]') + self.logger.info(f"v{str(self.version)}: Started") + self.logger.info("-" * 40) + + self.init() + self.services_basic_backup() + self.migrate_services() + self.migrate_benches() + + self.logger.info("-" * 40) def down(self): + richprint.change_head(f"Working on v{str(self.version)} rollback.") + self.logger.info("-" * 40) + + # undo each bench + for bench_name, bench_data in self.migration_executor.migrate_benches.items(): + if not bench_data['exception']: + self.undo_bench_migrate(bench_data['object']) + + for backup in self.backup_manager.backups: + self.backup_manager.restore(backup, force=True) + # richprint.print(f'Restored {backup.bench}'s {backup.src.name}.') + + self.undo_services_migrate() + + richprint.print(f"[bold]v{str(self.version)}[/bold] rollback successfull.") + self.logger.info("-" * 40) + + def services_basic_backup(self): + if not self.services_manager.compose_project.compose_file_manager.exists(): + raise MigrationExceptionInBench( + f"Services compose at {self.services_manager.compose_project.compose_file_manager} not found." + ) + self.backup_manager.backup(self.services_manager.compose_project.compose_file_manager.compose_path) + + def migrate_services(self): + pass + + def undo_services_migrate(self): + pass + + def migrate_benches(self): + main_error = False + + # migrate each bench + for bench_name, bench_path in self.benches_manager.get_all_benches().items(): + bench = MigrationBench(name=bench_name, path=bench_path.parent) + + if bench.name in self.migration_executor.migrate_benches.keys(): + bench_info = self.migration_executor.migrate_benches[bench.name] + if bench_info['exception']: + richprint.print(f"Skipping migration for failed bench [blue]{bench.name}[/blue].") + main_error = True + continue + + self.migration_executor.set_bench_data(bench, migration_version=self.version) + try: + self.bench_basic_backup(bench) + self.migrate_bench(bench) + except Exception as e: + traceback_str = capture_and_format_exception() + self.logger.error(f"{bench.name} [ EXCEPTION TRACEBACK ]:\n {traceback_str}") + richprint.update_live() + main_error = True + self.migration_executor.set_bench_data(bench, e, self.version) + + # restore all backup files + for backup in self.backup_manager.backups: + if backup.bench == bench.name: + self.backup_manager.restore(backup, force=True) + + self.undo_bench_migrate(bench) + self.logger.info(f'Undo successfull for bench: {bench.name}') + bench.compose_project.down_service(volumes=False, timeout=5) + + if main_error: + raise MigrationExceptionInBench('') + + def bench_basic_backup(self, bench: MigrationBench): + richprint.print(f"Migrating bench [bold][blue]{bench.name}[/blue][/bold]") + + # backup docker compose.yml + self.backup_manager.backup(bench.path / "docker-compose.yml", bench_name=bench.name) + + # backup common_site_config.json + bench_common_site_config = bench.path / "workspace" / "frappe-bench" / "sites" / "common_site_config.json" + self.backup_manager.backup(bench_common_site_config, bench_name=bench.name) + + # backup site_config.json + bench_site_config = bench.path / "workspace" / "frappe-bench" / "sites" / bench.name / "site_config.json" + self.backup_manager.backup(bench_site_config, bench_name=bench.name) + + server_db_info: DatabaseServerServiceInfo = DatabaseServerServiceInfo.import_from_compose_file( + 'global-db', self.services_manager.compose_project + ) + + self.bench_db_backup( + bench=bench, + server_db_info=server_db_info, + services_manager=self.services_manager, + backup_manager=self.backup_manager, + ) + + def migrate_bench(self, bench: MigrationBench): pass + + def undo_bench_migrate(self, bench: MigrationBench): + pass + + def bench_db_backup( + self, + bench: MigrationBench, + server_db_info: DatabaseServerServiceInfo, + services_manager: MigrationServicesManager, + backup_manager: BackupManager, + ): + bench_db_info = bench.get_db_connection_info() + bench_db_name = bench_db_info["name"] + + richprint.change_head(f'Commencing db [blue]{bench.name}[/blue] backup') + + mariadb_manager = MariaDBManager(server_db_info, services_manager.compose_project) + + from datetime import datetime + + current_datetime = datetime.now() + formatted_date = current_datetime.strftime("%d-%m-%Y--%H-%M-%S") + + container_backup_dir: Path = Path("/var/log/mysql") + host_backup_dir: Path = services_manager.services_path / 'mariadb' / 'logs' + db_sql_file_name = f"db-{bench.name}-{formatted_date}.sql" + + host_db_sql_file_path: Path = host_backup_dir / db_sql_file_name + container_db_sql_file_path: Path = container_backup_dir / db_sql_file_name + + backup_gz_file_backup_data_path: Path = ( + bench.path / backup_manager.bench_backup_dir / self.version.version / f'{db_sql_file_name}.gz' + ) + + mariadb_manager.db_export(bench_db_name, container_db_sql_file_path) + + import gzip + import shutil + + # Compress the file using gzip + with open(host_db_sql_file_path, 'rb') as f_in: + with gzip.open(backup_gz_file_backup_data_path, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + + host_db_sql_file_path.unlink() + + richprint.print(f'[blue]{bench.name}[/blue] db backup completed successfully.') diff --git a/frappe_manager/migration_manager/migration_executor.py b/frappe_manager/migration_manager/migration_executor.py index 9ddc8533..d693294e 100644 --- a/frappe_manager/migration_manager/migration_executor.py +++ b/frappe_manager/migration_manager/migration_executor.py @@ -140,28 +140,26 @@ def execute(self): except MigrationExceptionInBench as e: if self.migrate_benches: - print_head = True + passed_print_head = True + for bench, bench_status in self.migrate_benches.items(): if not bench_status["exception"]: - if print_head: - richprint.stdout.rule('[bold][red]Migration Passed Benches[/red][bold]', style='green') - print_head = False + if passed_print_head: + richprint.stdout.rule('[bold]Migration Passed Benches[bold]', style='green') + passed_print_head = False richprint.print(f"[green]Bench[/green]: {bench}", emoji_code=':construction:') - if not print_head: - richprint.stdout.rule(style='red') - - print_head = True + failed_print_head = True for bench, bench_status in self.migrate_benches.items(): if bench_status["exception"]: - if print_head: + if failed_print_head: richprint.stdout.rule( ':police_car_light: [bold][red]Migration Failed Benches[/red][bold] :police_car_light:', style='red', ) - print_head = False + failed_print_head = False richprint.error(f"[red]Bench[/red]: {bench}", emoji_code=':construction:') @@ -178,8 +176,11 @@ def execute(self): richprint.print(f"For error specifics, refer to {CLI_DIR}/logs/fm.log", emoji_code=':page_facing_up:') - if not print_head: + if not failed_print_head: richprint.stdout.rule(style='red') + else: + if not passed_print_head: + richprint.stdout.rule(style='green') archive_msg = [ 'Available options after migrations failure :', diff --git a/frappe_manager/migration_manager/migrations/migrate_0_10_0.py b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py index 0262744c..b11b3576 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_10_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py @@ -31,31 +31,36 @@ from frappe_manager.utils.docker import host_run_cp from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.utils.helpers import get_container_name_prefix, get_unix_groups, random_password_generate +from frappe_manager.migration_manager.backup_manager import BackupManager from frappe_manager.migration_manager.version import Version class MigrationV0100(MigrationBase): version = Version("0.10.0") - def __init__(self): - super().init() + def init(self): self.benches_dir = CLI_DIR / "sites" + self.backup_manager = BackupManager(str(self.version), self.benches_dir) self.string_timestamp = datetime.now().strftime("%d-%b-%y--%H-%M-%S") self.benches_manager = MigrationBenches(self.benches_dir) + self.services_manager: MigrationServicesManager = MigrationServicesManager() def get_rollback_version(self): # this was without any migrations return Version("0.10.1") def up(self): - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") + richprint.stdout.rule(f':package: [bold][blue]v{str(self.version)}[/blue][bold]') + self.logger.info(f"v{str(self.version)}: Started") self.logger.info("-" * 40) - self.services_manager: MigrationServicesManager = MigrationServicesManager() + self.init() + self.migrate_services() + self.migrate_benches() - # stop all benches - self.benches_manager.stop_benches(timeout=20) + self.logger.info("-" * 40) + def migrate_services(self): # create services self.services_create(self.services_manager.compose_project) self.services_manager.compose_project.pull_images() @@ -66,46 +71,46 @@ def up(self): self.services_manager.compose_project, ) - # migrate each bench + def migrate_benches(self): main_error = False - benches = self.benches_manager.get_all_benches() + # migrate each bench + for bench_name, bench_path in self.benches_manager.get_all_benches().items(): + bench = MigrationBench(name=bench_name, path=bench_path.parent) - for bench_name, bench_compose_path in benches.items(): - bench = MigrationBench(name=bench_name, path=bench_compose_path.parent) + if bench.name in self.migration_executor.migrate_benches.keys(): + bench_info = self.migration_executor.migrate_benches[bench.name] + if bench_info['exception']: + richprint.print(f"Skipping migration for failed bench {bench.name}.") + main_error = True + continue self.migration_executor.set_bench_data(bench, migration_version=self.version) - try: self.migrate_bench(bench) except Exception as e: import traceback traceback_str = traceback.format_exc() - self.logger.error(f"[ EXCEPTION TRACEBACK ]:\n {traceback_str}") + self.logger.error(f"{bench.name} [ EXCEPTION TRACEBACK ]:\n {traceback_str}") richprint.update_live() main_error = True self.migration_executor.set_bench_data(bench, e, self.version) + + # restore all backup files + for backup in self.backup_manager.backups: + if backup.bench == bench.name: + self.backup_manager.restore(backup, force=True) + self.undo_bench_migrate(bench) + self.logger.info(f'Undo successfull for bench: {bench.name}') bench.compose_project.down_service(volumes=False, timeout=5) if main_error: - raise MigrationExceptionInBench("") - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) + raise MigrationExceptionInBench('') def migrate_bench(self, bench: MigrationBench): - richprint.print(f"Migrating bench {bench.name}", prefix=f"[bold]v{str(self.version)}:[/bold] ") - - # backup docker compose.yml - self.backup_manager.backup(bench.path / "docker-compose.yml", bench_name=bench.name) - - # backup common_site_config.json - self.backup_manager.backup( - bench.path / "workspace" / "frappe-bench" / "sites" / "common_site_config.json", - bench_name=bench.name, - ) + richprint.change_head("Migrating bench compose") bench.compose_project.down_service(volumes=False) @@ -121,31 +126,12 @@ def migrate_bench(self, bench: MigrationBench): self.services_database_manager.add_user(bench_db_user, bench_db_pass, force=True) self.services_database_manager.grant_user_privilages(bench_db_name, bench_db_user) - def down(self): - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) - + def undo_services_migrate(self): self.services_manager.compose_project.down_service() - if self.services_manager.services_path.exists(): shutil.rmtree(self.services_manager.services_path) - # undo each bench - for bench, exception in self.migration_executor.migrate_benches.items(): - if not exception: - self.undo_bench_migrate(bench) - - for backup in self.backup_manager.backups: - self.backup_manager.restore(backup, force=True) - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) - def undo_bench_migrate(self, bench: MigrationBench): - for backup in self.backup_manager.backups: - if backup.bench == bench.name: - self.backup_manager.restore(backup, force=True) - configs_backup = bench.path / f"configs-{self.string_timestamp}.bak" configs_path = bench.path / "configs" @@ -179,22 +165,21 @@ def undo_bench_migrate(self, bench: MigrationBench): def migrate_bench_compose(self, bench: MigrationBench): richprint.change_head("Migrating database") - compose_version = bench.compose_project.compose_file_manager.get_version() - if not bench.compose_project.compose_file_manager.exists(): raise BenchDockerComposeFileNotFound(bench.name, bench.compose_project.compose_file_manager.compose_path) db_backup_file = self.db_migration_export(bench) - richprint.print("Database exported") + + richprint.print(f"[blue]{bench.name}[/blue] db exported from bench mariadb service.") # backup bench db db_backup = self.backup_manager.backup(db_backup_file, bench_name=bench.name, allow_restore=False) self.db_migration_import(bench=bench, db_backup_file=db_backup) - richprint.print("Database Imported") - status_msg = "Migrating bench compose" - richprint.change_head(status_msg) + richprint.print(f"[blue]{bench.name}[/blue] db imported to global-db service.") + + richprint.change_head("Migrating bench compose") # get all the payloads envs = bench.compose_project.compose_file_manager.get_all_envs() @@ -250,11 +235,13 @@ def migrate_bench_compose(self, bench: MigrationBench): # change the node socketio port bench.common_bench_config_set({"socketio_port": "80"}) - richprint.print(f"{status_msg} {compose_version} -> {self.version.version}: Done") + richprint.print(f"Migrated [blue]{bench.name}[/blue] compose file.") return db_backup def create_compose_dirs(self, bench: MigrationBench): + richprint.change_head("Creating services compose dirs.") + # directory creation configs_path = bench.path / "configs" @@ -291,14 +278,15 @@ def create_compose_dirs(self, bench: MigrationBench): new_dir = nginx_dir / directory new_dir.mkdir(parents=True, exist_ok=True) + richprint.print("Created services compose dirs.") + def db_migration_export(self, bench: MigrationBench) -> Path: self.logger.debug("[db export] bench: %s", bench.name) # start the benc hand also handle the missing docker images output = bench.compose_project.docker.compose.up( - services=["mariadb", "frappe"], detach=True, pull="missing", stream=True + services=["mariadb", "frappe"], detach=True, pull="missing", stream=False ) - richprint.live_lines(output, padding=(0, 0, 0, 2)) self.logger.debug("[db export] checking if mariadb started") @@ -348,7 +336,11 @@ def db_migration_import(self, bench: MigrationBench, db_backup_file: BackupData) self.services_database_manager.grant_user_privilages(bench_db_user, bench_db_name) + richprint.print(f"{bench.name} db imported.") + def services_create(self, services_compose_project: ComposeProject): + richprint.change_head("Creating services.") + envs = { "global-db": { "MYSQL_ROOT_PASSWORD_FILE": '/run/secrets/db_root_password', @@ -442,8 +434,10 @@ def services_create(self, services_compose_project: ComposeProject): services_compose_project.docker.compose.down(remove_orphans=True, timeout=1, volumes=True, stream=True) + richprint.print(f"Created services at {self.services_manager.services_path}.") + def generate_compose(self, inputs: dict, compose_file_manager: ComposeFile): - # TODO do something about this function + richprint.change_head(f"Generating services compose file.") try: # handle environment if "environment" in inputs.keys(): @@ -462,5 +456,6 @@ def generate_compose(self, inputs: dict, compose_file_manager: ComposeFile): uid = user[container_name]["uid"] gid = user[container_name]["gid"] compose_file_manager.set_user(container_name, uid, gid) + richprint.print(f"Generated services compose file.") except Exception: raise ServicesNotCreated() diff --git a/frappe_manager/migration_manager/migrations/migrate_0_11_0.py b/frappe_manager/migration_manager/migrations/migrate_0_11_0.py index bb95f5a7..f0759a01 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_11_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_11_0.py @@ -1,113 +1,37 @@ +from pathlib import Path from frappe_manager.migration_manager.migration_base import MigrationBase from frappe_manager.migration_manager.migration_exections import MigrationExceptionInBench -from frappe_manager.migration_manager.migration_helpers import MigrationBench, MigrationBenches +from frappe_manager.migration_manager.migration_helpers import ( + MigrationBench, + MigrationBenches, + MigrationServicesManager, +) from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.migration_manager.version import Version -from frappe_manager import CLI_DIR, CLI_SERVICES_DIRECTORY +from frappe_manager.migration_manager.backup_manager import BackupManager + class MigrationV0110(MigrationBase): version = Version("0.11.0") - def __init__(self): - super().init() - self.benches_dir = CLI_DIR / "sites" - self.services_path = CLI_SERVICES_DIRECTORY + def init(self): + self.cli_dir: Path = Path.home() / 'frappe' + self.benches_dir = self.cli_dir / "sites" + self.backup_manager = BackupManager(str(self.version), self.benches_dir) self.benches_manager = MigrationBenches(self.benches_dir) - - def up(self): - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - - benches = self.benches_manager.get_all_benches() - - main_error = False - - # migrate each bench - for bench_name, bench_path in benches.items(): - bench = MigrationBench(name=bench_name, path=bench_path.parent) - - if bench.name in self.migration_executor.migrate_benches.keys(): - bench_info = self.migration_executor.migrate_benches[bench.name] - if bench_info['exception']: - richprint.print(f"Skipping migration for failed bench{bench.name}.") - main_error = True - continue - - self.migration_executor.set_bench_data(bench,migration_version=self.version) - try: - self.migrate_bench(bench) - except Exception as e: - import traceback - traceback_str = traceback.format_exc() - self.logger.error(f"{bench.name} [ EXCEPTION TRACEBACK ]:\n {traceback_str}") - richprint.update_live() - main_error = True - self.migration_executor.set_bench_data(bench, e, self.version) - self.undo_bench_migrate(bench) - bench.compose_project.down_service(volumes=False, timeout=5) - - if main_error: - raise MigrationExceptionInBench('') - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - - def migrate_bench(self, bench: MigrationBench): - richprint.print(f"Migrating bench {bench.name}", prefix=f"[bold]v{str(self.version)}:[/bold] ") - - # backup docker compose.yml - self.backup_manager.backup(bench.path / "docker-compose.yml", bench_name=bench.name) - - # backup common_site_config.json - self.backup_manager.backup( - bench.path - / "workspace" - / "frappe-bench" - / "sites" - / "common_site_config.json", - bench_name=bench.name, + self.services_manager: MigrationServicesManager = MigrationServicesManager( + services_path=self.cli_dir / 'services' ) + def migrate_bench(self, bench: MigrationBench): bench.compose_project.down_service(volumes=False) self.migrate_bench_compose(bench) - def down(self): - # richprint.print(f"Started",prefix=f"[ Migration v{str(self.version)} ][ROLLBACK] : ") - richprint.print( - f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] " - ) - self.logger.info("-" * 40) - - # undo each bench - for bench, exception in self.migration_executor.migrate_benches.items(): - if not exception: - self.undo_bench_migrate(bench) - - for backup in self.backup_manager.backups: - self.backup_manager.restore(backup, force=True) - - richprint.print( - f"Successfull", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] " - ) - self.logger.info("-" * 40) - - def undo_bench_migrate(self, bench: MigrationBench): - - for backup in self.backup_manager.backups: - if backup.bench == bench.name: - self.backup_manager.restore(backup, force=True) - - self.logger.info(f'Undo successfull for bench: {bench.name}') - def migrate_bench_compose(self, bench: MigrationBench): - - status_msg = 'Migrating bench compose' - richprint.change_head(status_msg) - - compose_version = bench.compose_project.compose_file_manager.get_version() + richprint.change_head("Migrating bench compose") if not bench.compose_project.compose_file_manager.exists(): - richprint.print(f"{status_msg} {compose_version} -> {self.version}: Failed ") + richprint.print(f"Failed to migrate {bench.name} compose file.") raise MigrationExceptionInBench(f"{bench.compose_project.compose_file_manager.compose_path} not found.") # change image tag to the latest @@ -119,7 +43,9 @@ def migrate_bench_compose(self, bench: MigrationBench): image_info['tag'] = self.version.version_string() image_info['name'] = 'ghcr.io/rtcamp/frappe-manager-frappe' - output = bench.compose_project.docker.pull(container_name=f"{image_info['name']}:{image_info['tag']}", stream=True) + output = bench.compose_project.docker.pull( + container_name=f"{image_info['name']}:{image_info['tag']}", stream=True + ) richprint.live_lines(output, padding=(0, 0, 0, 2)) bench.compose_project.compose_file_manager.set_all_images(images_info) @@ -127,4 +53,4 @@ def migrate_bench_compose(self, bench: MigrationBench): bench.compose_project.compose_file_manager.set_version(str(self.version)) bench.compose_project.compose_file_manager.write_to_file() - richprint.print(f"{status_msg} {compose_version} -> {self.version}: Done") + richprint.print(f"Migrated [blue]{bench.name}[/blue] compose file.") diff --git a/frappe_manager/migration_manager/migrations/migrate_0_12_0.py b/frappe_manager/migration_manager/migrations/migrate_0_12_0.py index a9f29da9..b3362811 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_12_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_12_0.py @@ -1,30 +1,31 @@ +from pathlib import Path from frappe_manager.migration_manager.migration_base import MigrationBase from frappe_manager.docker_wrapper.DockerClient import DockerClient from frappe_manager.migration_manager.migration_exections import MigrationExceptionInBench -from frappe_manager.migration_manager.migration_helpers import MigrationBench, MigrationBenches +from frappe_manager.migration_manager.migration_helpers import ( + MigrationBench, + MigrationBenches, + MigrationServicesManager, +) from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.migration_manager.version import Version -from frappe_manager import CLI_DIR from frappe_manager.utils.helpers import get_container_name_prefix +from frappe_manager.migration_manager.backup_manager import BackupManager class MigrationV0120(MigrationBase): version = Version("0.12.0") - def __init__(self): - super().init() - self.benches_dir = CLI_DIR / "sites" + def init(self): + self.cli_dir: Path = Path.home() / 'frappe' + self.benches_dir = self.cli_dir / "sites" + self.backup_manager = BackupManager(str(self.version), self.benches_dir) self.benches_manager = MigrationBenches(self.benches_dir) + self.services_manager: MigrationServicesManager = MigrationServicesManager( + services_path=self.cli_dir / 'services' + ) - def up(self): - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - - # take backup of each of the bench docker compose - self.benches_manager.stop_benches() - - benches = self.benches_manager.get_all_benches() - + def migrate_services(self): # Pulling latest image self.image_info = {"tag": self.version.version_string(), "name": "ghcr.io/rtcamp/frappe-manager-frappe"} pull_image = f"{self.image_info['name']}:{self.image_info['tag']}" @@ -34,87 +35,16 @@ def up(self): richprint.live_lines(output, padding=(0, 0, 0, 2)) richprint.print(f"Image pulled [blue]{pull_image}[/blue]") - # migrate each bench - main_error = False - - # migrate each bench - for bench_name, bench_path in benches.items(): - bench = MigrationBench(name=bench_name, path=bench_path.parent) - - if bench.name in self.migration_executor.migrate_benches.keys(): - bench_info = self.migration_executor.migrate_benches[bench.name] - if bench_info['exception']: - richprint.print(f"Skipping migration for failed bench{bench.name}.") - main_error = True - continue - - self.migration_executor.set_bench_data(bench, migration_version=self.version) - try: - self.migrate_bench(bench) - except Exception as e: - import traceback - - traceback_str = traceback.format_exc() - self.logger.error(f"{bench.name} [ EXCEPTION TRACEBACK ]:\n {traceback_str}") - richprint.update_live() - main_error = True - self.migration_executor.set_bench_data(bench, e, self.version) - self.undo_bench_migrate(bench) - bench.compose_project.down_service(volumes=False, timeout=5) - - if main_error: - raise MigrationExceptionInBench('') - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - def migrate_bench(self, bench: MigrationBench): - richprint.print(f"Migrating bench {bench.name}", prefix=f"[bold]v{str(self.version)}:[/bold] ") - - # backup docker compose.yml - self.backup_manager.backup(bench.path / "docker-compose.yml", bench_name=bench.name) - - # backup common_site_config.json - self.backup_manager.backup( - bench.path / "workspace" / "frappe-bench" / "sites" / "common_site_config.json", - bench_name=bench.name, - ) - bench.compose_project.down_service(volumes=False) self.migrate_bench_compose(bench) self.migrate_workers_compose(bench) - def down(self): - # richprint.print(f"Started",prefix=f"[ Migration v{str(self.version)} ][ROLLBACK] : ") - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) - - # undo each bench - for bench, exception in self.migration_executor.migrate_benches.items(): - if not exception: - self.undo_bench_migrate(bench) - - for backup in self.backup_manager.backups: - self.backup_manager.restore(backup, force=True) - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) - - def undo_bench_migrate(self, bench: MigrationBench): - for backup in self.backup_manager.backups: - if backup.bench == bench.name: - self.backup_manager.restore(backup, force=True) - - self.logger.info(f"Undo successfull for bench: {bench.name}") - def migrate_bench_compose(self, bench: MigrationBench): - status_msg = "Migrating bench compose" - richprint.change_head(status_msg) - - compose_version = bench.compose_project.compose_file_manager.get_version() + richprint.change_head("Migrating bench compose") if not bench.compose_project.compose_file_manager.exists(): - richprint.print(f"{status_msg} {compose_version} -> {self.version}: Failed ") + richprint.print(f"Failed to migrate {bench.name} compose file.") raise MigrationExceptionInBench(f"{bench.compose_project.compose_file_manager.compose_path} not found.") images_info = bench.compose_project.compose_file_manager.get_all_images() @@ -130,19 +60,23 @@ def migrate_bench_compose(self, bench: MigrationBench): try: del compose_yml["services"][service]["restart"] except KeyError as e: - self.logger.error(f"{bench.name}: Not able to delete restart: always attribute from compose file.{e}") + self.logger.error( + f"{bench.name} worker not able to delete 'restart: always' attribute from compose file.{e}" + ) pass - richprint.print("Removed [blue]restart: always[/blue]") + richprint.print("Removed 'restart: always'.") bench.compose_project.compose_file_manager.set_version(str(self.version)) bench.compose_project.compose_file_manager.set_all_images(images_info) bench.compose_project.compose_file_manager.write_to_file() - richprint.print(f"{status_msg} {compose_version} -> {self.version}: Done") + + richprint.print(f"Migrated {bench.name} compose file.") def migrate_workers_compose(self, bench: MigrationBench): if bench.workers_compose_project.compose_file_manager.compose_path.exists(): richprint.print("Migrating workers compose") + # workers image set workers_info = bench.workers_compose_project.compose_file_manager.get_all_images() @@ -155,7 +89,7 @@ def migrate_workers_compose(self, bench: MigrationBench): del worker_compose_yml["services"][service]["restart"] except KeyError as e: self.logger.error( - f"{bench.name} worker: Not able to delete restart: always attribute from compose file.{e}" + f"{bench.name} worker not able to delete 'restart: always' attribute from compose file.{e}" ) pass @@ -168,4 +102,5 @@ def migrate_workers_compose(self, bench: MigrationBench): get_container_name_prefix(bench.name) ) bench.workers_compose_project.compose_file_manager.write_to_file() - richprint.print("Migrating workers compose: Done") + + richprint.print(f"Migrated [blue]{bench.name}[/blue] compose file.") diff --git a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py index 6e9b34da..52047f84 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py @@ -1,6 +1,7 @@ import json import os import copy +from pathlib import Path from frappe_manager.compose_manager import DockerVolumeMount from frappe_manager.compose_manager.ComposeFile import ComposeFile from frappe_manager.migration_manager.migration_base import MigrationBase @@ -16,18 +17,22 @@ from frappe_manager.site_manager.bench_config import BenchConfig, FMBenchEnvType from frappe_manager.ssl_manager import SUPPORTED_SSL_TYPES from frappe_manager.ssl_manager.certificate import SSLCertificate -from frappe_manager.utils.helpers import capture_and_format_exception, get_container_name_prefix +from frappe_manager.utils.helpers import get_container_name_prefix from frappe_manager.docker_wrapper.DockerClient import DockerClient +from frappe_manager.migration_manager.backup_manager import BackupManager class MigrationV0130(MigrationBase): version = Version("0.13.0") - def __init__(self): - super().init() - self.benches_dir = CLI_DIR / "sites" - self.services_path = CLI_SERVICES_DIRECTORY + def init(self): + self.cli_dir: Path = Path.home() / 'frappe' + self.benches_dir = self.cli_dir / "sites" + self.backup_manager = BackupManager(str(self.version), self.benches_dir) self.benches_manager = MigrationBenches(self.benches_dir) + self.services_manager: MigrationServicesManager = MigrationServicesManager( + services_path=self.cli_dir / 'services' + ) def migrate_services(self): # backup services compose @@ -36,14 +41,13 @@ def migrate_services(self): f"Services compose at {self.services_manager.compose_project.compose_file_manager} not found." ) - self.backup_manager.backup( - self.services_manager.compose_project.compose_file_manager.compose_path / "docker-compose.yml" - ) + self.backup_manager.backup(self.services_manager.compose_project.compose_file_manager.compose_path) + # remove version from services yml try: del self.services_manager.compose_project.compose_file_manager.yml['version'] except KeyError: - self.logger.warning(f"[services]: version attribute not found compose.") + self.logger.warning(f"[services]: 'version' attribute not found in compose file.") pass # include new volume info @@ -73,68 +77,14 @@ def migrate_services(self): if self.services_manager.compose_project.is_service_running('global-nginx-proxy'): self.services_manager.compose_project.docker.compose.up(services=['global-nginx-proxy']) - def up(self): - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - - self.services_manager: MigrationServicesManager = MigrationServicesManager() - - self.benches_manager.stop_benches() - - benches = self.benches_manager.get_all_benches() - - # migrate services - richprint.change_head("Migrating services") - self.migrate_services() - richprint.print("Migrating services: Done") - # rename main config fm_config_path = CLI_DIR / 'fm_config.toml' old_fm_config_path = CLI_DIR / '.fm.toml' if old_fm_config_path.exists(): old_fm_config_path.rename(fm_config_path) - # migrate each bench - main_error = False - - # migrate each bench - for bench_name, bench_path in benches.items(): - bench = MigrationBench(name=bench_name, path=bench_path.parent) - - if bench.name in self.migration_executor.migrate_benches.keys(): - bench_info = self.migration_executor.migrate_benches[bench.name] - if bench_info['exception']: - richprint.print(f"Skipping migration for failed bench{bench.name}.") - main_error = True - continue - - self.migration_executor.set_bench_data(bench, migration_version=self.version) - try: - self.migrate_bench(bench) - except Exception as e: - exception_str = capture_and_format_exception() - self.logger.error(f"{bench.name} [ EXCEPTION TRACEBACK ]:\n {exception_str}") - richprint.update_live() - main_error = True - self.migration_executor.set_bench_data(bench, e, self.version) - self.undo_bench_migrate(bench) - bench.compose_project.down_service(volumes=False, timeout=5) - - if main_error: - raise MigrationExceptionInBench('') - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - def migrate_bench(self, bench: MigrationBench): - richprint.print(f"Migrating bench {bench.name}", prefix=f"[bold]v{str(self.version)}:[/bold] ") - - # backup docker compose.yml - self.backup_manager.backup(bench.path / "docker-compose.yml", bench_name=bench.name) - - # backup common_site_config.json bench_common_site_config = bench.path / "workspace" / "frappe-bench" / "sites" / "common_site_config.json" - self.backup_manager.backup(bench_common_site_config, bench_name=bench.name) common_site_config_json = json.loads(bench_common_site_config.read_bytes()) if 'mail_server' in common_site_config_json: @@ -144,48 +94,23 @@ def migrate_bench(self, bench: MigrationBench): bench.compose_project.down_service(volumes=False) self.migrate_bench_compose(bench) - def down(self): - # richprint.print(f"Started",prefix=f"[ Migration v{str(self.version)} ][ROLLBACK] : ") - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) - - # undo each bench - for bench, exception in self.migration_executor.migrate_benches.items(): - if not exception: - self.undo_bench_migrate(bench) - - for backup in self.backup_manager.backups: - self.backup_manager.restore(backup, force=True) - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) - def undo_bench_migrate(self, bench: MigrationBench): - for backup in self.backup_manager.backups: - if backup.bench == bench.name: - self.backup_manager.restore(backup, force=True) + richprint.change_head("Removing Admin Tools compose file") admin_tools_compose_path = bench.path / 'docker-compose.admin-tools.yml' if admin_tools_compose_path.exists(): admin_tools_compose_path.unlink() - self.logger.info(f"Undo successfull for bench: {bench.name}") + richprint.print("Removed Admin Tools compose file") def migrate_bench_compose(self, bench: MigrationBench): - status_msg = "Migrating bench compose" - richprint.change_head(status_msg) - - compose_version = bench.compose_project.compose_file_manager.get_version() + richprint.change_head("Migrating bench compose") if not bench.compose_project.compose_file_manager.exists(): - richprint.print(f"{status_msg} {compose_version} -> {self.version.version}: Failed ") + richprint.print(f"Failed to migrate {bench.name} compose file.") raise MigrationExceptionInBench(f"{bench.compose_project.compose_file_manager.compose_path} not found.") - # generate bench config for bench and save it - status_msg = "Migrating bench compose" - richprint.change_head(status_msg) - # get all the payloads envs = bench.compose_project.compose_file_manager.get_all_envs() @@ -196,7 +121,7 @@ def migrate_bench_compose(self, bench: MigrationBench): try: del envs['nginx']['ENABLE_SSL'] except KeyError: - self.logger.warning(f"{bench.name}: ENABLE_SSL nginx's env not found.") + self.logger.warning(f"{bench.name} 'ENABLE_SSL' nginx's env not found.") pass # create new html in configs/nginx/html compose directory @@ -278,7 +203,7 @@ def migrate_bench_compose(self, bench: MigrationBench): try: del bench.compose_project.compose_file_manager.yml['version'] except KeyError: - self.logger.warning(f"{bench.name}: version attribute not found compose.") + self.logger.warning(f"{bench.name} 'version' attribute not found in compose file.") pass # include new volume info @@ -300,9 +225,7 @@ def migrate_bench_compose(self, bench: MigrationBench): bench.compose_project.compose_file_manager.set_version(str(self.version)) bench.compose_project.compose_file_manager.write_to_file() - bench.compose_project.compose_file_manager.set_all_images(images_info) - - richprint.print(f"{status_msg} {compose_version} -> {self.version.version}: Done") + richprint.print(f"Migrated [blue]{bench.name}[/blue] compose file.") def migrate_workers_compose(self, bench: MigrationBench): if bench.workers_compose_project.compose_file_manager.compose_path.exists(): @@ -315,7 +238,7 @@ def migrate_workers_compose(self, bench: MigrationBench): try: del bench.workers_compose_project.compose_file_manager.yml['version'] except KeyError: - self.logger.warning(f"{bench.name} workers: version attribute not found compose.") + self.logger.warning(f"{bench.name} workers 'version' attribute not found in compose file.") pass bench.workers_compose_project.compose_file_manager.set_top_networks_name( @@ -326,10 +249,11 @@ def migrate_workers_compose(self, bench: MigrationBench): ) bench.compose_project.compose_file_manager.set_version(str(self.version)) bench.workers_compose_project.compose_file_manager.write_to_file() - richprint.print("Migrating workers compose: Done") + + richprint.print(f"Migrated [blue]{bench.name}[/blue] workers compose file.") def migrate_admin_tools_compose(self, bench: MigrationBench): - richprint.change_head("Create admin-tools") + richprint.change_head("Create Admin Tools") bench_compose_yml = copy.deepcopy(bench.compose_project.compose_file_manager.yml) @@ -387,4 +311,4 @@ def migrate_admin_tools_compose(self, bench: MigrationBench): common_bench_config_path.write_text(json.dumps(current_common_site_config)) - richprint.change_head("Created Admin-tools.") + richprint.change_head(f"Created {bench.name} Admin Tools.") diff --git a/frappe_manager/migration_manager/migrations/migrate_0_13_1.py b/frappe_manager/migration_manager/migrations/migrate_0_13_1.py index 6a147172..e7f3b9c0 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_13_1.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_13_1.py @@ -1,35 +1,37 @@ +from pathlib import Path from frappe_manager.migration_manager.migration_base import MigrationBase from frappe_manager.migration_manager.migration_exections import MigrationExceptionInBench from frappe_manager.migration_manager.migration_helpers import ( + MigrationBenches, MigrationServicesManager, ) from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.migration_manager.version import Version -from frappe_manager import CLI_DIR, CLI_SERVICES_DIRECTORY +from frappe_manager.migration_manager.backup_manager import BackupManager class MigrationV0131(MigrationBase): version = Version("0.13.1") - def __init__(self): - super().init() - self.benches_dir = CLI_DIR / "sites" - self.services_path = CLI_SERVICES_DIRECTORY - - def up(self): - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - - self.services_manager: MigrationServicesManager = MigrationServicesManager() + def init(self): + self.cli_dir: Path = Path.home() / 'frappe' + self.benches_dir = self.cli_dir / "sites" + self.backup_manager = BackupManager(str(self.version), self.benches_dir) + self.benches_manager = MigrationBenches(self.benches_dir) + self.services_manager: MigrationServicesManager = MigrationServicesManager( + services_path=self.cli_dir / 'services' + ) + def migrate_services(self): if not self.services_manager.compose_project.compose_file_manager.exists(): raise MigrationExceptionInBench( f"Services compose at {self.services_manager.compose_project.compose_file_manager} not found." ) richprint.change_head("Adding fm header config to nginx-proxy") + # create file fmheaders.conf - fm_headers_conf_path = self.services_path / 'nginx-proxy' / 'confd' / 'fm_headers.conf' + fm_headers_conf_path = self.services_manager.services_path / 'nginx-proxy' / 'confd' / 'fm_headers.conf' add_header = f'add_header X-Powered-By "Frappe-Manager {self.version.version_string()}";' fm_headers_conf_path.write_text(add_header) @@ -39,17 +41,3 @@ def up(self): self.services_manager.compose_project.docker.compose.restart(services=['global-nginx-proxy'], stream=False) richprint.print("Added fm header config to nginx-proxy.") - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - - def down(self): - # richprint.print(f"Started",prefix=f"[ Migration v{str(self.version)} ][ROLLBACK] : ") - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) - - for backup in self.backup_manager.backups: - self.backup_manager.restore(backup, force=True) - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) diff --git a/frappe_manager/migration_manager/migrations/migrate_0_9_0.py b/frappe_manager/migration_manager/migrations/migrate_0_9_0.py index 1b4218b1..72204c00 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_9_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_9_0.py @@ -1,4 +1,5 @@ import shutil +from frappe_manager.migration_manager.backup_manager import BackupManager from frappe_manager.migration_manager.migration_base import MigrationBase from frappe_manager import CLI_DIR from frappe_manager.migration_manager.migration_helpers import MigrationBenches @@ -9,18 +10,19 @@ class MigrationV090(MigrationBase): version = Version("0.9.0") - def __init__(self): - super().init() - self.bences_dir = CLI_DIR / "sites" + def init(self): + self.benches_dir = CLI_DIR / "sites" + self.backup_manager = BackupManager(str(self.version)) - if self.bences_dir.exists(): + if self.benches_dir.exists(): self.skip = True def up(self): if self.skip: return True - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") + richprint.stdout.rule(f':package: [bold][blue]v{str(self.version)}[/blue][bold]') + self.logger.info(f"v{str(self.version)}: Started") self.logger.info("-" * 40) move_directory_list = [] @@ -32,40 +34,33 @@ def up(self): if docker_compose_path.exists(): move_directory_list.append(site_dir) - self.bences_dir.mkdir(parents=True, exist_ok=True) + self.benches_dir.mkdir(parents=True, exist_ok=True) - benches = MigrationBenches(self.bences_dir) + benches = MigrationBenches(self.benches_dir) benches.stop_benches() # move all the directories - richprint.print( - f"Moving benches from {CLI_DIR} to {self.bences_dir}", prefix=f"[bold]v{str(self.version)}:[/bold] " - ) + richprint.change_head(f"Moving benches from {CLI_DIR} to {self.benches_dir}") for site in move_directory_list: site_name = site.parts[-1] - new_path = self.bences_dir / site_name + new_path = self.benches_dir / site_name shutil.move(site, new_path) self.logger.debug(f"Moved:{site.exists()}") - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info(f"[{self.version}] : Migration starting") + richprint.print(f"Moved benches from {CLI_DIR} to {self.benches_dir}") self.logger.info("-" * 40) def down(self): if self.skip: return True - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") + richprint.change_head(f"Working on v{str(self.version)} rollback.") self.logger.info("-" * 40) - if self.bences_dir.exists(): - richprint.print( - f"Found benches directory change.", prefix=f"[bold]v{str(self.version.version)} [ROLLBACK]:[/bold] " - ) - + if self.benches_dir.exists(): move_directory_list = [] - for site_dir in self.bences_dir.iterdir(): + for site_dir in self.benches_dir.iterdir(): if site_dir.is_dir(): docker_compose_path = site_dir / "docker-compose.yml" @@ -73,17 +68,17 @@ def down(self): move_directory_list.append(site_dir) # stop all the sites - sites_mananger = MigrationBenches(self.bences_dir) + sites_mananger = MigrationBenches(self.benches_dir) sites_mananger.stop_benches() # move all the directories for site in move_directory_list: site_name = site.parts[-1] - new_path = self.bences_dir.parent / site_name + new_path = self.benches_dir.parent / site_name shutil.move(site, new_path) # delete the sitedir - shutil.rmtree(self.bences_dir) + shutil.rmtree(self.benches_dir) - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version.version)} [ROLLBACK]:[/bold] ") + richprint.print(f"[bold]v{str(self.version)}[/bold] rollback successfull.") self.logger.info("-" * 40) diff --git a/frappe_manager/services_manager/database_service_manager.py b/frappe_manager/services_manager/database_service_manager.py index 3112541a..ad6e6b4f 100644 --- a/frappe_manager/services_manager/database_service_manager.py +++ b/frappe_manager/services_manager/database_service_manager.py @@ -3,12 +3,23 @@ from typing import Dict, Any, Optional, Protocol, List, Union from frappe_manager.compose_project.compose_project import ComposeProject from frappe_manager.docker_wrapper.DockerException import DockerException -from frappe_manager.services_manager.services_exceptions import DatabaseServiceDBCreateFailed, DatabaseServiceDBExportFailed, DatabaseServiceDBImportFailed, DatabaseServiceDBNotFoundError, DatabaseServiceDBRemoveFailError, DatabaseServiceException, DatabaseServicePasswordNotFound, DatabaseServiceStartTimeout, DatabaseServiceUserRemoveFailError +from frappe_manager.services_manager.services_exceptions import ( + DatabaseServiceDBCreateFailed, + DatabaseServiceDBExportFailed, + DatabaseServiceDBImportFailed, + DatabaseServiceDBNotFoundError, + DatabaseServiceDBRemoveFailError, + DatabaseServiceException, + DatabaseServicePasswordNotFound, + DatabaseServiceStartTimeout, + DatabaseServiceUserRemoveFailError, +) from frappe_manager.display_manager.DisplayManager import richprint from pydantic import BaseModel from frappe_manager.utils.docker import SubprocessOutput + # TODO this class will be used for validation for main config class DatabaseServerServiceInfo(BaseModel): host: str @@ -24,7 +35,7 @@ def import_from_compose_file(cls, compose_service_name: str, compose_project: Co """ compose_service_envs = compose_project.compose_file_manager.get_envs(container=compose_service_name) - info: Dict[str, Any]= {} + info: Dict[str, Any] = {} info["user"] = 'root' # this also being considered as servicename info["host"] = compose_service_name @@ -44,29 +55,30 @@ def import_from_compose_file(cls, compose_service_name: str, compose_project: Co return cls(**info) + class DatabaseServiceManager(Protocol): database_server_info: DatabaseServerServiceInfo compose_project: ComposeProject - def __init__(self,database_server_info: DatabaseServerServiceInfo, compose_project: ComposeProject) -> None: + def __init__(self, database_server_info: DatabaseServerServiceInfo, compose_project: ComposeProject) -> None: ... def remove_user(self, db_user: str, db_user_host: str = '%', remove_all_host: bool = False): ... - def add_user(self, db_user: str, db_pass: str, db_user_host: str = '%', force: bool = False, timeout = 25): + def add_user(self, db_user: str, db_pass: str, db_user_host: str = '%', force: bool = False, timeout=25): ... - def grant_user_privilages(self, db_user:str, db_name: str): + def grant_user_privilages(self, db_user: str, db_name: str): ... - def check_user_exists(self,db_user: str): + def check_user_exists(self, db_user: str): ... - def check_db_exists(self,db_name: str): + def check_db_exists(self, db_name: str): ... - def remove_db(self,db_name: str): + def remove_db(self, db_name: str): ... def wait_till_db_start(self, interval: int = 5, timeout: int = 30) -> bool: @@ -75,8 +87,14 @@ def wait_till_db_start(self, interval: int = 5, timeout: int = 30) -> bool: def db_import(self, db_name: str, host_db_file_path: Path, force: bool = False): ... + class MariaDBManager(DatabaseServiceManager): - def __init__(self, database_server_info: DatabaseServerServiceInfo, compose_project: ComposeProject, run_on_compose_service: Optional[str] = None) -> None: + def __init__( + self, + database_server_info: DatabaseServerServiceInfo, + compose_project: ComposeProject, + run_on_compose_service: Optional[str] = None, + ) -> None: """ Database manager """ @@ -92,7 +110,9 @@ def __init__(self, database_server_info: DatabaseServerServiceInfo, compose_proj self.base_query = '-e ' self.quiet = True - def db_run_query(self, query: str, raise_exception_obj: Optional[DatabaseServiceException] = None, capture_output: bool = False): + def db_run_query( + self, query: str, raise_exception_obj: Optional[DatabaseServiceException] = None, capture_output: bool = False + ): base_command = self.base_command if capture_output: @@ -101,7 +121,9 @@ def db_run_query(self, query: str, raise_exception_obj: Optional[DatabaseService db_query = base_command + self.base_query + query try: - output = self.compose_project.docker.compose.exec(self.run_on_compose_service, command=db_query, stream=not capture_output) + output = self.compose_project.docker.compose.exec( + self.run_on_compose_service, command=db_query, stream=not capture_output + ) if capture_output: return output richprint.live_lines(output) @@ -117,27 +139,31 @@ def wait_till_db_start(self, interval: int = 5, timeout: int = 30) -> bool: else: return True total_timeout = interval * timeout - raise DatabaseServiceStartTimeout(total_timeout,self.run_on_compose_service) + raise DatabaseServiceStartTimeout(total_timeout, self.run_on_compose_service) def is_db_running(self) -> bool: db_started_command = f"mysqladmin -P{self.database_server_info.port} -h{self.database_server_info.host} -u'{self.database_server_info.user}' -p'{self.database_server_info.password}' ping" try: - output = self.compose_project.docker.compose.exec(self.run_on_compose_service, command=db_started_command, stream=False) + output = self.compose_project.docker.compose.exec( + self.run_on_compose_service, command=db_started_command, stream=False + ) return 'mysqld is alive' in " ".join(output.stdout) except DockerException as e: return False - def get_db_users(self) -> Dict[str,str]: + def get_db_users(self) -> Dict[str, str]: show_db_user_command = f"'SELECT User, Host FROM mysql.user;'" - exception = DatabaseServiceException(self.database_server_info.host,'Failed to determine mysql users.') - output: SubprocessOutput = self.db_run_query(show_db_user_command, raise_exception_obj=exception,capture_output=True) - user_list: Dict[str,str] = {} + exception = DatabaseServiceException(self.database_server_info.host, 'Failed to determine mysql users.') + output: SubprocessOutput = self.db_run_query( + show_db_user_command, raise_exception_obj=exception, capture_output=True + ) + user_list: Dict[str, str] = {} for line in output.stdout: username, host = line.split('\t') user_list[username] = host return user_list - def check_user_exists(self,username:str, host:Optional[str] = None) -> bool: + def check_user_exists(self, username: str, host: Optional[str] = None) -> bool: user_list = self.get_db_users() if not username in user_list: return False @@ -149,8 +175,10 @@ def check_user_exists(self,username:str, host:Optional[str] = None) -> bool: def get_all_databases(self) -> List[str]: db_exits_commmand = f"'SHOW DATABASES;'" - db_exits_exception = DatabaseServiceException(self.database_server_info.host, 'Failed to get list of all databases.') - output: SubprocessOutput = self.db_run_query(db_exits_commmand,db_exits_exception, capture_output=True) + db_exits_exception = DatabaseServiceException( + self.database_server_info.host, 'Failed to get list of all databases.' + ) + output: SubprocessOutput = self.db_run_query(db_exits_commmand, db_exits_exception, capture_output=True) return output.stdout def check_db_exists(self, db_name: str): @@ -158,7 +186,7 @@ def check_db_exists(self, db_name: str): return db_name in databases def remove_user(self, db_user: str, db_user_host: str = '%', remove_all_host: bool = False): - users = { db_user : db_user_host } + users = {db_user: db_user_host} if remove_all_host: users = self.get_db_users() @@ -166,31 +194,35 @@ def remove_user(self, db_user: str, db_user_host: str = '%', remove_all_host: bo for user, host in users.items(): if db_user == user: remove_db_user_command = f"'DROP USER `{user}`@`{host}`;'" - remove_db_user_exception = DatabaseServiceUserRemoveFailError(user,host) + remove_db_user_exception = DatabaseServiceUserRemoveFailError(user, host) self.db_run_query(remove_db_user_command, remove_db_user_exception) - def remove_db(self,db_name: str): + def remove_db(self, db_name: str): remove_db_command = f"'DROP DATABASE `{db_name}`;'" - remove_db_exception = DatabaseServiceDBRemoveFailError(db_name,self.database_server_info.host) - self.db_run_query(remove_db_command,remove_db_exception) + remove_db_exception = DatabaseServiceDBRemoveFailError(db_name, self.database_server_info.host) + self.db_run_query(remove_db_command, remove_db_exception) - def grant_user_privilages(self, db_user:str, db_name: str): + def grant_user_privilages(self, db_user: str, db_name: str): grant_user_command = f"'GRANT ALL PRIVILEGES ON `{db_name}`.* TO `{db_user}`@`%`;'" - grant_user_exception = DatabaseServiceException(self.database_server_info.host,f'Failed to grant prvilages for user {db_user} on {db_name}.') - self.db_run_query(grant_user_command,grant_user_exception) + grant_user_exception = DatabaseServiceException( + self.database_server_info.host, f'Failed to grant prvilages for user {db_user} on {db_name}.' + ) + self.db_run_query(grant_user_command, grant_user_exception) - def add_user(self, db_user: str, db_pass: str, db_user_host: str = '%', force: bool = False, timeout = 25): + def add_user(self, db_user: str, db_pass: str, db_user_host: str = '%', force: bool = False, timeout=25): if self.check_user_exists(db_user, db_user_host): if force: - self.remove_user(db_user,db_user_host) + self.remove_user(db_user, db_user_host) else: - raise DatabaseServiceException(self.run_on_compose_service,f'User {db_user} for {db_user_host} already exists.') + raise DatabaseServiceException( + self.run_on_compose_service, f'User {db_user} for {db_user_host} already exists.' + ) add_user_command = f"'CREATE USER `{db_user}`@`%` IDENTIFIED BY \"{db_pass}\";'" - add_user_exception = DatabaseServiceException(self.database_server_info.host,f'Failed to add user {db_user}.') - self.db_run_query(add_user_command,add_user_exception) + add_user_exception = DatabaseServiceException(self.database_server_info.host, f'Failed to add user {db_user}.') + self.db_run_query(add_user_command, add_user_exception) - def db_export(self, db_name: str, export_file_path: Union[str,Path]): + def db_export(self, db_name: str, export_file_path: Union[str, Path]): if not self.check_db_exists(db_name): raise DatabaseServiceDBNotFoundError(db_name, self.run_on_compose_service) @@ -198,23 +230,25 @@ def db_export(self, db_name: str, export_file_path: Union[str,Path]): export_file_path = str(export_file_path.absolute()) db_export_command = f"mysqldump -u'{self.database_server_info.user}' -p'{self.database_server_info.password}' -h'{self.database_server_info.host}' -P{self.database_server_info.port} {db_name} --result-file={export_file_path}" + try: - output = self.compose_project.docker.compose.exec(self.run_on_compose_service, command=db_export_command, stream=False) + output = self.compose_project.docker.compose.exec( + self.run_on_compose_service, command=db_export_command, stream=False + ) except DockerException: - raise DatabaseServiceDBExportFailed(self.run_on_compose_service,db_name) + raise DatabaseServiceDBExportFailed(self.run_on_compose_service, db_name) - def db_create(self,db_name): + def db_create(self, db_name): create_db_command = f"'CREATE DATABASE IF NOT EXISTS `{db_name}`';" create_db_exception = DatabaseServiceDBCreateFailed(self.run_on_compose_service, db_name) - self.db_run_query(create_db_command,create_db_exception) + self.db_run_query(create_db_command, create_db_exception) def db_import(self, db_name: str, host_db_file_path: Path, force: bool = False): - if not self.check_db_exists(db_name): if force: self.db_create(db_name) else: - raise DatabaseServiceDBNotFoundError(db_name,self.run_on_compose_service) + raise DatabaseServiceDBNotFoundError(db_name, self.run_on_compose_service) container_db_file_name = host_db_file_path.name source = str(host_db_file_path.absolute()) @@ -222,6 +256,8 @@ def db_import(self, db_name: str, host_db_file_path: Path, force: bool = False): db_import_command = self.base_command + f" {db_name} -e 'source /tmp/{container_db_file_name}'" try: output = self.compose_project.docker.compose.cp(source, destination, stream=False) - output = self.compose_project.docker.compose.exec(self.run_on_compose_service, command=db_import_command, stream=False) + output = self.compose_project.docker.compose.exec( + self.run_on_compose_service, command=db_import_command, stream=False + ) except DockerException: - raise DatabaseServiceDBImportFailed(self.run_on_compose_service,source) + raise DatabaseServiceDBImportFailed(self.run_on_compose_service, source) From baf9360c8e66f4eb5c56fe6c37433e0a3fee2f54 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 10 May 2024 04:52:32 +0530 Subject: [PATCH 6/9] update: misc status and function docs --- frappe_manager/commands.py | 1 + frappe_manager/site_manager/SiteManager.py | 3 ++- frappe_manager/sub_commands/ssl_command.py | 4 ++++ frappe_manager/sub_commands/update_command.py | 2 ++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index 1943713f..7628ff3c 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -422,6 +422,7 @@ def logs( follow: Annotated[bool, typer.Option("--follow", "-f", help="Follow logs.")] = False, ): """Show frappe server logs or container logs for a given bench.""" + services_manager = ctx.obj["services"] verbose = ctx.obj['verbose'] bench = Bench.get_object(benchname, services_manager) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index 8665f459..bbda7486 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -61,7 +61,8 @@ def list_benches(self): if not bench_list: richprint.exit( - "Seems like you haven't created any sites yet. To create a bench, use the command: 'fm create '." + "Seems like you haven't created any sites yet. To create a bench, use the command: 'fm create '.", + emoji_code=":white_check_mark:", ) list_table = Table(show_lines=True, show_header=True, highlight=True) diff --git a/frappe_manager/sub_commands/ssl_command.py b/frappe_manager/sub_commands/ssl_command.py index 267d23b2..114f5760 100644 --- a/frappe_manager/sub_commands/ssl_command.py +++ b/frappe_manager/sub_commands/ssl_command.py @@ -21,6 +21,8 @@ def delete( ), ] = None, ): + """Delete bench ssl certficate.""" + services_manager = ctx.obj["services"] bench = Bench.get_object(benchname, services_manager) richprint.change_head("Removing SSL certificate") @@ -40,6 +42,8 @@ def renew( ] = None, all: Annotated[bool, typer.Option(help="Renew ssl cert for all benches.")] = False, ): + """Renew bench ssl certficate.""" + services_manager = ctx.obj["services"] benches = BenchesManager(CLI_BENCHES_DIRECTORY, services=services_manager) diff --git a/frappe_manager/sub_commands/update_command.py b/frappe_manager/sub_commands/update_command.py index e2d477c0..6aea10ef 100644 --- a/frappe_manager/sub_commands/update_command.py +++ b/frappe_manager/sub_commands/update_command.py @@ -37,4 +37,6 @@ def update_callback( def images( ctx: typer.Context, ): + """Pull latest FM stack docker images.""" + pull_docker_images() From 7bd3e075a85c427e6a8310f478bbcbbad67ea55e Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 10 May 2024 18:54:19 +0530 Subject: [PATCH 7/9] fix: fm code force start flag double start of bench --- frappe_manager/site_manager/site.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index a7341e5f..faf812d1 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -17,7 +17,6 @@ from frappe_manager.migration_manager.backup_manager import BackupManager from frappe_manager.services_manager.services import ServicesManager from frappe_manager.site_manager import VSCODE_LAUNCH_JSON, VSCODE_SETTINGS_JSON, VSCODE_TASKS_JSON -from frappe_manager.site_manager import bench_config from frappe_manager.site_manager.admin_tools import AdminTools from frappe_manager.site_manager.bench_config import BenchConfig, FMBenchEnvType from frappe_manager.site_manager.site_exceptions import ( @@ -415,6 +414,7 @@ def start(self, force: bool = False): self.sync_bench_common_site_config(global_db_info.host, global_db_info.port) richprint.change_head("Starting bench services") + self.admin_tools.remove_nginx_location_config() self.compose_project.start_service(force_recreate=force) self.sync_bench_config_configuration() self.save_bench_config() @@ -426,12 +426,6 @@ def start(self, force: bool = False): self.workers.compose_project.start_service(force_recreate=force) richprint.print("Started bench workers services.") - # # start admin_tools if exists - # if self.admin_tools.compose_project.compose_file_manager.exists(): - # richprint.change_head("Starting bench admin tools services") - # self.admin_tools.compose_project.start_service(force_recreate=force) - # richprint.print("Started bench admin tools services.") - richprint.change_head('Starting frappe server') self.frappe_logs_till_start() richprint.print('Started frappe server.') @@ -966,7 +960,7 @@ def attach_to_bench(self, user: str, extensions: List[str], workdir: str, debugg self.compose_project.compose_file_manager.set_labels("frappe", labels) self.compose_project.compose_file_manager.write_to_file() richprint.print(f"Regenerated bench compose.") - self.start() + self.compose_project.start_service(['frappe']) # sync debugger files if debugger: From 0b035a14079dcb235932e91214009acfbb16f466 Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 10 May 2024 19:00:24 +0530 Subject: [PATCH 8/9] fix: create command template site creation --- frappe_manager/site_manager/site.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/frappe_manager/site_manager/site.py b/frappe_manager/site_manager/site.py index faf812d1..a2bd5c5a 100644 --- a/frappe_manager/site_manager/site.py +++ b/frappe_manager/site_manager/site.py @@ -192,7 +192,11 @@ def create(self, is_template_bench: bool = False): if is_template_bench: self.remove_attached_secrets() - richprint.exit(f"Created template bench: {self.name}", emoji_code=":white_check_mark:") + global_db_info = self.services.database_manager.database_server_info + self.sync_bench_common_site_config(global_db_info.host, global_db_info.port) + self.save_bench_config() + richprint.print(f"Created template bench: {self.name}", emoji_code=":white_check_mark:") + return richprint.change_head(f"Starting bench services") self.compose_project.start_service(force_recreate=True) @@ -219,20 +223,8 @@ def create(self, is_template_bench: bool = False): self.logger.info(f"{self.name}: Bench site is active and responding.") - # self.create_certificate() - - # richprint.change_head("Configuring bench admin tools.") - - # if self.bench_config.admin_tools: - # self.sync_admin_tools_compose() - # self.restart_frappe_server() - - # richprint.print("Cofigured bench admin tools.") - self.info() - # self.save_bench_config() - if not ".localhost" in self.name: richprint.print( f"Please note that You will have to add a host entry to your system's hosts file to access the bench locally." From f289135592ff39964c24bd467ddeea495d8c8adb Mon Sep 17 00:00:00 2001 From: Xieyt Date: Fri, 10 May 2024 19:06:58 +0530 Subject: [PATCH 9/9] chore: fix status msg --- frappe_manager/migration_manager/migration_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe_manager/migration_manager/migration_base.py b/frappe_manager/migration_manager/migration_base.py index a4fdf73d..e5d3adad 100644 --- a/frappe_manager/migration_manager/migration_base.py +++ b/frappe_manager/migration_manager/migration_base.py @@ -159,7 +159,7 @@ def bench_db_backup( bench_db_info = bench.get_db_connection_info() bench_db_name = bench_db_info["name"] - richprint.change_head(f'Commencing db [blue]{bench.name}[/blue] backup') + richprint.change_head(f'Commencing db {bench.name} backup') mariadb_manager = MariaDBManager(server_db_info, services_manager.compose_project)