diff --git a/frappe_manager/commands.py b/frappe_manager/commands.py index 1943713f..7628ff3c 100644 --- a/frappe_manager/commands.py +++ b/frappe_manager/commands.py @@ -422,6 +422,7 @@ def logs( follow: Annotated[bool, typer.Option("--follow", "-f", help="Follow logs.")] = False, ): """Show frappe server logs or container logs for a given bench.""" + services_manager = ctx.obj["services"] verbose = ctx.obj['verbose'] bench = Bench.get_object(benchname, services_manager) diff --git a/frappe_manager/migration_manager/backup_manager.py b/frappe_manager/migration_manager/backup_manager.py index 0c3d109e..e0301c3e 100644 --- a/frappe_manager/migration_manager/backup_manager.py +++ b/frappe_manager/migration_manager/backup_manager.py @@ -1,21 +1,23 @@ import shutil from datetime import datetime from pathlib import Path -from frappe_manager import CLI_DIR +from frappe_manager import CLI_BENCHES_DIRECTORY, CLI_DIR from dataclasses import dataclass from typing import Optional from frappe_manager.logger import log random_strings = [] + @dataclass -class BackupData(): +class BackupData: src: Path dest: Path bench: Optional[str] = None + bench_path: Optional[Path] = None + prefix_timestamp: bool = False allow_restore: bool = True _is_restored: bool = False - prefix_timestamp = False _prefix_length: int = 5 @property @@ -27,7 +29,6 @@ def is_restored(self, v: bool) -> None: self._is_restored = v def __post_init__(self): - file_name = self.dest.name if self.prefix_timestamp: @@ -43,43 +44,49 @@ def __post_init__(self): break self.real_dest = self.dest.parent - - if self.bench: - self.real_dest = self.real_dest / self.bench - self.real_dest: Path = self.real_dest / file_name def exists(self): return self.dest.exists() -CLI_MIGARATIONS_DIR = CLI_DIR / 'migrations'/ f"{datetime.now().strftime('%d-%b-%y--%H-%M-%S')}" / 'backups' +current_migration_timestamp = f"{datetime.now().strftime('%d-%b-%y--%H-%M-%S')}" +CLI_MIGARATIONS_DIR = CLI_DIR / 'backups' / 'migrations' / current_migration_timestamp -class BackupManager(): - def __init__(self,name, base_dir: Path = CLI_MIGARATIONS_DIR): - self.root_backup_dir = base_dir - self.backup_dir = self.root_backup_dir / name +class BackupManager: + def __init__(self, name, benches_dir=CLI_BENCHES_DIRECTORY, backup_dir: Path = CLI_MIGARATIONS_DIR): + self.name = name + self.root_backup_dir: Path = backup_dir + self.benches_dir: Path = benches_dir + self.backup_dir: Path = self.root_backup_dir / self.name + self.bench_backup_dir: Path = Path('backups') / 'migrations' / current_migration_timestamp self.backups = [] self.logger = log.get_logger() # create backup dir if not exists - self.backup_dir.mkdir(parents=True,exist_ok=True) - - def backup(self, src: Path, dest: Optional[Path] = None, bench_name: Optional[str] = None, allow_restore: bool = True ): - + self.backup_dir.mkdir(parents=True, exist_ok=True) + + def backup( + self, + src: Path, + dest: Optional[Path] = None, + bench_name: Optional[str] = None, + allow_restore: bool = True, + ): if not src.exists(): return None if not dest: dest = self.backup_dir / src.name - backup_data = BackupData(src, dest, allow_restore=allow_restore) + if bench_name: + dest: Path = self.benches_dir / bench_name / self.bench_backup_dir / self.name / src.name + + backup_data = BackupData(src, dest, bench=bench_name, allow_restore=allow_restore) - if bench_name: - backup_data = BackupData(src, dest,bench=bench_name) - if not backup_data.real_dest.parent.exists(): - backup_data.real_dest.parent.mkdir(parents=True,exist_ok=True) + if not backup_data.real_dest.parent.exists(): + backup_data.real_dest.parent.mkdir(parents=True, exist_ok=True) self.logger.debug(f"Backup: {backup_data.src} => {backup_data.real_dest} ") @@ -94,7 +101,7 @@ def backup(self, src: Path, dest: Optional[Path] = None, bench_name: Optional[st return backup_data - def restore(self, backup_data, force = False): + def restore(self, backup_data, force=False): """ Restore a file from a backup. """ diff --git a/frappe_manager/migration_manager/migration_base.py b/frappe_manager/migration_manager/migration_base.py index d79d54a7..a4fdf73d 100644 --- a/frappe_manager/migration_manager/migration_base.py +++ b/frappe_manager/migration_manager/migration_base.py @@ -1,21 +1,33 @@ +from abc import ABC from logging import Logger -from typing import Protocol, runtime_checkable +from pathlib import Path +from frappe_manager import CLI_DIR from frappe_manager.migration_manager.backup_manager import BackupManager +from frappe_manager.migration_manager.migration_exections import MigrationExceptionInBench +from frappe_manager.migration_manager.migration_helpers import ( + MigrationBench, + MigrationBenches, + MigrationServicesManager, +) from frappe_manager.migration_manager.version import Version from frappe_manager.logger import log +from frappe_manager.services_manager.database_service_manager import DatabaseServerServiceInfo, MariaDBManager +from frappe_manager.display_manager.DisplayManager import richprint +from frappe_manager.utils.helpers import capture_and_format_exception -@runtime_checkable -class MigrationBase(Protocol): +# @runtime_checkable +class MigrationBase(ABC): version: Version = Version("0.0.0") + benches_dir: Path = CLI_DIR / "sites" skip: bool = False migration_executor = None - backup_manager: BackupManager # Declare the backup_manager variable - logger: Logger + logger: Logger = log.get_logger() def init(self): - self.backup_manager = BackupManager(str(self.version)) # Assign the value to backup_manager - self.logger = log.get_logger() + self.backup_manager = BackupManager(str(self.version), self.benches_dir) + self.benches_manager = MigrationBenches(self.benches_dir) + self.services_manager: MigrationServicesManager = MigrationServicesManager(services_path=CLI_DIR / 'services') def set_migration_executor(self, migration_executor): self.migration_executor = migration_executor @@ -24,7 +36,159 @@ def get_rollback_version(self): return self.version def up(self): - pass + if self.skip: + return True + + richprint.stdout.rule(f':package: [bold][blue]v{str(self.version)}[/blue][bold]') + self.logger.info(f"v{str(self.version)}: Started") + self.logger.info("-" * 40) + + self.init() + self.services_basic_backup() + self.migrate_services() + self.migrate_benches() + + self.logger.info("-" * 40) def down(self): + richprint.change_head(f"Working on v{str(self.version)} rollback.") + self.logger.info("-" * 40) + + # undo each bench + for bench_name, bench_data in self.migration_executor.migrate_benches.items(): + if not bench_data['exception']: + self.undo_bench_migrate(bench_data['object']) + + for backup in self.backup_manager.backups: + self.backup_manager.restore(backup, force=True) + # richprint.print(f'Restored {backup.bench}'s {backup.src.name}.') + + self.undo_services_migrate() + + richprint.print(f"[bold]v{str(self.version)}[/bold] rollback successfull.") + self.logger.info("-" * 40) + + def services_basic_backup(self): + if not self.services_manager.compose_project.compose_file_manager.exists(): + raise MigrationExceptionInBench( + f"Services compose at {self.services_manager.compose_project.compose_file_manager} not found." + ) + self.backup_manager.backup(self.services_manager.compose_project.compose_file_manager.compose_path) + + def migrate_services(self): + pass + + def undo_services_migrate(self): + pass + + def migrate_benches(self): + main_error = False + + # migrate each bench + for bench_name, bench_path in self.benches_manager.get_all_benches().items(): + bench = MigrationBench(name=bench_name, path=bench_path.parent) + + if bench.name in self.migration_executor.migrate_benches.keys(): + bench_info = self.migration_executor.migrate_benches[bench.name] + if bench_info['exception']: + richprint.print(f"Skipping migration for failed bench [blue]{bench.name}[/blue].") + main_error = True + continue + + self.migration_executor.set_bench_data(bench, migration_version=self.version) + try: + self.bench_basic_backup(bench) + self.migrate_bench(bench) + except Exception as e: + traceback_str = capture_and_format_exception() + self.logger.error(f"{bench.name} [ EXCEPTION TRACEBACK ]:\n {traceback_str}") + richprint.update_live() + main_error = True + self.migration_executor.set_bench_data(bench, e, self.version) + + # restore all backup files + for backup in self.backup_manager.backups: + if backup.bench == bench.name: + self.backup_manager.restore(backup, force=True) + + self.undo_bench_migrate(bench) + self.logger.info(f'Undo successfull for bench: {bench.name}') + bench.compose_project.down_service(volumes=False, timeout=5) + + if main_error: + raise MigrationExceptionInBench('') + + def bench_basic_backup(self, bench: MigrationBench): + richprint.print(f"Migrating bench [bold][blue]{bench.name}[/blue][/bold]") + + # backup docker compose.yml + self.backup_manager.backup(bench.path / "docker-compose.yml", bench_name=bench.name) + + # backup common_site_config.json + bench_common_site_config = bench.path / "workspace" / "frappe-bench" / "sites" / "common_site_config.json" + self.backup_manager.backup(bench_common_site_config, bench_name=bench.name) + + # backup site_config.json + bench_site_config = bench.path / "workspace" / "frappe-bench" / "sites" / bench.name / "site_config.json" + self.backup_manager.backup(bench_site_config, bench_name=bench.name) + + server_db_info: DatabaseServerServiceInfo = DatabaseServerServiceInfo.import_from_compose_file( + 'global-db', self.services_manager.compose_project + ) + + self.bench_db_backup( + bench=bench, + server_db_info=server_db_info, + services_manager=self.services_manager, + backup_manager=self.backup_manager, + ) + + def migrate_bench(self, bench: MigrationBench): pass + + def undo_bench_migrate(self, bench: MigrationBench): + pass + + def bench_db_backup( + self, + bench: MigrationBench, + server_db_info: DatabaseServerServiceInfo, + services_manager: MigrationServicesManager, + backup_manager: BackupManager, + ): + bench_db_info = bench.get_db_connection_info() + bench_db_name = bench_db_info["name"] + + richprint.change_head(f'Commencing db [blue]{bench.name}[/blue] backup') + + mariadb_manager = MariaDBManager(server_db_info, services_manager.compose_project) + + from datetime import datetime + + current_datetime = datetime.now() + formatted_date = current_datetime.strftime("%d-%m-%Y--%H-%M-%S") + + container_backup_dir: Path = Path("/var/log/mysql") + host_backup_dir: Path = services_manager.services_path / 'mariadb' / 'logs' + db_sql_file_name = f"db-{bench.name}-{formatted_date}.sql" + + host_db_sql_file_path: Path = host_backup_dir / db_sql_file_name + container_db_sql_file_path: Path = container_backup_dir / db_sql_file_name + + backup_gz_file_backup_data_path: Path = ( + bench.path / backup_manager.bench_backup_dir / self.version.version / f'{db_sql_file_name}.gz' + ) + + mariadb_manager.db_export(bench_db_name, container_db_sql_file_path) + + import gzip + import shutil + + # Compress the file using gzip + with open(host_db_sql_file_path, 'rb') as f_in: + with gzip.open(backup_gz_file_backup_data_path, 'wb') as f_out: + shutil.copyfileobj(f_in, f_out) + + host_db_sql_file_path.unlink() + + richprint.print(f'[blue]{bench.name}[/blue] db backup completed successfully.') diff --git a/frappe_manager/migration_manager/migration_executor.py b/frappe_manager/migration_manager/migration_executor.py index 9ddc8533..d693294e 100644 --- a/frappe_manager/migration_manager/migration_executor.py +++ b/frappe_manager/migration_manager/migration_executor.py @@ -140,28 +140,26 @@ def execute(self): except MigrationExceptionInBench as e: if self.migrate_benches: - print_head = True + passed_print_head = True + for bench, bench_status in self.migrate_benches.items(): if not bench_status["exception"]: - if print_head: - richprint.stdout.rule('[bold][red]Migration Passed Benches[/red][bold]', style='green') - print_head = False + if passed_print_head: + richprint.stdout.rule('[bold]Migration Passed Benches[bold]', style='green') + passed_print_head = False richprint.print(f"[green]Bench[/green]: {bench}", emoji_code=':construction:') - if not print_head: - richprint.stdout.rule(style='red') - - print_head = True + failed_print_head = True for bench, bench_status in self.migrate_benches.items(): if bench_status["exception"]: - if print_head: + if failed_print_head: richprint.stdout.rule( ':police_car_light: [bold][red]Migration Failed Benches[/red][bold] :police_car_light:', style='red', ) - print_head = False + failed_print_head = False richprint.error(f"[red]Bench[/red]: {bench}", emoji_code=':construction:') @@ -178,8 +176,11 @@ def execute(self): richprint.print(f"For error specifics, refer to {CLI_DIR}/logs/fm.log", emoji_code=':page_facing_up:') - if not print_head: + if not failed_print_head: richprint.stdout.rule(style='red') + else: + if not passed_print_head: + richprint.stdout.rule(style='green') archive_msg = [ 'Available options after migrations failure :', diff --git a/frappe_manager/migration_manager/migrations/migrate_0_10_0.py b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py index 0262744c..b11b3576 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_10_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_10_0.py @@ -31,31 +31,36 @@ from frappe_manager.utils.docker import host_run_cp from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.utils.helpers import get_container_name_prefix, get_unix_groups, random_password_generate +from frappe_manager.migration_manager.backup_manager import BackupManager from frappe_manager.migration_manager.version import Version class MigrationV0100(MigrationBase): version = Version("0.10.0") - def __init__(self): - super().init() + def init(self): self.benches_dir = CLI_DIR / "sites" + self.backup_manager = BackupManager(str(self.version), self.benches_dir) self.string_timestamp = datetime.now().strftime("%d-%b-%y--%H-%M-%S") self.benches_manager = MigrationBenches(self.benches_dir) + self.services_manager: MigrationServicesManager = MigrationServicesManager() def get_rollback_version(self): # this was without any migrations return Version("0.10.1") def up(self): - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") + richprint.stdout.rule(f':package: [bold][blue]v{str(self.version)}[/blue][bold]') + self.logger.info(f"v{str(self.version)}: Started") self.logger.info("-" * 40) - self.services_manager: MigrationServicesManager = MigrationServicesManager() + self.init() + self.migrate_services() + self.migrate_benches() - # stop all benches - self.benches_manager.stop_benches(timeout=20) + self.logger.info("-" * 40) + def migrate_services(self): # create services self.services_create(self.services_manager.compose_project) self.services_manager.compose_project.pull_images() @@ -66,46 +71,46 @@ def up(self): self.services_manager.compose_project, ) - # migrate each bench + def migrate_benches(self): main_error = False - benches = self.benches_manager.get_all_benches() + # migrate each bench + for bench_name, bench_path in self.benches_manager.get_all_benches().items(): + bench = MigrationBench(name=bench_name, path=bench_path.parent) - for bench_name, bench_compose_path in benches.items(): - bench = MigrationBench(name=bench_name, path=bench_compose_path.parent) + if bench.name in self.migration_executor.migrate_benches.keys(): + bench_info = self.migration_executor.migrate_benches[bench.name] + if bench_info['exception']: + richprint.print(f"Skipping migration for failed bench {bench.name}.") + main_error = True + continue self.migration_executor.set_bench_data(bench, migration_version=self.version) - try: self.migrate_bench(bench) except Exception as e: import traceback traceback_str = traceback.format_exc() - self.logger.error(f"[ EXCEPTION TRACEBACK ]:\n {traceback_str}") + self.logger.error(f"{bench.name} [ EXCEPTION TRACEBACK ]:\n {traceback_str}") richprint.update_live() main_error = True self.migration_executor.set_bench_data(bench, e, self.version) + + # restore all backup files + for backup in self.backup_manager.backups: + if backup.bench == bench.name: + self.backup_manager.restore(backup, force=True) + self.undo_bench_migrate(bench) + self.logger.info(f'Undo successfull for bench: {bench.name}') bench.compose_project.down_service(volumes=False, timeout=5) if main_error: - raise MigrationExceptionInBench("") - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) + raise MigrationExceptionInBench('') def migrate_bench(self, bench: MigrationBench): - richprint.print(f"Migrating bench {bench.name}", prefix=f"[bold]v{str(self.version)}:[/bold] ") - - # backup docker compose.yml - self.backup_manager.backup(bench.path / "docker-compose.yml", bench_name=bench.name) - - # backup common_site_config.json - self.backup_manager.backup( - bench.path / "workspace" / "frappe-bench" / "sites" / "common_site_config.json", - bench_name=bench.name, - ) + richprint.change_head("Migrating bench compose") bench.compose_project.down_service(volumes=False) @@ -121,31 +126,12 @@ def migrate_bench(self, bench: MigrationBench): self.services_database_manager.add_user(bench_db_user, bench_db_pass, force=True) self.services_database_manager.grant_user_privilages(bench_db_name, bench_db_user) - def down(self): - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) - + def undo_services_migrate(self): self.services_manager.compose_project.down_service() - if self.services_manager.services_path.exists(): shutil.rmtree(self.services_manager.services_path) - # undo each bench - for bench, exception in self.migration_executor.migrate_benches.items(): - if not exception: - self.undo_bench_migrate(bench) - - for backup in self.backup_manager.backups: - self.backup_manager.restore(backup, force=True) - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) - def undo_bench_migrate(self, bench: MigrationBench): - for backup in self.backup_manager.backups: - if backup.bench == bench.name: - self.backup_manager.restore(backup, force=True) - configs_backup = bench.path / f"configs-{self.string_timestamp}.bak" configs_path = bench.path / "configs" @@ -179,22 +165,21 @@ def undo_bench_migrate(self, bench: MigrationBench): def migrate_bench_compose(self, bench: MigrationBench): richprint.change_head("Migrating database") - compose_version = bench.compose_project.compose_file_manager.get_version() - if not bench.compose_project.compose_file_manager.exists(): raise BenchDockerComposeFileNotFound(bench.name, bench.compose_project.compose_file_manager.compose_path) db_backup_file = self.db_migration_export(bench) - richprint.print("Database exported") + + richprint.print(f"[blue]{bench.name}[/blue] db exported from bench mariadb service.") # backup bench db db_backup = self.backup_manager.backup(db_backup_file, bench_name=bench.name, allow_restore=False) self.db_migration_import(bench=bench, db_backup_file=db_backup) - richprint.print("Database Imported") - status_msg = "Migrating bench compose" - richprint.change_head(status_msg) + richprint.print(f"[blue]{bench.name}[/blue] db imported to global-db service.") + + richprint.change_head("Migrating bench compose") # get all the payloads envs = bench.compose_project.compose_file_manager.get_all_envs() @@ -250,11 +235,13 @@ def migrate_bench_compose(self, bench: MigrationBench): # change the node socketio port bench.common_bench_config_set({"socketio_port": "80"}) - richprint.print(f"{status_msg} {compose_version} -> {self.version.version}: Done") + richprint.print(f"Migrated [blue]{bench.name}[/blue] compose file.") return db_backup def create_compose_dirs(self, bench: MigrationBench): + richprint.change_head("Creating services compose dirs.") + # directory creation configs_path = bench.path / "configs" @@ -291,14 +278,15 @@ def create_compose_dirs(self, bench: MigrationBench): new_dir = nginx_dir / directory new_dir.mkdir(parents=True, exist_ok=True) + richprint.print("Created services compose dirs.") + def db_migration_export(self, bench: MigrationBench) -> Path: self.logger.debug("[db export] bench: %s", bench.name) # start the benc hand also handle the missing docker images output = bench.compose_project.docker.compose.up( - services=["mariadb", "frappe"], detach=True, pull="missing", stream=True + services=["mariadb", "frappe"], detach=True, pull="missing", stream=False ) - richprint.live_lines(output, padding=(0, 0, 0, 2)) self.logger.debug("[db export] checking if mariadb started") @@ -348,7 +336,11 @@ def db_migration_import(self, bench: MigrationBench, db_backup_file: BackupData) self.services_database_manager.grant_user_privilages(bench_db_user, bench_db_name) + richprint.print(f"{bench.name} db imported.") + def services_create(self, services_compose_project: ComposeProject): + richprint.change_head("Creating services.") + envs = { "global-db": { "MYSQL_ROOT_PASSWORD_FILE": '/run/secrets/db_root_password', @@ -442,8 +434,10 @@ def services_create(self, services_compose_project: ComposeProject): services_compose_project.docker.compose.down(remove_orphans=True, timeout=1, volumes=True, stream=True) + richprint.print(f"Created services at {self.services_manager.services_path}.") + def generate_compose(self, inputs: dict, compose_file_manager: ComposeFile): - # TODO do something about this function + richprint.change_head(f"Generating services compose file.") try: # handle environment if "environment" in inputs.keys(): @@ -462,5 +456,6 @@ def generate_compose(self, inputs: dict, compose_file_manager: ComposeFile): uid = user[container_name]["uid"] gid = user[container_name]["gid"] compose_file_manager.set_user(container_name, uid, gid) + richprint.print(f"Generated services compose file.") except Exception: raise ServicesNotCreated() diff --git a/frappe_manager/migration_manager/migrations/migrate_0_11_0.py b/frappe_manager/migration_manager/migrations/migrate_0_11_0.py index bb95f5a7..f0759a01 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_11_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_11_0.py @@ -1,113 +1,37 @@ +from pathlib import Path from frappe_manager.migration_manager.migration_base import MigrationBase from frappe_manager.migration_manager.migration_exections import MigrationExceptionInBench -from frappe_manager.migration_manager.migration_helpers import MigrationBench, MigrationBenches +from frappe_manager.migration_manager.migration_helpers import ( + MigrationBench, + MigrationBenches, + MigrationServicesManager, +) from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.migration_manager.version import Version -from frappe_manager import CLI_DIR, CLI_SERVICES_DIRECTORY +from frappe_manager.migration_manager.backup_manager import BackupManager + class MigrationV0110(MigrationBase): version = Version("0.11.0") - def __init__(self): - super().init() - self.benches_dir = CLI_DIR / "sites" - self.services_path = CLI_SERVICES_DIRECTORY + def init(self): + self.cli_dir: Path = Path.home() / 'frappe' + self.benches_dir = self.cli_dir / "sites" + self.backup_manager = BackupManager(str(self.version), self.benches_dir) self.benches_manager = MigrationBenches(self.benches_dir) - - def up(self): - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - - benches = self.benches_manager.get_all_benches() - - main_error = False - - # migrate each bench - for bench_name, bench_path in benches.items(): - bench = MigrationBench(name=bench_name, path=bench_path.parent) - - if bench.name in self.migration_executor.migrate_benches.keys(): - bench_info = self.migration_executor.migrate_benches[bench.name] - if bench_info['exception']: - richprint.print(f"Skipping migration for failed bench{bench.name}.") - main_error = True - continue - - self.migration_executor.set_bench_data(bench,migration_version=self.version) - try: - self.migrate_bench(bench) - except Exception as e: - import traceback - traceback_str = traceback.format_exc() - self.logger.error(f"{bench.name} [ EXCEPTION TRACEBACK ]:\n {traceback_str}") - richprint.update_live() - main_error = True - self.migration_executor.set_bench_data(bench, e, self.version) - self.undo_bench_migrate(bench) - bench.compose_project.down_service(volumes=False, timeout=5) - - if main_error: - raise MigrationExceptionInBench('') - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - - def migrate_bench(self, bench: MigrationBench): - richprint.print(f"Migrating bench {bench.name}", prefix=f"[bold]v{str(self.version)}:[/bold] ") - - # backup docker compose.yml - self.backup_manager.backup(bench.path / "docker-compose.yml", bench_name=bench.name) - - # backup common_site_config.json - self.backup_manager.backup( - bench.path - / "workspace" - / "frappe-bench" - / "sites" - / "common_site_config.json", - bench_name=bench.name, + self.services_manager: MigrationServicesManager = MigrationServicesManager( + services_path=self.cli_dir / 'services' ) + def migrate_bench(self, bench: MigrationBench): bench.compose_project.down_service(volumes=False) self.migrate_bench_compose(bench) - def down(self): - # richprint.print(f"Started",prefix=f"[ Migration v{str(self.version)} ][ROLLBACK] : ") - richprint.print( - f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] " - ) - self.logger.info("-" * 40) - - # undo each bench - for bench, exception in self.migration_executor.migrate_benches.items(): - if not exception: - self.undo_bench_migrate(bench) - - for backup in self.backup_manager.backups: - self.backup_manager.restore(backup, force=True) - - richprint.print( - f"Successfull", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] " - ) - self.logger.info("-" * 40) - - def undo_bench_migrate(self, bench: MigrationBench): - - for backup in self.backup_manager.backups: - if backup.bench == bench.name: - self.backup_manager.restore(backup, force=True) - - self.logger.info(f'Undo successfull for bench: {bench.name}') - def migrate_bench_compose(self, bench: MigrationBench): - - status_msg = 'Migrating bench compose' - richprint.change_head(status_msg) - - compose_version = bench.compose_project.compose_file_manager.get_version() + richprint.change_head("Migrating bench compose") if not bench.compose_project.compose_file_manager.exists(): - richprint.print(f"{status_msg} {compose_version} -> {self.version}: Failed ") + richprint.print(f"Failed to migrate {bench.name} compose file.") raise MigrationExceptionInBench(f"{bench.compose_project.compose_file_manager.compose_path} not found.") # change image tag to the latest @@ -119,7 +43,9 @@ def migrate_bench_compose(self, bench: MigrationBench): image_info['tag'] = self.version.version_string() image_info['name'] = 'ghcr.io/rtcamp/frappe-manager-frappe' - output = bench.compose_project.docker.pull(container_name=f"{image_info['name']}:{image_info['tag']}", stream=True) + output = bench.compose_project.docker.pull( + container_name=f"{image_info['name']}:{image_info['tag']}", stream=True + ) richprint.live_lines(output, padding=(0, 0, 0, 2)) bench.compose_project.compose_file_manager.set_all_images(images_info) @@ -127,4 +53,4 @@ def migrate_bench_compose(self, bench: MigrationBench): bench.compose_project.compose_file_manager.set_version(str(self.version)) bench.compose_project.compose_file_manager.write_to_file() - richprint.print(f"{status_msg} {compose_version} -> {self.version}: Done") + richprint.print(f"Migrated [blue]{bench.name}[/blue] compose file.") diff --git a/frappe_manager/migration_manager/migrations/migrate_0_12_0.py b/frappe_manager/migration_manager/migrations/migrate_0_12_0.py index a9f29da9..b3362811 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_12_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_12_0.py @@ -1,30 +1,31 @@ +from pathlib import Path from frappe_manager.migration_manager.migration_base import MigrationBase from frappe_manager.docker_wrapper.DockerClient import DockerClient from frappe_manager.migration_manager.migration_exections import MigrationExceptionInBench -from frappe_manager.migration_manager.migration_helpers import MigrationBench, MigrationBenches +from frappe_manager.migration_manager.migration_helpers import ( + MigrationBench, + MigrationBenches, + MigrationServicesManager, +) from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.migration_manager.version import Version -from frappe_manager import CLI_DIR from frappe_manager.utils.helpers import get_container_name_prefix +from frappe_manager.migration_manager.backup_manager import BackupManager class MigrationV0120(MigrationBase): version = Version("0.12.0") - def __init__(self): - super().init() - self.benches_dir = CLI_DIR / "sites" + def init(self): + self.cli_dir: Path = Path.home() / 'frappe' + self.benches_dir = self.cli_dir / "sites" + self.backup_manager = BackupManager(str(self.version), self.benches_dir) self.benches_manager = MigrationBenches(self.benches_dir) + self.services_manager: MigrationServicesManager = MigrationServicesManager( + services_path=self.cli_dir / 'services' + ) - def up(self): - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - - # take backup of each of the bench docker compose - self.benches_manager.stop_benches() - - benches = self.benches_manager.get_all_benches() - + def migrate_services(self): # Pulling latest image self.image_info = {"tag": self.version.version_string(), "name": "ghcr.io/rtcamp/frappe-manager-frappe"} pull_image = f"{self.image_info['name']}:{self.image_info['tag']}" @@ -34,87 +35,16 @@ def up(self): richprint.live_lines(output, padding=(0, 0, 0, 2)) richprint.print(f"Image pulled [blue]{pull_image}[/blue]") - # migrate each bench - main_error = False - - # migrate each bench - for bench_name, bench_path in benches.items(): - bench = MigrationBench(name=bench_name, path=bench_path.parent) - - if bench.name in self.migration_executor.migrate_benches.keys(): - bench_info = self.migration_executor.migrate_benches[bench.name] - if bench_info['exception']: - richprint.print(f"Skipping migration for failed bench{bench.name}.") - main_error = True - continue - - self.migration_executor.set_bench_data(bench, migration_version=self.version) - try: - self.migrate_bench(bench) - except Exception as e: - import traceback - - traceback_str = traceback.format_exc() - self.logger.error(f"{bench.name} [ EXCEPTION TRACEBACK ]:\n {traceback_str}") - richprint.update_live() - main_error = True - self.migration_executor.set_bench_data(bench, e, self.version) - self.undo_bench_migrate(bench) - bench.compose_project.down_service(volumes=False, timeout=5) - - if main_error: - raise MigrationExceptionInBench('') - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - def migrate_bench(self, bench: MigrationBench): - richprint.print(f"Migrating bench {bench.name}", prefix=f"[bold]v{str(self.version)}:[/bold] ") - - # backup docker compose.yml - self.backup_manager.backup(bench.path / "docker-compose.yml", bench_name=bench.name) - - # backup common_site_config.json - self.backup_manager.backup( - bench.path / "workspace" / "frappe-bench" / "sites" / "common_site_config.json", - bench_name=bench.name, - ) - bench.compose_project.down_service(volumes=False) self.migrate_bench_compose(bench) self.migrate_workers_compose(bench) - def down(self): - # richprint.print(f"Started",prefix=f"[ Migration v{str(self.version)} ][ROLLBACK] : ") - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) - - # undo each bench - for bench, exception in self.migration_executor.migrate_benches.items(): - if not exception: - self.undo_bench_migrate(bench) - - for backup in self.backup_manager.backups: - self.backup_manager.restore(backup, force=True) - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) - - def undo_bench_migrate(self, bench: MigrationBench): - for backup in self.backup_manager.backups: - if backup.bench == bench.name: - self.backup_manager.restore(backup, force=True) - - self.logger.info(f"Undo successfull for bench: {bench.name}") - def migrate_bench_compose(self, bench: MigrationBench): - status_msg = "Migrating bench compose" - richprint.change_head(status_msg) - - compose_version = bench.compose_project.compose_file_manager.get_version() + richprint.change_head("Migrating bench compose") if not bench.compose_project.compose_file_manager.exists(): - richprint.print(f"{status_msg} {compose_version} -> {self.version}: Failed ") + richprint.print(f"Failed to migrate {bench.name} compose file.") raise MigrationExceptionInBench(f"{bench.compose_project.compose_file_manager.compose_path} not found.") images_info = bench.compose_project.compose_file_manager.get_all_images() @@ -130,19 +60,23 @@ def migrate_bench_compose(self, bench: MigrationBench): try: del compose_yml["services"][service]["restart"] except KeyError as e: - self.logger.error(f"{bench.name}: Not able to delete restart: always attribute from compose file.{e}") + self.logger.error( + f"{bench.name} worker not able to delete 'restart: always' attribute from compose file.{e}" + ) pass - richprint.print("Removed [blue]restart: always[/blue]") + richprint.print("Removed 'restart: always'.") bench.compose_project.compose_file_manager.set_version(str(self.version)) bench.compose_project.compose_file_manager.set_all_images(images_info) bench.compose_project.compose_file_manager.write_to_file() - richprint.print(f"{status_msg} {compose_version} -> {self.version}: Done") + + richprint.print(f"Migrated {bench.name} compose file.") def migrate_workers_compose(self, bench: MigrationBench): if bench.workers_compose_project.compose_file_manager.compose_path.exists(): richprint.print("Migrating workers compose") + # workers image set workers_info = bench.workers_compose_project.compose_file_manager.get_all_images() @@ -155,7 +89,7 @@ def migrate_workers_compose(self, bench: MigrationBench): del worker_compose_yml["services"][service]["restart"] except KeyError as e: self.logger.error( - f"{bench.name} worker: Not able to delete restart: always attribute from compose file.{e}" + f"{bench.name} worker not able to delete 'restart: always' attribute from compose file.{e}" ) pass @@ -168,4 +102,5 @@ def migrate_workers_compose(self, bench: MigrationBench): get_container_name_prefix(bench.name) ) bench.workers_compose_project.compose_file_manager.write_to_file() - richprint.print("Migrating workers compose: Done") + + richprint.print(f"Migrated [blue]{bench.name}[/blue] compose file.") diff --git a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py index 6e9b34da..52047f84 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_13_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_13_0.py @@ -1,6 +1,7 @@ import json import os import copy +from pathlib import Path from frappe_manager.compose_manager import DockerVolumeMount from frappe_manager.compose_manager.ComposeFile import ComposeFile from frappe_manager.migration_manager.migration_base import MigrationBase @@ -16,18 +17,22 @@ from frappe_manager.site_manager.bench_config import BenchConfig, FMBenchEnvType from frappe_manager.ssl_manager import SUPPORTED_SSL_TYPES from frappe_manager.ssl_manager.certificate import SSLCertificate -from frappe_manager.utils.helpers import capture_and_format_exception, get_container_name_prefix +from frappe_manager.utils.helpers import get_container_name_prefix from frappe_manager.docker_wrapper.DockerClient import DockerClient +from frappe_manager.migration_manager.backup_manager import BackupManager class MigrationV0130(MigrationBase): version = Version("0.13.0") - def __init__(self): - super().init() - self.benches_dir = CLI_DIR / "sites" - self.services_path = CLI_SERVICES_DIRECTORY + def init(self): + self.cli_dir: Path = Path.home() / 'frappe' + self.benches_dir = self.cli_dir / "sites" + self.backup_manager = BackupManager(str(self.version), self.benches_dir) self.benches_manager = MigrationBenches(self.benches_dir) + self.services_manager: MigrationServicesManager = MigrationServicesManager( + services_path=self.cli_dir / 'services' + ) def migrate_services(self): # backup services compose @@ -36,14 +41,13 @@ def migrate_services(self): f"Services compose at {self.services_manager.compose_project.compose_file_manager} not found." ) - self.backup_manager.backup( - self.services_manager.compose_project.compose_file_manager.compose_path / "docker-compose.yml" - ) + self.backup_manager.backup(self.services_manager.compose_project.compose_file_manager.compose_path) + # remove version from services yml try: del self.services_manager.compose_project.compose_file_manager.yml['version'] except KeyError: - self.logger.warning(f"[services]: version attribute not found compose.") + self.logger.warning(f"[services]: 'version' attribute not found in compose file.") pass # include new volume info @@ -73,68 +77,14 @@ def migrate_services(self): if self.services_manager.compose_project.is_service_running('global-nginx-proxy'): self.services_manager.compose_project.docker.compose.up(services=['global-nginx-proxy']) - def up(self): - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - - self.services_manager: MigrationServicesManager = MigrationServicesManager() - - self.benches_manager.stop_benches() - - benches = self.benches_manager.get_all_benches() - - # migrate services - richprint.change_head("Migrating services") - self.migrate_services() - richprint.print("Migrating services: Done") - # rename main config fm_config_path = CLI_DIR / 'fm_config.toml' old_fm_config_path = CLI_DIR / '.fm.toml' if old_fm_config_path.exists(): old_fm_config_path.rename(fm_config_path) - # migrate each bench - main_error = False - - # migrate each bench - for bench_name, bench_path in benches.items(): - bench = MigrationBench(name=bench_name, path=bench_path.parent) - - if bench.name in self.migration_executor.migrate_benches.keys(): - bench_info = self.migration_executor.migrate_benches[bench.name] - if bench_info['exception']: - richprint.print(f"Skipping migration for failed bench{bench.name}.") - main_error = True - continue - - self.migration_executor.set_bench_data(bench, migration_version=self.version) - try: - self.migrate_bench(bench) - except Exception as e: - exception_str = capture_and_format_exception() - self.logger.error(f"{bench.name} [ EXCEPTION TRACEBACK ]:\n {exception_str}") - richprint.update_live() - main_error = True - self.migration_executor.set_bench_data(bench, e, self.version) - self.undo_bench_migrate(bench) - bench.compose_project.down_service(volumes=False, timeout=5) - - if main_error: - raise MigrationExceptionInBench('') - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - def migrate_bench(self, bench: MigrationBench): - richprint.print(f"Migrating bench {bench.name}", prefix=f"[bold]v{str(self.version)}:[/bold] ") - - # backup docker compose.yml - self.backup_manager.backup(bench.path / "docker-compose.yml", bench_name=bench.name) - - # backup common_site_config.json bench_common_site_config = bench.path / "workspace" / "frappe-bench" / "sites" / "common_site_config.json" - self.backup_manager.backup(bench_common_site_config, bench_name=bench.name) common_site_config_json = json.loads(bench_common_site_config.read_bytes()) if 'mail_server' in common_site_config_json: @@ -144,48 +94,23 @@ def migrate_bench(self, bench: MigrationBench): bench.compose_project.down_service(volumes=False) self.migrate_bench_compose(bench) - def down(self): - # richprint.print(f"Started",prefix=f"[ Migration v{str(self.version)} ][ROLLBACK] : ") - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) - - # undo each bench - for bench, exception in self.migration_executor.migrate_benches.items(): - if not exception: - self.undo_bench_migrate(bench) - - for backup in self.backup_manager.backups: - self.backup_manager.restore(backup, force=True) - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) - def undo_bench_migrate(self, bench: MigrationBench): - for backup in self.backup_manager.backups: - if backup.bench == bench.name: - self.backup_manager.restore(backup, force=True) + richprint.change_head("Removing Admin Tools compose file") admin_tools_compose_path = bench.path / 'docker-compose.admin-tools.yml' if admin_tools_compose_path.exists(): admin_tools_compose_path.unlink() - self.logger.info(f"Undo successfull for bench: {bench.name}") + richprint.print("Removed Admin Tools compose file") def migrate_bench_compose(self, bench: MigrationBench): - status_msg = "Migrating bench compose" - richprint.change_head(status_msg) - - compose_version = bench.compose_project.compose_file_manager.get_version() + richprint.change_head("Migrating bench compose") if not bench.compose_project.compose_file_manager.exists(): - richprint.print(f"{status_msg} {compose_version} -> {self.version.version}: Failed ") + richprint.print(f"Failed to migrate {bench.name} compose file.") raise MigrationExceptionInBench(f"{bench.compose_project.compose_file_manager.compose_path} not found.") - # generate bench config for bench and save it - status_msg = "Migrating bench compose" - richprint.change_head(status_msg) - # get all the payloads envs = bench.compose_project.compose_file_manager.get_all_envs() @@ -196,7 +121,7 @@ def migrate_bench_compose(self, bench: MigrationBench): try: del envs['nginx']['ENABLE_SSL'] except KeyError: - self.logger.warning(f"{bench.name}: ENABLE_SSL nginx's env not found.") + self.logger.warning(f"{bench.name} 'ENABLE_SSL' nginx's env not found.") pass # create new html in configs/nginx/html compose directory @@ -278,7 +203,7 @@ def migrate_bench_compose(self, bench: MigrationBench): try: del bench.compose_project.compose_file_manager.yml['version'] except KeyError: - self.logger.warning(f"{bench.name}: version attribute not found compose.") + self.logger.warning(f"{bench.name} 'version' attribute not found in compose file.") pass # include new volume info @@ -300,9 +225,7 @@ def migrate_bench_compose(self, bench: MigrationBench): bench.compose_project.compose_file_manager.set_version(str(self.version)) bench.compose_project.compose_file_manager.write_to_file() - bench.compose_project.compose_file_manager.set_all_images(images_info) - - richprint.print(f"{status_msg} {compose_version} -> {self.version.version}: Done") + richprint.print(f"Migrated [blue]{bench.name}[/blue] compose file.") def migrate_workers_compose(self, bench: MigrationBench): if bench.workers_compose_project.compose_file_manager.compose_path.exists(): @@ -315,7 +238,7 @@ def migrate_workers_compose(self, bench: MigrationBench): try: del bench.workers_compose_project.compose_file_manager.yml['version'] except KeyError: - self.logger.warning(f"{bench.name} workers: version attribute not found compose.") + self.logger.warning(f"{bench.name} workers 'version' attribute not found in compose file.") pass bench.workers_compose_project.compose_file_manager.set_top_networks_name( @@ -326,10 +249,11 @@ def migrate_workers_compose(self, bench: MigrationBench): ) bench.compose_project.compose_file_manager.set_version(str(self.version)) bench.workers_compose_project.compose_file_manager.write_to_file() - richprint.print("Migrating workers compose: Done") + + richprint.print(f"Migrated [blue]{bench.name}[/blue] workers compose file.") def migrate_admin_tools_compose(self, bench: MigrationBench): - richprint.change_head("Create admin-tools") + richprint.change_head("Create Admin Tools") bench_compose_yml = copy.deepcopy(bench.compose_project.compose_file_manager.yml) @@ -387,4 +311,4 @@ def migrate_admin_tools_compose(self, bench: MigrationBench): common_bench_config_path.write_text(json.dumps(current_common_site_config)) - richprint.change_head("Created Admin-tools.") + richprint.change_head(f"Created {bench.name} Admin Tools.") diff --git a/frappe_manager/migration_manager/migrations/migrate_0_13_1.py b/frappe_manager/migration_manager/migrations/migrate_0_13_1.py index 6a147172..e7f3b9c0 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_13_1.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_13_1.py @@ -1,35 +1,37 @@ +from pathlib import Path from frappe_manager.migration_manager.migration_base import MigrationBase from frappe_manager.migration_manager.migration_exections import MigrationExceptionInBench from frappe_manager.migration_manager.migration_helpers import ( + MigrationBenches, MigrationServicesManager, ) from frappe_manager.display_manager.DisplayManager import richprint from frappe_manager.migration_manager.version import Version -from frappe_manager import CLI_DIR, CLI_SERVICES_DIRECTORY +from frappe_manager.migration_manager.backup_manager import BackupManager class MigrationV0131(MigrationBase): version = Version("0.13.1") - def __init__(self): - super().init() - self.benches_dir = CLI_DIR / "sites" - self.services_path = CLI_SERVICES_DIRECTORY - - def up(self): - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - - self.services_manager: MigrationServicesManager = MigrationServicesManager() + def init(self): + self.cli_dir: Path = Path.home() / 'frappe' + self.benches_dir = self.cli_dir / "sites" + self.backup_manager = BackupManager(str(self.version), self.benches_dir) + self.benches_manager = MigrationBenches(self.benches_dir) + self.services_manager: MigrationServicesManager = MigrationServicesManager( + services_path=self.cli_dir / 'services' + ) + def migrate_services(self): if not self.services_manager.compose_project.compose_file_manager.exists(): raise MigrationExceptionInBench( f"Services compose at {self.services_manager.compose_project.compose_file_manager} not found." ) richprint.change_head("Adding fm header config to nginx-proxy") + # create file fmheaders.conf - fm_headers_conf_path = self.services_path / 'nginx-proxy' / 'confd' / 'fm_headers.conf' + fm_headers_conf_path = self.services_manager.services_path / 'nginx-proxy' / 'confd' / 'fm_headers.conf' add_header = f'add_header X-Powered-By "Frappe-Manager {self.version.version_string()}";' fm_headers_conf_path.write_text(add_header) @@ -39,17 +41,3 @@ def up(self): self.services_manager.compose_project.docker.compose.restart(services=['global-nginx-proxy'], stream=False) richprint.print("Added fm header config to nginx-proxy.") - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info("-" * 40) - - def down(self): - # richprint.print(f"Started",prefix=f"[ Migration v{str(self.version)} ][ROLLBACK] : ") - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) - - for backup in self.backup_manager.backups: - self.backup_manager.restore(backup, force=True) - - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") - self.logger.info("-" * 40) diff --git a/frappe_manager/migration_manager/migrations/migrate_0_9_0.py b/frappe_manager/migration_manager/migrations/migrate_0_9_0.py index 1b4218b1..72204c00 100644 --- a/frappe_manager/migration_manager/migrations/migrate_0_9_0.py +++ b/frappe_manager/migration_manager/migrations/migrate_0_9_0.py @@ -1,4 +1,5 @@ import shutil +from frappe_manager.migration_manager.backup_manager import BackupManager from frappe_manager.migration_manager.migration_base import MigrationBase from frappe_manager import CLI_DIR from frappe_manager.migration_manager.migration_helpers import MigrationBenches @@ -9,18 +10,19 @@ class MigrationV090(MigrationBase): version = Version("0.9.0") - def __init__(self): - super().init() - self.bences_dir = CLI_DIR / "sites" + def init(self): + self.benches_dir = CLI_DIR / "sites" + self.backup_manager = BackupManager(str(self.version)) - if self.bences_dir.exists(): + if self.benches_dir.exists(): self.skip = True def up(self): if self.skip: return True - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)}:[/bold] ") + richprint.stdout.rule(f':package: [bold][blue]v{str(self.version)}[/blue][bold]') + self.logger.info(f"v{str(self.version)}: Started") self.logger.info("-" * 40) move_directory_list = [] @@ -32,40 +34,33 @@ def up(self): if docker_compose_path.exists(): move_directory_list.append(site_dir) - self.bences_dir.mkdir(parents=True, exist_ok=True) + self.benches_dir.mkdir(parents=True, exist_ok=True) - benches = MigrationBenches(self.bences_dir) + benches = MigrationBenches(self.benches_dir) benches.stop_benches() # move all the directories - richprint.print( - f"Moving benches from {CLI_DIR} to {self.bences_dir}", prefix=f"[bold]v{str(self.version)}:[/bold] " - ) + richprint.change_head(f"Moving benches from {CLI_DIR} to {self.benches_dir}") for site in move_directory_list: site_name = site.parts[-1] - new_path = self.bences_dir / site_name + new_path = self.benches_dir / site_name shutil.move(site, new_path) self.logger.debug(f"Moved:{site.exists()}") - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version)}:[/bold] ") - self.logger.info(f"[{self.version}] : Migration starting") + richprint.print(f"Moved benches from {CLI_DIR} to {self.benches_dir}") self.logger.info("-" * 40) def down(self): if self.skip: return True - richprint.print(f"Started", prefix=f"[bold]v{str(self.version)} [ROLLBACK]:[/bold] ") + richprint.change_head(f"Working on v{str(self.version)} rollback.") self.logger.info("-" * 40) - if self.bences_dir.exists(): - richprint.print( - f"Found benches directory change.", prefix=f"[bold]v{str(self.version.version)} [ROLLBACK]:[/bold] " - ) - + if self.benches_dir.exists(): move_directory_list = [] - for site_dir in self.bences_dir.iterdir(): + for site_dir in self.benches_dir.iterdir(): if site_dir.is_dir(): docker_compose_path = site_dir / "docker-compose.yml" @@ -73,17 +68,17 @@ def down(self): move_directory_list.append(site_dir) # stop all the sites - sites_mananger = MigrationBenches(self.bences_dir) + sites_mananger = MigrationBenches(self.benches_dir) sites_mananger.stop_benches() # move all the directories for site in move_directory_list: site_name = site.parts[-1] - new_path = self.bences_dir.parent / site_name + new_path = self.benches_dir.parent / site_name shutil.move(site, new_path) # delete the sitedir - shutil.rmtree(self.bences_dir) + shutil.rmtree(self.benches_dir) - richprint.print(f"Successfull", prefix=f"[bold]v{str(self.version.version)} [ROLLBACK]:[/bold] ") + richprint.print(f"[bold]v{str(self.version)}[/bold] rollback successfull.") self.logger.info("-" * 40) diff --git a/frappe_manager/services_manager/database_service_manager.py b/frappe_manager/services_manager/database_service_manager.py index 3112541a..ad6e6b4f 100644 --- a/frappe_manager/services_manager/database_service_manager.py +++ b/frappe_manager/services_manager/database_service_manager.py @@ -3,12 +3,23 @@ from typing import Dict, Any, Optional, Protocol, List, Union from frappe_manager.compose_project.compose_project import ComposeProject from frappe_manager.docker_wrapper.DockerException import DockerException -from frappe_manager.services_manager.services_exceptions import DatabaseServiceDBCreateFailed, DatabaseServiceDBExportFailed, DatabaseServiceDBImportFailed, DatabaseServiceDBNotFoundError, DatabaseServiceDBRemoveFailError, DatabaseServiceException, DatabaseServicePasswordNotFound, DatabaseServiceStartTimeout, DatabaseServiceUserRemoveFailError +from frappe_manager.services_manager.services_exceptions import ( + DatabaseServiceDBCreateFailed, + DatabaseServiceDBExportFailed, + DatabaseServiceDBImportFailed, + DatabaseServiceDBNotFoundError, + DatabaseServiceDBRemoveFailError, + DatabaseServiceException, + DatabaseServicePasswordNotFound, + DatabaseServiceStartTimeout, + DatabaseServiceUserRemoveFailError, +) from frappe_manager.display_manager.DisplayManager import richprint from pydantic import BaseModel from frappe_manager.utils.docker import SubprocessOutput + # TODO this class will be used for validation for main config class DatabaseServerServiceInfo(BaseModel): host: str @@ -24,7 +35,7 @@ def import_from_compose_file(cls, compose_service_name: str, compose_project: Co """ compose_service_envs = compose_project.compose_file_manager.get_envs(container=compose_service_name) - info: Dict[str, Any]= {} + info: Dict[str, Any] = {} info["user"] = 'root' # this also being considered as servicename info["host"] = compose_service_name @@ -44,29 +55,30 @@ def import_from_compose_file(cls, compose_service_name: str, compose_project: Co return cls(**info) + class DatabaseServiceManager(Protocol): database_server_info: DatabaseServerServiceInfo compose_project: ComposeProject - def __init__(self,database_server_info: DatabaseServerServiceInfo, compose_project: ComposeProject) -> None: + def __init__(self, database_server_info: DatabaseServerServiceInfo, compose_project: ComposeProject) -> None: ... def remove_user(self, db_user: str, db_user_host: str = '%', remove_all_host: bool = False): ... - def add_user(self, db_user: str, db_pass: str, db_user_host: str = '%', force: bool = False, timeout = 25): + def add_user(self, db_user: str, db_pass: str, db_user_host: str = '%', force: bool = False, timeout=25): ... - def grant_user_privilages(self, db_user:str, db_name: str): + def grant_user_privilages(self, db_user: str, db_name: str): ... - def check_user_exists(self,db_user: str): + def check_user_exists(self, db_user: str): ... - def check_db_exists(self,db_name: str): + def check_db_exists(self, db_name: str): ... - def remove_db(self,db_name: str): + def remove_db(self, db_name: str): ... def wait_till_db_start(self, interval: int = 5, timeout: int = 30) -> bool: @@ -75,8 +87,14 @@ def wait_till_db_start(self, interval: int = 5, timeout: int = 30) -> bool: def db_import(self, db_name: str, host_db_file_path: Path, force: bool = False): ... + class MariaDBManager(DatabaseServiceManager): - def __init__(self, database_server_info: DatabaseServerServiceInfo, compose_project: ComposeProject, run_on_compose_service: Optional[str] = None) -> None: + def __init__( + self, + database_server_info: DatabaseServerServiceInfo, + compose_project: ComposeProject, + run_on_compose_service: Optional[str] = None, + ) -> None: """ Database manager """ @@ -92,7 +110,9 @@ def __init__(self, database_server_info: DatabaseServerServiceInfo, compose_proj self.base_query = '-e ' self.quiet = True - def db_run_query(self, query: str, raise_exception_obj: Optional[DatabaseServiceException] = None, capture_output: bool = False): + def db_run_query( + self, query: str, raise_exception_obj: Optional[DatabaseServiceException] = None, capture_output: bool = False + ): base_command = self.base_command if capture_output: @@ -101,7 +121,9 @@ def db_run_query(self, query: str, raise_exception_obj: Optional[DatabaseService db_query = base_command + self.base_query + query try: - output = self.compose_project.docker.compose.exec(self.run_on_compose_service, command=db_query, stream=not capture_output) + output = self.compose_project.docker.compose.exec( + self.run_on_compose_service, command=db_query, stream=not capture_output + ) if capture_output: return output richprint.live_lines(output) @@ -117,27 +139,31 @@ def wait_till_db_start(self, interval: int = 5, timeout: int = 30) -> bool: else: return True total_timeout = interval * timeout - raise DatabaseServiceStartTimeout(total_timeout,self.run_on_compose_service) + raise DatabaseServiceStartTimeout(total_timeout, self.run_on_compose_service) def is_db_running(self) -> bool: db_started_command = f"mysqladmin -P{self.database_server_info.port} -h{self.database_server_info.host} -u'{self.database_server_info.user}' -p'{self.database_server_info.password}' ping" try: - output = self.compose_project.docker.compose.exec(self.run_on_compose_service, command=db_started_command, stream=False) + output = self.compose_project.docker.compose.exec( + self.run_on_compose_service, command=db_started_command, stream=False + ) return 'mysqld is alive' in " ".join(output.stdout) except DockerException as e: return False - def get_db_users(self) -> Dict[str,str]: + def get_db_users(self) -> Dict[str, str]: show_db_user_command = f"'SELECT User, Host FROM mysql.user;'" - exception = DatabaseServiceException(self.database_server_info.host,'Failed to determine mysql users.') - output: SubprocessOutput = self.db_run_query(show_db_user_command, raise_exception_obj=exception,capture_output=True) - user_list: Dict[str,str] = {} + exception = DatabaseServiceException(self.database_server_info.host, 'Failed to determine mysql users.') + output: SubprocessOutput = self.db_run_query( + show_db_user_command, raise_exception_obj=exception, capture_output=True + ) + user_list: Dict[str, str] = {} for line in output.stdout: username, host = line.split('\t') user_list[username] = host return user_list - def check_user_exists(self,username:str, host:Optional[str] = None) -> bool: + def check_user_exists(self, username: str, host: Optional[str] = None) -> bool: user_list = self.get_db_users() if not username in user_list: return False @@ -149,8 +175,10 @@ def check_user_exists(self,username:str, host:Optional[str] = None) -> bool: def get_all_databases(self) -> List[str]: db_exits_commmand = f"'SHOW DATABASES;'" - db_exits_exception = DatabaseServiceException(self.database_server_info.host, 'Failed to get list of all databases.') - output: SubprocessOutput = self.db_run_query(db_exits_commmand,db_exits_exception, capture_output=True) + db_exits_exception = DatabaseServiceException( + self.database_server_info.host, 'Failed to get list of all databases.' + ) + output: SubprocessOutput = self.db_run_query(db_exits_commmand, db_exits_exception, capture_output=True) return output.stdout def check_db_exists(self, db_name: str): @@ -158,7 +186,7 @@ def check_db_exists(self, db_name: str): return db_name in databases def remove_user(self, db_user: str, db_user_host: str = '%', remove_all_host: bool = False): - users = { db_user : db_user_host } + users = {db_user: db_user_host} if remove_all_host: users = self.get_db_users() @@ -166,31 +194,35 @@ def remove_user(self, db_user: str, db_user_host: str = '%', remove_all_host: bo for user, host in users.items(): if db_user == user: remove_db_user_command = f"'DROP USER `{user}`@`{host}`;'" - remove_db_user_exception = DatabaseServiceUserRemoveFailError(user,host) + remove_db_user_exception = DatabaseServiceUserRemoveFailError(user, host) self.db_run_query(remove_db_user_command, remove_db_user_exception) - def remove_db(self,db_name: str): + def remove_db(self, db_name: str): remove_db_command = f"'DROP DATABASE `{db_name}`;'" - remove_db_exception = DatabaseServiceDBRemoveFailError(db_name,self.database_server_info.host) - self.db_run_query(remove_db_command,remove_db_exception) + remove_db_exception = DatabaseServiceDBRemoveFailError(db_name, self.database_server_info.host) + self.db_run_query(remove_db_command, remove_db_exception) - def grant_user_privilages(self, db_user:str, db_name: str): + def grant_user_privilages(self, db_user: str, db_name: str): grant_user_command = f"'GRANT ALL PRIVILEGES ON `{db_name}`.* TO `{db_user}`@`%`;'" - grant_user_exception = DatabaseServiceException(self.database_server_info.host,f'Failed to grant prvilages for user {db_user} on {db_name}.') - self.db_run_query(grant_user_command,grant_user_exception) + grant_user_exception = DatabaseServiceException( + self.database_server_info.host, f'Failed to grant prvilages for user {db_user} on {db_name}.' + ) + self.db_run_query(grant_user_command, grant_user_exception) - def add_user(self, db_user: str, db_pass: str, db_user_host: str = '%', force: bool = False, timeout = 25): + def add_user(self, db_user: str, db_pass: str, db_user_host: str = '%', force: bool = False, timeout=25): if self.check_user_exists(db_user, db_user_host): if force: - self.remove_user(db_user,db_user_host) + self.remove_user(db_user, db_user_host) else: - raise DatabaseServiceException(self.run_on_compose_service,f'User {db_user} for {db_user_host} already exists.') + raise DatabaseServiceException( + self.run_on_compose_service, f'User {db_user} for {db_user_host} already exists.' + ) add_user_command = f"'CREATE USER `{db_user}`@`%` IDENTIFIED BY \"{db_pass}\";'" - add_user_exception = DatabaseServiceException(self.database_server_info.host,f'Failed to add user {db_user}.') - self.db_run_query(add_user_command,add_user_exception) + add_user_exception = DatabaseServiceException(self.database_server_info.host, f'Failed to add user {db_user}.') + self.db_run_query(add_user_command, add_user_exception) - def db_export(self, db_name: str, export_file_path: Union[str,Path]): + def db_export(self, db_name: str, export_file_path: Union[str, Path]): if not self.check_db_exists(db_name): raise DatabaseServiceDBNotFoundError(db_name, self.run_on_compose_service) @@ -198,23 +230,25 @@ def db_export(self, db_name: str, export_file_path: Union[str,Path]): export_file_path = str(export_file_path.absolute()) db_export_command = f"mysqldump -u'{self.database_server_info.user}' -p'{self.database_server_info.password}' -h'{self.database_server_info.host}' -P{self.database_server_info.port} {db_name} --result-file={export_file_path}" + try: - output = self.compose_project.docker.compose.exec(self.run_on_compose_service, command=db_export_command, stream=False) + output = self.compose_project.docker.compose.exec( + self.run_on_compose_service, command=db_export_command, stream=False + ) except DockerException: - raise DatabaseServiceDBExportFailed(self.run_on_compose_service,db_name) + raise DatabaseServiceDBExportFailed(self.run_on_compose_service, db_name) - def db_create(self,db_name): + def db_create(self, db_name): create_db_command = f"'CREATE DATABASE IF NOT EXISTS `{db_name}`';" create_db_exception = DatabaseServiceDBCreateFailed(self.run_on_compose_service, db_name) - self.db_run_query(create_db_command,create_db_exception) + self.db_run_query(create_db_command, create_db_exception) def db_import(self, db_name: str, host_db_file_path: Path, force: bool = False): - if not self.check_db_exists(db_name): if force: self.db_create(db_name) else: - raise DatabaseServiceDBNotFoundError(db_name,self.run_on_compose_service) + raise DatabaseServiceDBNotFoundError(db_name, self.run_on_compose_service) container_db_file_name = host_db_file_path.name source = str(host_db_file_path.absolute()) @@ -222,6 +256,8 @@ def db_import(self, db_name: str, host_db_file_path: Path, force: bool = False): db_import_command = self.base_command + f" {db_name} -e 'source /tmp/{container_db_file_name}'" try: output = self.compose_project.docker.compose.cp(source, destination, stream=False) - output = self.compose_project.docker.compose.exec(self.run_on_compose_service, command=db_import_command, stream=False) + output = self.compose_project.docker.compose.exec( + self.run_on_compose_service, command=db_import_command, stream=False + ) except DockerException: - raise DatabaseServiceDBImportFailed(self.run_on_compose_service,source) + raise DatabaseServiceDBImportFailed(self.run_on_compose_service, source) diff --git a/frappe_manager/site_manager/SiteManager.py b/frappe_manager/site_manager/SiteManager.py index 8665f459..bbda7486 100644 --- a/frappe_manager/site_manager/SiteManager.py +++ b/frappe_manager/site_manager/SiteManager.py @@ -61,7 +61,8 @@ def list_benches(self): if not bench_list: richprint.exit( - "Seems like you haven't created any sites yet. To create a bench, use the command: 'fm create '." + "Seems like you haven't created any sites yet. To create a bench, use the command: 'fm create '.", + emoji_code=":white_check_mark:", ) list_table = Table(show_lines=True, show_header=True, highlight=True) diff --git a/frappe_manager/sub_commands/ssl_command.py b/frappe_manager/sub_commands/ssl_command.py index 267d23b2..114f5760 100644 --- a/frappe_manager/sub_commands/ssl_command.py +++ b/frappe_manager/sub_commands/ssl_command.py @@ -21,6 +21,8 @@ def delete( ), ] = None, ): + """Delete bench ssl certficate.""" + services_manager = ctx.obj["services"] bench = Bench.get_object(benchname, services_manager) richprint.change_head("Removing SSL certificate") @@ -40,6 +42,8 @@ def renew( ] = None, all: Annotated[bool, typer.Option(help="Renew ssl cert for all benches.")] = False, ): + """Renew bench ssl certficate.""" + services_manager = ctx.obj["services"] benches = BenchesManager(CLI_BENCHES_DIRECTORY, services=services_manager) diff --git a/frappe_manager/sub_commands/update_command.py b/frappe_manager/sub_commands/update_command.py index e2d477c0..6aea10ef 100644 --- a/frappe_manager/sub_commands/update_command.py +++ b/frappe_manager/sub_commands/update_command.py @@ -37,4 +37,6 @@ def update_callback( def images( ctx: typer.Context, ): + """Pull latest FM stack docker images.""" + pull_docker_images()