Merge pull request #173 from rtCamp/fix/migrations

Refactor and fix migrations and improve migration backups management.

frappe_manager/commands.py

@@ -422,6 +422,7 @@ def logs(
     follow: Annotated[bool, typer.Option("--follow", "-f", help="Follow logs.")] = False,
 ):
     """Show frappe server logs or container logs for a given bench."""
+
     services_manager = ctx.obj["services"]
     verbose = ctx.obj['verbose']
     bench = Bench.get_object(benchname, services_manager)

frappe_manager/migration_manager/backup_manager.py

@@ -1,21 +1,23 @@
 import shutil
 from datetime import datetime
 from pathlib import Path
-from frappe_manager import CLI_DIR
+from frappe_manager import CLI_BENCHES_DIRECTORY, CLI_DIR
 from dataclasses import dataclass
 from typing import Optional
 from frappe_manager.logger import log
 
 random_strings = []
 
+
 @dataclass
-class BackupData():
+class BackupData:
     src: Path
     dest: Path
     bench: Optional[str] = None
+    bench_path: Optional[Path] = None
+    prefix_timestamp: bool = False
     allow_restore: bool = True
     _is_restored: bool = False
-    prefix_timestamp = False
     _prefix_length: int = 5
 
     @property
@@ -27,7 +29,6 @@ def is_restored(self, v: bool) -> None:
         self._is_restored = v
 
     def __post_init__(self):
-
         file_name = self.dest.name
 
         if self.prefix_timestamp:
@@ -43,43 +44,49 @@ def __post_init__(self):
                     break
 
         self.real_dest = self.dest.parent
-
-        if self.bench:
-            self.real_dest = self.real_dest  / self.bench
-
         self.real_dest: Path = self.real_dest / file_name
 
     def exists(self):
         return self.dest.exists()
 
 
-CLI_MIGARATIONS_DIR = CLI_DIR / 'migrations'/ f"{datetime.now().strftime('%d-%b-%y--%H-%M-%S')}"  / 'backups'
+current_migration_timestamp = f"{datetime.now().strftime('%d-%b-%y--%H-%M-%S')}"
+CLI_MIGARATIONS_DIR = CLI_DIR / 'backups' / 'migrations' / current_migration_timestamp
 
-class BackupManager():
 
-    def __init__(self,name, base_dir: Path = CLI_MIGARATIONS_DIR):
-        self.root_backup_dir = base_dir
-        self.backup_dir = self.root_backup_dir / name
+class BackupManager:
+    def __init__(self, name, benches_dir=CLI_BENCHES_DIRECTORY, backup_dir: Path = CLI_MIGARATIONS_DIR):
+        self.name = name
+        self.root_backup_dir: Path = backup_dir
+        self.benches_dir: Path = benches_dir
+        self.backup_dir: Path = self.root_backup_dir / self.name
+        self.bench_backup_dir: Path = Path('backups') / 'migrations' / current_migration_timestamp
         self.backups = []
         self.logger = log.get_logger()
 
         # create backup dir if not exists
-        self.backup_dir.mkdir(parents=True,exist_ok=True)
-
-    def backup(self, src: Path, dest: Optional[Path] = None, bench_name: Optional[str] = None, allow_restore: bool = True ):
-
+        self.backup_dir.mkdir(parents=True, exist_ok=True)
+
+    def backup(
+        self,
+        src: Path,
+        dest: Optional[Path] = None,
+        bench_name: Optional[str] = None,
+        allow_restore: bool = True,
+    ):
         if not src.exists():
             return None
 
         if not dest:
             dest = self.backup_dir / src.name
 
-        backup_data = BackupData(src, dest, allow_restore=allow_restore)
+            if bench_name:
+                dest: Path = self.benches_dir / bench_name / self.bench_backup_dir / self.name / src.name
+
+        backup_data = BackupData(src, dest, bench=bench_name, allow_restore=allow_restore)
 
-        if bench_name:
-            backup_data = BackupData(src, dest,bench=bench_name)
-            if not backup_data.real_dest.parent.exists():
-                backup_data.real_dest.parent.mkdir(parents=True,exist_ok=True)
+        if not backup_data.real_dest.parent.exists():
+            backup_data.real_dest.parent.mkdir(parents=True, exist_ok=True)
 
         self.logger.debug(f"Backup: {backup_data.src} => {backup_data.real_dest} ")
 
@@ -94,7 +101,7 @@ def backup(self, src: Path, dest: Optional[Path] = None, bench_name: Optional[st
 
         return backup_data
 
-    def restore(self, backup_data, force = False):
+    def restore(self, backup_data, force=False):
         """
         Restore a file from a backup.
         """

frappe_manager/migration_manager/migration_base.py

@@ -1,21 +1,33 @@
+from abc import ABC
 from logging import Logger
-from typing import Protocol, runtime_checkable
+from pathlib import Path
+from frappe_manager import CLI_DIR
 from frappe_manager.migration_manager.backup_manager import BackupManager
+from frappe_manager.migration_manager.migration_exections import MigrationExceptionInBench
+from frappe_manager.migration_manager.migration_helpers import (
+    MigrationBench,
+    MigrationBenches,
+    MigrationServicesManager,
+)
 from frappe_manager.migration_manager.version import Version
 from frappe_manager.logger import log
+from frappe_manager.services_manager.database_service_manager import DatabaseServerServiceInfo, MariaDBManager
+from frappe_manager.display_manager.DisplayManager import richprint
+from frappe_manager.utils.helpers import capture_and_format_exception
 
 
-@runtime_checkable
-class MigrationBase(Protocol):
+# @runtime_checkable
+class MigrationBase(ABC):
     version: Version = Version("0.0.0")
+    benches_dir: Path = CLI_DIR / "sites"
     skip: bool = False
     migration_executor = None
-    backup_manager: BackupManager  # Declare the backup_manager variable
-    logger: Logger
+    logger: Logger = log.get_logger()
 
     def init(self):
-        self.backup_manager = BackupManager(str(self.version))  # Assign the value to backup_manager
-        self.logger = log.get_logger()
+        self.backup_manager = BackupManager(str(self.version), self.benches_dir)
+        self.benches_manager = MigrationBenches(self.benches_dir)
+        self.services_manager: MigrationServicesManager = MigrationServicesManager(services_path=CLI_DIR / 'services')
 
     def set_migration_executor(self, migration_executor):
         self.migration_executor = migration_executor
@@ -24,7 +36,159 @@ def get_rollback_version(self):
         return self.version
 
     def up(self):
-        pass
+        if self.skip:
+            return True
+
+        richprint.stdout.rule(f':package: [bold][blue]v{str(self.version)}[/blue][bold]')
+        self.logger.info(f"v{str(self.version)}: Started")
+        self.logger.info("-" * 40)
+
+        self.init()
+        self.services_basic_backup()
+        self.migrate_services()
+        self.migrate_benches()
+
+        self.logger.info("-" * 40)
 
     def down(self):
+        richprint.change_head(f"Working on v{str(self.version)} rollback.")
+        self.logger.info("-" * 40)
+
+        # undo each bench
+        for bench_name, bench_data in self.migration_executor.migrate_benches.items():
+            if not bench_data['exception']:
+                self.undo_bench_migrate(bench_data['object'])
+
+        for backup in self.backup_manager.backups:
+            self.backup_manager.restore(backup, force=True)
+            # richprint.print(f'Restored {backup.bench}'s {backup.src.name}.')
+
+        self.undo_services_migrate()
+
+        richprint.print(f"[bold]v{str(self.version)}[/bold] rollback successfull.")
+        self.logger.info("-" * 40)
+
+    def services_basic_backup(self):
+        if not self.services_manager.compose_project.compose_file_manager.exists():
+            raise MigrationExceptionInBench(
+                f"Services compose at {self.services_manager.compose_project.compose_file_manager} not found."
+            )
+        self.backup_manager.backup(self.services_manager.compose_project.compose_file_manager.compose_path)
+
+    def migrate_services(self):
+        pass
+
+    def undo_services_migrate(self):
+        pass
+
+    def migrate_benches(self):
+        main_error = False
+
+        # migrate each bench
+        for bench_name, bench_path in self.benches_manager.get_all_benches().items():
+            bench = MigrationBench(name=bench_name, path=bench_path.parent)
+
+            if bench.name in self.migration_executor.migrate_benches.keys():
+                bench_info = self.migration_executor.migrate_benches[bench.name]
+                if bench_info['exception']:
+                    richprint.print(f"Skipping migration for failed bench [blue]{bench.name}[/blue].")
+                    main_error = True
+                    continue
+
+            self.migration_executor.set_bench_data(bench, migration_version=self.version)
+            try:
+                self.bench_basic_backup(bench)
+                self.migrate_bench(bench)
+            except Exception as e:
+                traceback_str = capture_and_format_exception()
+                self.logger.error(f"{bench.name} [ EXCEPTION TRACEBACK ]:\n {traceback_str}")
+                richprint.update_live()
+                main_error = True
+                self.migration_executor.set_bench_data(bench, e, self.version)
+
+                # restore all backup files
+                for backup in self.backup_manager.backups:
+                    if backup.bench == bench.name:
+                        self.backup_manager.restore(backup, force=True)
+
+                self.undo_bench_migrate(bench)
+                self.logger.info(f'Undo successfull for bench: {bench.name}')
+                bench.compose_project.down_service(volumes=False, timeout=5)
+
+        if main_error:
+            raise MigrationExceptionInBench('')
+
+    def bench_basic_backup(self, bench: MigrationBench):
+        richprint.print(f"Migrating bench [bold][blue]{bench.name}[/blue][/bold]")
+
+        # backup docker compose.yml
+        self.backup_manager.backup(bench.path / "docker-compose.yml", bench_name=bench.name)
+
+        # backup common_site_config.json
+        bench_common_site_config = bench.path / "workspace" / "frappe-bench" / "sites" / "common_site_config.json"
+        self.backup_manager.backup(bench_common_site_config, bench_name=bench.name)
+
+        # backup site_config.json
+        bench_site_config = bench.path / "workspace" / "frappe-bench" / "sites" / bench.name / "site_config.json"
+        self.backup_manager.backup(bench_site_config, bench_name=bench.name)
+
+        server_db_info: DatabaseServerServiceInfo = DatabaseServerServiceInfo.import_from_compose_file(
+            'global-db', self.services_manager.compose_project
+        )
+
+        self.bench_db_backup(
+            bench=bench,
+            server_db_info=server_db_info,
+            services_manager=self.services_manager,
+            backup_manager=self.backup_manager,
+        )
+
+    def migrate_bench(self, bench: MigrationBench):
         pass
+
+    def undo_bench_migrate(self, bench: MigrationBench):
+        pass
+
+    def bench_db_backup(
+        self,
+        bench: MigrationBench,
+        server_db_info: DatabaseServerServiceInfo,
+        services_manager: MigrationServicesManager,
+        backup_manager: BackupManager,
+    ):
+        bench_db_info = bench.get_db_connection_info()
+        bench_db_name = bench_db_info["name"]
+
+        richprint.change_head(f'Commencing db [blue]{bench.name}[/blue] backup')
+
+        mariadb_manager = MariaDBManager(server_db_info, services_manager.compose_project)
+
+        from datetime import datetime
+
+        current_datetime = datetime.now()
+        formatted_date = current_datetime.strftime("%d-%m-%Y--%H-%M-%S")
+
+        container_backup_dir: Path = Path("/var/log/mysql")
+        host_backup_dir: Path = services_manager.services_path / 'mariadb' / 'logs'
+        db_sql_file_name = f"db-{bench.name}-{formatted_date}.sql"
+
+        host_db_sql_file_path: Path = host_backup_dir / db_sql_file_name
+        container_db_sql_file_path: Path = container_backup_dir / db_sql_file_name
+
+        backup_gz_file_backup_data_path: Path = (
+            bench.path / backup_manager.bench_backup_dir / self.version.version / f'{db_sql_file_name}.gz'
+        )
+
+        mariadb_manager.db_export(bench_db_name, container_db_sql_file_path)
+
+        import gzip
+        import shutil
+
+        # Compress the file using gzip
+        with open(host_db_sql_file_path, 'rb') as f_in:
+            with gzip.open(backup_gz_file_backup_data_path, 'wb') as f_out:
+                shutil.copyfileobj(f_in, f_out)
+
+        host_db_sql_file_path.unlink()
+
+        richprint.print(f'[blue]{bench.name}[/blue] db backup completed successfully.')

frappe_manager/migration_manager/migration_executor.py

@@ -140,28 +140,26 @@ def execute(self):
 
         except MigrationExceptionInBench as e:
             if self.migrate_benches:
-                print_head = True
+                passed_print_head = True
+
                 for bench, bench_status in self.migrate_benches.items():
                     if not bench_status["exception"]:
-                        if print_head:
-                            richprint.stdout.rule('[bold][red]Migration Passed Benches[/red][bold]', style='green')
-                            print_head = False
+                        if passed_print_head:
+                            richprint.stdout.rule('[bold]Migration Passed Benches[bold]', style='green')
+                            passed_print_head = False
 
                         richprint.print(f"[green]Bench[/green]: {bench}", emoji_code=':construction:')
 
-                if not print_head:
-                    richprint.stdout.rule(style='red')
-
-                print_head = True
+                failed_print_head = True
 
                 for bench, bench_status in self.migrate_benches.items():
                     if bench_status["exception"]:
-                        if print_head:
+                        if failed_print_head:
                             richprint.stdout.rule(
                                 ':police_car_light: [bold][red]Migration Failed Benches[/red][bold] :police_car_light:',
                                 style='red',
                             )
-                            print_head = False
+                            failed_print_head = False
 
                         richprint.error(f"[red]Bench[/red]: {bench}", emoji_code=':construction:')
 
@@ -178,8 +176,11 @@ def execute(self):
 
                 richprint.print(f"For error specifics, refer to {CLI_DIR}/logs/fm.log", emoji_code=':page_facing_up:')
 
-                if not print_head:
+                if not failed_print_head:
                     richprint.stdout.rule(style='red')
+                else:
+                    if not passed_print_head:
+                        richprint.stdout.rule(style='green')
 
                 archive_msg = [
                     'Available options after migrations failure :',