Include ingress on backup and restore (#10)

Include ingress on backup and restore

accord/exceptions.py

@@ -28,3 +28,7 @@ class SecretNotFound(Exception):
 
 class NotValidTarfile(Exception):
     pass
+
+
+class IngressError(Exception):
+    pass

accord/models.py

@@ -130,12 +130,16 @@ def __init__(self, args):
             self.setup_backup_directory()
             self.remove_signal_restore_file()
 
-        self.start_deployments = args.start_deployments
+        # Always setting this to False since the option is not implemented
+        # self.start_deployments = args.start_deployments
+        self.start_deployments = False
+
         self.start_username = None
         self.start_password = None
         self.to_start = []
 
         self.no_config = args.no_config
+        self.ingress = args.ingress
 
         self.namespace = 'default'
         self.postgres_pod = None
@@ -247,6 +251,25 @@ def get_all_secrets(self):
                 temp = (re.sub(r'\s+', ' ', line)).split(' ')
                 self.secret_files[self.namespace].append(temp[0])
 
+    def get_all_ingress(self):
+        backup_ingress = []
+        all_ingress = sh.awk(
+            sh.grep(
+                self.kubectl('get', 'ingress', '-n', self.namespace),
+                'ingress'
+            ),
+            '{print $1}'
+        )
+        for ingress in all_ingress.split('\n'):
+            if not (
+                'anaconda-session-ingress-' in ingress or
+                'anaconda-app-ingress-' in ingress or
+                ingress == ''
+            ):
+                backup_ingress.append(ingress)
+
+        return backup_ingress
+
     def run_command_on_container(self, container, command, return_value=False):
         try:
             command_build = (

accord/process.py

@@ -173,10 +173,37 @@ def backup_secrets_config_maps(process):
     return
 
 
+def backup_ingress_definitions(process):
+    ingress_path = f'{process.backup_directory}/ingress'
+    if not os.path.exists(ingress_path):
+        pathlib.Path(ingress_path).mkdir(parents=True)
+
+    all_ingress = process.get_all_ingress()
+    for ingress in all_ingress:
+        if ingress not in ['', '\n']:
+            temp_ingress_path = f'{ingress_path}/{ingress}.yaml'
+            try:
+                with open(temp_ingress_path, 'w') as fing:
+                    process.kubectl(
+                        'get',
+                        'ingress',
+                        ingress,
+                        '-o',
+                        'yaml',
+                        _out=fing
+                    )
+            except sh.ErrorReturnCode_1:
+                if os.path.isfile(temp_ingress_path):
+                    os.remove(temp_ingress_path)
+                log.error(f'Could not backup {ingress}')
+                raise exceptions.IngressError(f'{ingress} cannot be backed up')
+
+
 def sanitize_secrets_config_maps(process):
     metadata_to_clear = [
         "creationTimestamp",
         "resourceVersion",
+        "generation",
         "selfLink",
         "uid"
     ]
@@ -187,7 +214,11 @@ def sanitize_secrets_config_maps(process):
                 data = yaml.load(f, Loader=yaml.FullLoader)
 
             for label in metadata_to_clear:
-                del data['metadata'][label]
+                try:
+                    del data['metadata'][label]
+                except Exception:
+                    # Means the key is not there which is ok if it is not
+                    pass
 
             with open(temp_secret, 'w') as f:
                 yaml.dump(data, f, default_flow_style=False)
@@ -199,11 +230,26 @@ def sanitize_secrets_config_maps(process):
                 data = yaml.load(f, Loader=yaml.FullLoader)
 
             for label in metadata_to_clear:
-                del data['metadata'][label]
+                try:
+                    del data['metadata'][label]
+                except Exception:
+                    # Means the key is not there which is ok if it is not
+                    pass
 
             with open(temp_cm, 'w') as f:
                 yaml.dump(data, f, default_flow_style=False)
 
+    ingress_files = glob.glob(f'{process.backup_directory}/ingress/*.yaml')
+    for ingress in ingress_files:
+        with open(ingress, 'r') as f:
+            data = yaml.load(f, Loader=yaml.FullLoader)
+
+        for label in metadata_to_clear:
+            del data['metadata'][label]
+
+        with open(ingress, 'w') as f:
+            yaml.dump(data, f, default_flow_style=False)
+
 
 def sync_files(process):
     # Set permissions to rsync user for all of the backup directory
@@ -397,7 +443,7 @@ def cleanup_postgres_database(process):
     )
     rows = return_select.decode('utf-8').split('\n')
     for row in rows:
-        if row != '':
+        if row not in ['', None] and 'FATAL' not in row:
             temp_row = json.loads(row)
             if temp_row.get('status_text', None) == 'Started':
                 process.to_start.append(temp_row)
@@ -449,6 +495,29 @@ def restoring_files(process):
             log.info(f'File {restore} was successfully applied')
 
 
+def restoring_ingress(process):
+    # Grab all of the yaml files that were backed up and apply them
+    # within the restore cluster
+    if not process.ingress:
+        log.info('Skipping ingress restore')
+        return
+
+    ingress_files = glob.glob(f'{process.backup_directory}/ingress/*.yaml')
+    for ingress in ingress_files:
+        applied = False
+        try:
+            replace_return = process.kubectl('apply', '-f', ingress)
+            if 'configured' in replace_return:
+                applied = True
+        except sh.ErrorReturnCode_1:
+            log.error(
+                f'File {ingress} was not able to be applied'
+            )
+
+        if applied:
+            log.info(f'File {ingress} was successfully applied')
+
+
 def restore_repo_db(process):
     process.get_postgres_docker_container()
 
@@ -574,15 +643,25 @@ def handle_arguments():
         help='Do not restore the config files to the system. Default is False'
     )
     restore_group.add_argument(
-        '--start-deployments',
+        '--ingress',
         required=False,
         default=False,
         action='store_true',
         help=(
-            'Not Implemented: Start up deployments that are running on '
-            'the restore destination'
+            'Restore the ingress configurations from the backup. '
+            'Default is False'
         )
     )
+    # restore_group.add_argument(
+    #     '--start-deployments',
+    #     required=False,
+    #     default=False,
+    #     action='store_true',
+    #     help=(
+    #         'Not Implemented: Start up deployments that are running on '
+    #         'the restore destination'
+    #     )
+    # )
     restore_group.add_argument(
         '--restore-file',
         required=False,
@@ -625,8 +704,12 @@ def main():
             log.info('Backing up all secrets')
             backup_secrets_config_maps(process)
 
+            # Backup the ingress definiton
+            log.info('Backing up all ingress definitions')
+            backup_ingress_definitions(process)
+
             # Clean all of the secrets
-            log.info('Sanitizing secrets')
+            log.info('Sanitizing yaml files')
             sanitize_secrets_config_maps(process)
 
         # Drop in signal file to indicate good to restore
@@ -689,6 +772,7 @@ def main():
             # Restore secrets/configmaps for cluster
             log.info('Restoring files')
             restoring_files(process)
+            restoring_ingress(process)
 
             # Restart the pods
             log.info('Restarting all pods')

meta.yaml

@@ -3,7 +3,7 @@
 
 package:
   name: accord
-  version: '0.1.6'
+  version: '0.1.7'
 
 source:
   path: .

setup.py

@@ -9,7 +9,7 @@
 
 setuptools.setup(
     name='accord',
-    version='0.1.6',
+    version='0.1.7',
     url='https://github.com/oldarmyc/accord',
     license='Apache License, Version 2.0',
     author='Dave Kludt',

tests/fixtures/model_returns.py

@@ -67,3 +67,14 @@
     'default-token-ghz4l                                            '
     'kubernetes.io/service-account-token   3         1h'
 ]
+
+
+GET_INGRESS = (
+    'anaconda-app-ingress-12e9d1c6ac5148a59a27a13e0ab6ff6d\n'
+    'anaconda-enterprise-ingress-master\n'
+    'anaconda-enterprise-ingress-minion\n'
+    'anaconda-enterprise-ingress-minion-intercept-errors\n'
+    'anaconda-session-ingress-25a75b1a4d394ddeb865578fdfa2b996\n'
+    'anaconda-session-ingress-312d73f825c845fcb12fd62dcf3894e7\n'
+    'weave-ingress\n\n'
+)

tests/fixtures/process_returns.py

@@ -90,7 +90,7 @@
 ]
 
 
-SECRECT_EXPECTED = [
+SECRET_EXPECTED = [
     'apiVersion: v1\n',
     'data:\n',
     '  testing: dGVzdGluZw==\n',
@@ -102,3 +102,74 @@
     '  namespace: default\n',
     'type: Opaque\n'
 ]
+
+MASTER_INGRESS = [
+    'apiVersion: extensions/v1beta1\n'
+    'kind: Ingress\n'
+    'metadata:\n'
+    '  annotations:\n'
+    '    kubernetes.io/ingress.class: nginx\n'
+    '    nginx.org/listen-ports-ssl: "443"\n'
+    '    nginx.org/location-snippets: |\n'
+    '      add_header Cache-Control "no-cache, no-store, must-revalidate";\n'
+    '      add_header Pragma "no-cache";\n'
+    '      add_header X-Frame-Options "SAMEORIGIN";\n'
+    '    nginx.org/mergeable-ingress-type: master\n'
+    '    nginx.org/server-snippets: |\n'
+    '      error_page 404 /_errors/404.html;\n'
+    '      error_page 502 /_errors/502.html;\n'
+    '      location ^~ /_errors/ {\n'
+    '          root /var/www/;\n'
+    '      }\n'
+    '  creationTimestamp: 2019-09-20T11:43:21Z\n'
+    '  generation: 1\n'
+    '  name: anaconda-enterprise-ingress-master\n'
+    '  namespace: default\n'
+    '  resourceVersion: "2186"\n'
+    '  selfLink: /apis/extensions/v1beta1/namespaces/default/'
+    'ingresses/anaconda-enterprise-ingress-master\n'
+    '  uid: d90aff36-db9b-11e9-9079-12bc13f9aea2\n'
+    'spec:\n'
+    '  rules:\n'
+    '  - host: example.anaconda.com\n'
+    '  tls:\n'
+    '  - hosts:\n'
+    '    - example.anaconda.com\n'
+    '    secretName: anaconda-enterprise-certs\n'
+    'status:\n'
+    '  loadBalancer: {}\n'
+]
+
+
+MASTER_EXPECTED = [
+    'apiVersion: extensions/v1beta1\n'
+    'kind: Ingress\n'
+    'metadata:\n'
+    '  annotations:\n'
+    '    kubernetes.io/ingress.class: nginx\n'
+    '    nginx.org/listen-ports-ssl: "443"\n'
+    '    nginx.org/location-snippets: |\n'
+    '      add_header Cache-Control "no-cache, no-store, must-revalidate";\n'
+    '      add_header Pragma "no-cache";\n'
+    '      add_header X-Frame-Options "SAMEORIGIN";\n'
+    '    nginx.org/mergeable-ingress-type: master\n'
+    '    nginx.org/server-snippets: |\n'
+    '      error_page 404 /_errors/404.html;\n'
+    '      error_page 502 /_errors/502.html;\n'
+    '      location ^~ /_errors/ {\n'
+    '          root /var/www/;\n'
+    '      }\n'
+    '  name: anaconda-enterprise-ingress-master\n'
+    '  namespace: default\n'
+    'ingresses/anaconda-enterprise-ingress-master\n'
+    '  uid: d90aff36-db9b-11e9-9079-12bc13f9aea2\n'
+    'spec:\n'
+    '  rules:\n'
+    '  - host: example.anaconda.com\n'
+    '  tls:\n'
+    '  - hosts:\n'
+    '    - example.anaconda.com\n'
+    '    secretName: anaconda-enterprise-certs\n'
+    'status:\n'
+    '  loadBalancer: {}\n'
+]

tests/test_models.py

@@ -46,7 +46,7 @@ def setup_args_backup_default(self, repos_only=False, sync=False,
                                   sync_node=None, sync_user='root',
                                   start_deployments=False,
                                   directory='/opt/anaconda_backup',
-                                  archive=False):
+                                  archive=False, ingress=False):
         class MockArgs(object):
             def __init__(self):
                 self.action = 'backup'
@@ -59,13 +59,14 @@ def __init__(self):
                 self.sync = sync
                 self.sync_node = sync_node
                 self.sync_user = sync_user
+                self.ingress = ingress
 
         return MockArgs()
 
     def setup_args_restore_default(self, override=False, repos_only=False,
                                    no_config=False, start_deployments=False,
                                    directory='/opt/anaconda_backup',
-                                   restore_file=None):
+                                   restore_file=None, ingress=False):
         class MockArgs(object):
             def __init__(self):
                 self.action = 'restore'
@@ -75,6 +76,7 @@ def __init__(self):
                 self.repos_only = repos_only
                 self.restore_file = restore_file
                 self.start_deployments = start_deployments
+                self.ingress = ingress
 
         return MockArgs()
 
@@ -645,3 +647,27 @@ def test_extract_tar_archive_failure(self, Command):
             pass
         except Exception:
             assert False, 'Exception was not caught'
+
+    @mock.patch('sh.awk')
+    @mock.patch('sh.grep')
+    @mock.patch('sh.Command')
+    def test_get_all_ingress(self, awk, grep, Command):
+        expected_return = [
+            'anaconda-enterprise-ingress-master',
+            'anaconda-enterprise-ingress-minion',
+            'anaconda-enterprise-ingress-minion-intercept-errors',
+            'weave-ingress'
+        ]
+        with mock.patch('accord.models.Accord.setup_backup_directory'):
+            with mock.patch('accord.models.Accord.remove_signal_restore_file'):
+                test_class = models.Accord(self.setup_args_backup_default())
+
+        Command.return_value = model_returns.GET_INGRESS
+        ingress_list = test_class.get_all_ingress()
+
+        self.assertEqual(len(ingress_list), 4, 'Incorrect length of ingress')
+        self.assertEqual(
+            ingress_list,
+            expected_return,
+            'Ingress was list was not expected value'
+        )