From 24910096fffc47f28b0a879d6944009c5dbf7284 Mon Sep 17 00:00:00 2001 From: Alexander Rolfes Date: Sat, 7 Sep 2024 01:21:23 +0200 Subject: [PATCH 1/6] On playback write layer ident into data.fs --- perfact/zodbsync/zodbsync.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/perfact/zodbsync/zodbsync.py b/perfact/zodbsync/zodbsync.py index 85cf1c0..c030aa8 100644 --- a/perfact/zodbsync/zodbsync.py +++ b/perfact/zodbsync/zodbsync.py @@ -93,11 +93,13 @@ def mod_read(obj=None, onerrorstop=False, default_owner=None, if 'owner' in meta: del meta['owner'] + meta['zodbsync_layer'] = getattr(obj, 'zodbsync_layer', None) + return meta def mod_write(data, parent=None, obj_id=None, override=False, root=None, - default_owner=None, force_default_owner=False): + default_owner=None, force_default_owner=False, layer=None): ''' Given object data in , store the object, creating it if it was missing. With = True, this method will remove an existing object @@ -162,6 +164,9 @@ def mod_write(data, parent=None, obj_id=None, override=False, root=None, for handler in mod_implemented_handlers(obj, meta_type): handler.write(obj, d) + # Also write zodbsync layer information + obj.zodbsync_layer = layer + if temp_obj: children = temp_obj.manage_cutObjects(temp_obj.objectIds()) obj.manage_pasteObjects(children) @@ -448,7 +453,12 @@ def fs_write(self, path, data): old_data = self.fs_read(pathinfo['fspath']) # Build object - meta = {key: value for key, value in data.items() if key != 'source'} + exclude_keys = ['source', 'zodbsync_layer'] + meta = { + key: value + for key, value in data.items() + if key not in exclude_keys + } fmt = mod_format(meta) if isinstance(fmt, str): fmt = fmt.encode('utf-8') @@ -739,6 +749,10 @@ def _playback_path(self, pathinfo): # fspath is None if the object is to be deleted fs_data = pathinfo['fspath'] and self.fs_parse(pathinfo['fspath']) + # extend fs_data with layerinfo + fs_data['zodbsync_layer'] = pathinfo['layers'][ + pathinfo['layeridx']]['ident'] + # Traverse to the object if it exists parent_obj = None obj = self.app @@ -823,6 +837,7 @@ def _playback_path(self, pathinfo): root=(obj if parent_obj is None else None), default_owner=self.default_owner, force_default_owner=self.force_default_owner, + layer=pathinfo['layers'][pathinfo['layeridx']]['ident'] ) except Exception: # If we do not want to get errors from missing From bfd8dbe9e86e0ec0ffe7c639dd03a873f52c9316 Mon Sep 17 00:00:00 2001 From: Alexander Rolfes Date: Sun, 8 Sep 2024 16:05:56 +0200 Subject: [PATCH 2/6] Atleast keep tests working --- perfact/zodbsync/zodbsync.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/perfact/zodbsync/zodbsync.py b/perfact/zodbsync/zodbsync.py index c030aa8..9797566 100644 --- a/perfact/zodbsync/zodbsync.py +++ b/perfact/zodbsync/zodbsync.py @@ -750,8 +750,9 @@ def _playback_path(self, pathinfo): fs_data = pathinfo['fspath'] and self.fs_parse(pathinfo['fspath']) # extend fs_data with layerinfo - fs_data['zodbsync_layer'] = pathinfo['layers'][ - pathinfo['layeridx']]['ident'] + if fs_data: + fs_data['zodbsync_layer'] = pathinfo['layers'][ + pathinfo['layeridx']]['ident'] # Traverse to the object if it exists parent_obj = None From 6c0b02c56e87887b3065d096e8c09eba199c97d0 Mon Sep 17 00:00:00 2001 From: Alexander Rolfes Date: Wed, 18 Sep 2024 22:49:41 +0200 Subject: [PATCH 3/6] Write new layer to Data.FS on record --- perfact/zodbsync/zodbsync.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/perfact/zodbsync/zodbsync.py b/perfact/zodbsync/zodbsync.py index 9797566..e0d433b 100644 --- a/perfact/zodbsync/zodbsync.py +++ b/perfact/zodbsync/zodbsync.py @@ -687,6 +687,12 @@ def record_obj(self, obj, path, recurse=True, skip_errors=False): raise pathinfo = self.fs_write(path, data) + path_layer = pathinfo['layers'][pathinfo['layeridx']]['ident'] + + current_layer = getattr(obj, 'zodbsync_layer', None) + if current_layer != path_layer: + with self.tm: + obj.zodbsync_layer = path_layer if not recurse: return From e90b02cb05345bff8ac478d9931213c07d74534a Mon Sep 17 00:00:00 2001 From: Alexander Rolfes Date: Wed, 18 Sep 2024 22:49:52 +0200 Subject: [PATCH 4/6] Add test for new functionality --- perfact/zodbsync/tests/test_sync.py | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/perfact/zodbsync/tests/test_sync.py b/perfact/zodbsync/tests/test_sync.py index dfdd044..d8619d8 100644 --- a/perfact/zodbsync/tests/test_sync.py +++ b/perfact/zodbsync/tests/test_sync.py @@ -1667,6 +1667,7 @@ def addlayer(self, seqnum='00', frozen=True): with open(path, 'w') as f: f.write('base_dir = "{}"\n'.format(layer)) f.write('frozen = {}\n'.format(frozen)) + f.write('ident = "{}"\n'.format(name)) os.mkdir(os.path.join(layer, '__root__')) # Force re-reading config if hasattr(self, 'runner'): @@ -2186,3 +2187,32 @@ def test_layer_frozen(self): with open(source_fmt.format(self.repo.path)) as f: # ... content is in custom layer! assert f.read() == 'text_content' + + def test_layer_info_datafs(self): + """ + Validate the correct writing and clearing of the layer ident + in the Data.FS + """ + with self.runner.sync.tm: + self.app.manage_addProduct['OFSP'].manage_addFile(id='blob') + + with self.addlayer(frozen=True) as layer: + self.run('record', '/blob') + assert getattr(self.app.blob, 'zodbsync_layer', None) is None + # Move file to layer and check that layer info is stored in Data.FS + shutil.move( + '{}/__root__/blob'.format(self.repo.path), + '{}/__root__/blob'.format(layer), + ) + self.run('record', '/') + with self.runner.sync.tm: + assert getattr(self.app.blob, 'zodbsync_layer') is not None + # Change file in Data.FS and verify that layer info is cleared + with self.runner.sync.tm: + self.app.blob.manage_edit( + filedata='text_content', + content_type='text/plain', + title='BLOB' + ) + self.run('record', '/') + assert getattr(self.app.blob, 'zodbsync_layer', None) is None From 016d38471896010f6265e8d4ff99d5d286519523 Mon Sep 17 00:00:00 2001 From: Alexander Rolfes Date: Wed, 18 Sep 2024 23:18:37 +0200 Subject: [PATCH 5/6] Fix broken test? --- perfact/zodbsync/tests/test_sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/perfact/zodbsync/tests/test_sync.py b/perfact/zodbsync/tests/test_sync.py index d8619d8..982dbcc 100644 --- a/perfact/zodbsync/tests/test_sync.py +++ b/perfact/zodbsync/tests/test_sync.py @@ -1996,7 +1996,8 @@ def test_layer_remove_subfolder(self): os.path.join(root, 'Test'), os.path.join(layer, '__root__/Test'), ) - self.app.Test.manage_delObjects(ids=['Sub']) + with self.runner.sync.tm: + self.app.Test.manage_delObjects(ids=['Sub']) self.run('record', '/') assert not os.path.exists(os.path.join(root, 'Test/__meta__')) assert not os.path.exists(os.path.join(root, 'Test/Sub/__meta__')) From a2689f426844fc0b6bebdae5321c2690d1325ecf Mon Sep 17 00:00:00 2001 From: Alexander Rolfes Date: Wed, 18 Sep 2024 23:29:06 +0200 Subject: [PATCH 6/6] Get rid of unnecessary transaction magic --- perfact/zodbsync/tests/test_sync.py | 6 ++---- perfact/zodbsync/zodbsync.py | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/perfact/zodbsync/tests/test_sync.py b/perfact/zodbsync/tests/test_sync.py index 982dbcc..5b5c9de 100644 --- a/perfact/zodbsync/tests/test_sync.py +++ b/perfact/zodbsync/tests/test_sync.py @@ -1996,8 +1996,7 @@ def test_layer_remove_subfolder(self): os.path.join(root, 'Test'), os.path.join(layer, '__root__/Test'), ) - with self.runner.sync.tm: - self.app.Test.manage_delObjects(ids=['Sub']) + self.app.Test.manage_delObjects(ids=['Sub']) self.run('record', '/') assert not os.path.exists(os.path.join(root, 'Test/__meta__')) assert not os.path.exists(os.path.join(root, 'Test/Sub/__meta__')) @@ -2206,8 +2205,7 @@ def test_layer_info_datafs(self): '{}/__root__/blob'.format(layer), ) self.run('record', '/') - with self.runner.sync.tm: - assert getattr(self.app.blob, 'zodbsync_layer') is not None + assert getattr(self.app.blob, 'zodbsync_layer') is not None # Change file in Data.FS and verify that layer info is cleared with self.runner.sync.tm: self.app.blob.manage_edit( diff --git a/perfact/zodbsync/zodbsync.py b/perfact/zodbsync/zodbsync.py index e0d433b..ae956ab 100644 --- a/perfact/zodbsync/zodbsync.py +++ b/perfact/zodbsync/zodbsync.py @@ -691,8 +691,7 @@ def record_obj(self, obj, path, recurse=True, skip_errors=False): current_layer = getattr(obj, 'zodbsync_layer', None) if current_layer != path_layer: - with self.tm: - obj.zodbsync_layer = path_layer + obj.zodbsync_layer = path_layer if not recurse: return