Restart on filesystem diff

perfact/zodbsync/commands/watch.py

@@ -49,6 +49,12 @@ def _increment_txnid(s):
     return bytes(arr)
 
 
+class TreeOutdatedException(Exception):
+    """Exception which is raised if the internal tree structure
+    is not matching the actual Filesystem anymore"""
+    pass
+
+
 class Watch(SubCommand):
     """Periodically check for changes and record them"""
     # Connects to ZEO, builds a mirror of the tree structure of the objects,
@@ -311,7 +317,13 @@ def _update_children(self, oid):
                     )
                 )
                 os.makedirs(self.base_dir+newpath)
-                os.rename(self.base_dir+oldpath, self.base_dir+newpath)
+                try:
+                    os.rename(self.base_dir+oldpath, self.base_dir+newpath)
+                except OSError as err:
+                    if err.errno == 2:  # no such file or directory
+                        raise TreeOutdatedException()
+                    self.logger.exception('Unexpected OSError')
+                    raise
                 self._update_path(child_oid, newpath)
 
         # go through new children and check if they have old parents
@@ -483,22 +495,29 @@ def step(self):
         changes."""
         self.unregister_signals()
         self.acquire_lock(timeout=300)
-        self.register_signals()
-
-        start_txnid = _increment_txnid(self.last_visible_txn)
-        self._set_last_visible_txn()
-        self._read_changed_oids(
-            txn_start=start_txnid,
-            txn_stop=self.last_visible_txn,
-        )
-        # make sure we see a consistent snapshot, even though we later
-        # abort this transaction since we do not write anything
-        self.sync.tm.begin()
-        self._update_objects()
-        self.sync.tm.abort()
+        try:
+            self.register_signals()
 
-        self._store_last_visible_txn()
-        self.release_lock()
+            start_txnid = _increment_txnid(self.last_visible_txn)
+            self._set_last_visible_txn()
+            self._read_changed_oids(
+                txn_start=start_txnid,
+                txn_stop=self.last_visible_txn,
+            )
+            # make sure we see a consistent snapshot, even though we later
+            # abort this transaction since we do not write anything
+            self.sync.tm.begin()
+            self._update_objects()
+            self.sync.tm.abort()
+
+            self._store_last_visible_txn()
+        except TreeOutdatedException:
+            self.logger.info(
+                'Exiting due to inconsistencies in filesystem'
+            )
+            self.exit.set()
+        finally:
+            self.release_lock()
 
     def run(self, interval=10):
         """ Setup and run in a loop. """

perfact/zodbsync/tests/test_sync.py

@@ -6,6 +6,7 @@
 import subprocess
 import pickle
 import pytest
+import shutil
 
 import ZEO
 import transaction
@@ -869,6 +870,43 @@ def test_watch_structure_changes_and_playback_local_changes(self, conn):
         self.watcher_step_until(watcher,
                                 lambda: "('title', '"+new_title+"')" in meta)
 
+    def test_watch_structure_changes_and_playback_deleted_folder(self, conn):
+        """
+        create structure while 'watch' command is running,
+        remove a folder, then play those changes back and check,
+        whether the step function correctly "crashes"
+        """
+
+        # start watch daemon
+        watcher = self.mkrunner('watch')
+        watcher.setup()
+        app = conn.app
+        folder_1 = "folder_1"
+        s_folder_1 = "s_folder_1"
+
+        # create folder and wait until watch notices change
+        with conn.tm:
+            app.manage_addFolder(id=folder_1)
+        self.watcher_step_until(watcher,
+                                lambda: os.path.isdir(
+                                    self.repo.path + '/__root__/'+folder_1))
+
+        # create subfolder and wait until watch notices change
+        with conn.tm:
+            app.folder_1.manage_addFolder(id=s_folder_1, title=s_folder_1)
+        path = self.repo.path + '/__root__/'+folder_1+'/'+s_folder_1
+        self.watcher_step_until(watcher,
+                                lambda: os.path.isdir(path))
+
+        # remove folder s_folder_1
+        shutil.rmtree(path)
+
+        # playback changes and check if those are existent in zodb
+        self.run('playback', '/')
+
+        # wait for watch to notices played back changes
+        self.watcher_step_until(watcher, watcher.exit.is_set)
+
     def test_commit_on_branch_and_exec_merge(self):
         '''
         change to a git feature branch and create a