From 50a5d479fd58c51f314a5a09ea1862a156d3a754 Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 11 Oct 2024 16:58:32 +0200 Subject: [PATCH] unify filesystem schema behavior with other destinations --- .../impl/filesystem/filesystem.py | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/dlt/destinations/impl/filesystem/filesystem.py b/dlt/destinations/impl/filesystem/filesystem.py index 1c2965f84a..bccf8ec686 100644 --- a/dlt/destinations/impl/filesystem/filesystem.py +++ b/dlt/destinations/impl/filesystem/filesystem.py @@ -655,24 +655,28 @@ def _get_stored_schema_by_hash_or_newest( """Get the schema by supplied hash, falls back to getting the newest version matching the existing schema name""" version_hash = self._to_path_safe_string(version_hash) # find newest schema for pipeline or by version hash - selected_path = None - newest_load_id = "0" - for filepath, fileparts in self._iter_stored_schema_files(): - if ( - not version_hash - and (fileparts[0] == self.schema.name or any_schema_name) - and fileparts[1] > newest_load_id - ): - newest_load_id = fileparts[1] - selected_path = filepath - elif fileparts[2] == version_hash: - selected_path = filepath - break + try: + selected_path = None + newest_load_id = "0" + for filepath, fileparts in self._iter_stored_schema_files(): + if ( + not version_hash + and (fileparts[0] == self.schema.name or any_schema_name) + and fileparts[1] > newest_load_id + ): + newest_load_id = fileparts[1] + selected_path = filepath + elif fileparts[2] == version_hash: + selected_path = filepath + break - if selected_path: - return StorageSchemaInfo( - **json.loads(self.fs_client.read_text(selected_path, encoding="utf-8")) - ) + if selected_path: + return StorageSchemaInfo( + **json.loads(self.fs_client.read_text(selected_path, encoding="utf-8")) + ) + except DestinationUndefinedEntity: + # ignore missing table + pass return None