refactor: proper exceptions, check for protection if possible, refact…

…or code

ckanext/unfold/adapters/_7z.py

@@ -1,18 +1,18 @@
 from __future__ import annotations
 
 import logging
-from typing import Any, Optional
 from io import BytesIO
+from typing import Any, Optional
 
-import requests
 import py7zr
+import requests
 from py7zr import FileInfo, exceptions
 
 import ckan.plugins.toolkit as tk
 
+import ckanext.unfold.exception as unf_exception
 import ckanext.unfold.types as unf_types
 import ckanext.unfold.utils as unf_utils
-import ckanext.unfold.exception as unf_exception
 
 log = logging.getLogger(__name__)
 
@@ -25,13 +25,14 @@ def build_directory_tree(filepath: str, remote: Optional[bool] = False):
             with py7zr.SevenZipFile(filepath) as archive:
                 if archive.needs_password():
                     raise unf_exception.UnfoldError(
-                        f"Archive is protected with password"
+                        "Archive is protected with password"
                     )
 
                 file_list: list[FileInfo] = archive.list()
     except exceptions.ArchiveError as e:
-        log.error(f"Error openning 7z archive: {e}")
-        return []
+        raise unf_exception.UnfoldError(f"Error openning archive: {e}")
+    except requests.RequestException as e:
+        raise unf_exception.UnfoldError(f"Error fetching remote archive: {e}")
 
     nodes: list[unf_types.Node] = []
 
@@ -80,4 +81,9 @@ def get7zlist_from_url(url) -> list[FileInfo]:
     to download it partially and fetch only file list."""
     resp = requests.get(url)
 
+    archive = py7zr.SevenZipFile(BytesIO(resp.content))
+
+    if archive.needs_password():
+        raise unf_exception.UnfoldError("Archive is protected with password")
+
     return py7zr.SevenZipFile(BytesIO(resp.content)).list()

ckanext/unfold/adapters/gzip.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, cast, Optional
+from typing import Any, Optional, cast
 
 from ckan import model as model
 

ckanext/unfold/adapters/rar.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 
 import logging
+from datetime import datetime as dt
 from io import BytesIO
 from typing import Any, Optional
-from datetime import datetime as dt
 
 import rarfile
 import requests
@@ -29,7 +29,7 @@ def build_directory_tree(
             with rarfile.RarFile(filepath) as archive:
                 if archive.needs_password():
                     raise unf_exception.UnfoldError(
-                        f"Archive is protected with password"
+                        "Archive is protected with password"
                     )
 
                 file_list: list[RarInfo] = archive.infolist()
@@ -82,7 +82,8 @@ def _fetch_mtime(entry: RarInfo) -> str:
 
     if not modified_at and isinstance(entry.date_time, tuple):
         modified_at = tk.h.render_datetime(
-            dt(*entry.date_time), date_format=unf_utils.DEFAULT_DATE_FORMAT
+            dt(*entry.date_time),  # type: ignore
+            date_format=unf_utils.DEFAULT_DATE_FORMAT,
         )
 
     return modified_at or "--"
@@ -93,4 +94,9 @@ def get_rarlist_from_url(url) -> list[RarInfo]:
     to download it partially and fetch only file list."""
     resp = requests.get(url)
 
-    return rarfile.RarFile(BytesIO(resp.content)).infolist()
+    archive = rarfile.RarFile(BytesIO(resp.content))
+
+    if archive.needs_password():
+        raise unf_exception.UnfoldError("Archive is protected with password")
+
+    return archive.infolist()

ckanext/unfold/adapters/tar.py

@@ -11,6 +11,7 @@
 
 import ckan.plugins.toolkit as tk
 
+import ckanext.unfold.exception as unf_exception
 import ckanext.unfold.types as unf_types
 import ckanext.unfold.utils as unf_utils
 
@@ -27,10 +28,14 @@ def build_directory_tree(
             file_list = get_tarlist_from_url(filepath)
         else:
             with tarfile.open(filepath, mode) as archive:
+                # TODO: tarfile library doesn't have built-in support
+                # for checking whether a TAR file is protected with a password
+                # investigate it if someone will have such a problem lately
                 file_list: list[TarInfo] = archive.getmembers()
     except TarError as e:
-        log.error(f"Error openning rar archive: {e}")
-        return []
+        raise unf_exception.UnfoldError(f"Error openning archive: {e}")
+    except requests.RequestException as e:
+        raise unf_exception.UnfoldError(f"Error fetching remote archive: {e}")
 
     nodes: list[unf_types.Node] = []
 

ckanext/unfold/adapters/zip.py

@@ -10,6 +10,7 @@
 
 import ckan.plugins.toolkit as tk
 
+import ckanext.unfold.exception as unf_exception
 import ckanext.unfold.types as unf_types
 import ckanext.unfold.utils as unf_utils
 
@@ -24,13 +25,15 @@ def build_directory_tree(
             file_list = get_ziplist_from_url(filepath)
         else:
             with ZipFile(filepath) as archive:
+                # zip format do not support encrypted filenames so we don't have
+                # to check for pass protection, we have enough information from
+                # `infolist` method
+
                 file_list: list[ZipInfo] = archive.infolist()
     except (LargeZipFile, BadZipFile) as e:
-        log.error(f"Error openning archive: {e}")
-        return []
+        raise unf_exception.UnfoldError(f"Error openning archive: {e}")
     except requests.RequestException as e:
-        log.error(f"Error fetching remote archive: {e}")
-        return []
+        raise unf_exception.UnfoldError(f"Error fetching remote archive: {e}")
 
     nodes: list[unf_types.Node] = []
 

ckanext/unfold/assets/webassets.yml

@@ -2,7 +2,7 @@ unfold-js:
   filter: rjsmin
   output: ckanext-unfold/%(version)s-unfold.js
   contents:
-    - unfold-init-jstree.js
+    - js/unfold-init-jstree.js
     - vendor/jstree.min.js
     - vendor/jstree-table.js
   extra:

ckanext/unfold/logic/action.py

@@ -2,18 +2,16 @@
 from ckan.plugins import toolkit as tk
 
 import ckanext.unfold.exception as unf_exception
-import ckanext.unfold.helpers as unf_helpers
 import ckanext.unfold.logic.schema as unf_schema
+import ckanext.unfold.utils as unf_utils
 
 
 @tk.side_effect_free
 @validate(unf_schema.get_archive_structure)
 def get_archive_structure(context, data_dict):
-    # TODO: use it, rewrite the init tree script
-    # make ajax call and init it only if there's no error
     resource = tk.get_action("resource_show")(context, {"id": data_dict["id"]})
 
     try:
-        return unf_helpers.get_archive_tree(resource)
+        return unf_utils.get_archive_tree(resource)
     except unf_exception.UnfoldError as e:
         return {"error": str(e)}

ckanext/unfold/plugin.py

@@ -11,9 +11,7 @@
 from ckanext.unfold.logic.schema import get_preview_schema
 
 
-@tk.blanket.helpers
 @tk.blanket.actions
-# @tk.blanket.blueprints
 class UnfoldPlugin(plugins.SingletonPlugin):
     plugins.implements(plugins.IConfigurer)
     plugins.implements(plugins.IResourceView, inherit=True)

ckanext/unfold/templates/unfold_preview.html

@@ -1,12 +1,11 @@
-{#} {% set archive_tree = h.get_archive_tree(resource) %} {#}
-        {#}data-module-data="{{ archive_tree }}" {#}
-
 <div class="unfold-preview">
     <div class="ckanext-jstree-search">
         <div class="jstree-search--input">
             <label class="d-none" for="field-sitewide-search">{{ _('Search') }}</label>
-            <input id="jstree-search" class="form-control"  type="text" name="jstree-search" placeholder="{{ _('Search') }}" aria-label="{{ _('Search') }}"/>
-            <button class="btn search-icon" type="submit" aria-label="{{ _('Search icon') }}"><i class="fa fa-search"></i></button>
+            <input id="jstree-search" class="form-control" type="text" name="jstree-search"
+                placeholder="{{ _('Search') }}" aria-label="{{ _('Search') }}" />
+            <button class="btn search-icon" type="submit" aria-label="{{ _('Search icon') }}"><i
+                    class="fa fa-search"></i></button>
         </div>
 
         <div class="jstree-search--clear">

ckanext/unfold/tests/test_unfold.py

@@ -1,5 +1,4 @@
 import os
-from io import BytesIO
 
 import pytest
 

ckanext/unfold/utils.py

@@ -1,15 +1,20 @@
 from __future__ import annotations
 
 import json
+import logging
 import math
 import pathlib
+from typing import Any, Optional
 
+import ckan.lib.uploader as uploader
 from ckan.lib.redis import connect_to_redis
 
+import ckanext.unfold.adapters as unf_adapters
 import ckanext.unfold.types as unf_types
-
+import ckanext.unfold.utils as unf_utils
 
 DEFAULT_DATE_FORMAT = "%d/%m/%Y - %H:%M"
+log = logging.getLogger(__name__)
 
 
 def get_icon_by_format(fmt: str) -> str:
@@ -76,3 +81,34 @@ def delete_archive_structure(resource_id: str) -> None:
     conn = connect_to_redis()
     conn.delete(f"ckanext:unfold:tree:{resource_id}")
     conn.close()
+
+
+def get_archive_tree(resource: dict[str, Any]) -> list[unf_types.Node]:
+    remote = False
+
+    if resource.get("url_type") == "upload":
+        upload = uploader.get_resource_uploader(resource)
+        filepath = upload.get_path(resource["id"])
+    else:
+        if not resource.get("url"):
+            return []
+
+        filepath = resource["url"]
+        remote = True
+
+    tree = unf_utils.get_archive_structure(resource["id"])
+
+    if not tree:
+        tree = parse_archive(resource["format"].lower(), filepath, remote)
+        unf_utils.save_archive_structure(tree, resource["id"])
+
+    return tree
+
+
+def parse_archive(
+    fmt: str, filepath: str, remote: Optional[bool] = False
+) -> list[unf_types.Node]:
+    if fmt not in unf_adapters.ADAPTERS:
+        raise TypeError(f"No adapter for `{fmt}` archives")
+
+    return unf_adapters.ADAPTERS[fmt](filepath, remote=remote)