test unloading...

.pre-commit-config.yaml

@@ -1,26 +1,26 @@
 fail_fast: true
 repos:
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.17.0
+    rev: v3.19.1
     hooks:
       - id: pyupgrade
         args:
           - --py38-plus
           - --keep-runtime-typing
 
   - repo: https://github.com/ambv/black
-    rev: 24.8.0
+    rev: 24.10.0
     hooks:
       - id: black
         language_version: python3.10
 
   - repo: https://github.com/abravalheri/validate-pyproject
-    rev: v0.19
+    rev: v0.23
     hooks:
       - id: validate-pyproject
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.1
+    rev: v1.14.1
     hooks:
       - id: mypy
         verbose: true
@@ -93,7 +93,7 @@ repos:
   #    - id: pydocstyle
 
   - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.17
+    rev: 0.7.21
     hooks:
       - id: mdformat
         additional_dependencies:
@@ -148,7 +148,7 @@ repos:
   #          ]
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0 # Use the ref you want to point at
+    rev: v5.0.0 # Use the ref you want to point at
     hooks:
       - id: check-added-large-files
         name: check for added large files
@@ -190,7 +190,7 @@ repos:
         entry: check-executables-have-shebangs
         language: python
         types: [text, executable]
-        stages: [commit, push, manual]
+        stages: [pre-commit, pre-push, manual]
       - id: check-json
         name: check json
         description: checks json files for parseable syntax.
@@ -203,7 +203,7 @@ repos:
         entry: check-shebang-scripts-are-executable
         language: python
         types: [text]
-        stages: [commit, push, manual]
+        stages: [pre-commit, pre-push, manual]
       - id: pretty-format-json
         name: pretty format json
         description: sets a standard for formatting json files.
@@ -282,7 +282,7 @@ repos:
         entry: end-of-file-fixer
         language: python
         types: [text]
-        stages: [commit, push, manual]
+        stages: [pre-commit, pre-push, manual]
       - id: file-contents-sorter
         name: file contents sorter
         description: sorts the lines in specified files (defaults to alphabetical). you must provide list of target files as input in your .pre-commit-config.yaml file.
@@ -331,4 +331,4 @@ repos:
         entry: trailing-whitespace-fixer
         language: python
         types: [text]
-        stages: [commit, push, manual]
+        stages: [pre-commit, pre-push, manual]

setup.py

@@ -40,7 +40,7 @@ def recursive_flatten_ignore_str(seq: Sequence) -> Sequence:
     def unroll_nested_reqs(req_str: str, base_path: Path):
         """description"""
         if req_str.startswith("-r"):
-            with open(base_path / req_str.strip("-r").strip()) as f:
+            with open(base_path / req_str.replace("-r", "").strip()) as f:
                 return [unroll_nested_reqs(req.strip(), base_path) for req in readlines_ignore_comments(f)]
         else:
             return (req_str,)
@@ -162,7 +162,7 @@ def extras(self) -> dict:
 
         for file in path.iterdir():
             if file.name.startswith("requirements_"):
-                group_name_ = "_".join(file.name.strip(".txt").split("_")[1:])
+                group_name_ = "_".join(file.name.replace(".txt", "").split("_")[1:])
                 these_extras[group_name_] = read_reqs(file.name, path)
 
         all_dependencies = []

tests/test_unload/some_module/__init__.py

@@ -0,0 +1 @@
+__doc__ = f"This is {__name__}"

tests/test_unload/some_module_bak/__init__.py

@@ -0,0 +1 @@
+__doc__ = f"This is {__name__}"

tests/test_unload/unload.py

@@ -0,0 +1,86 @@
+import gc
+import os
+import shutil
+import sys
+
+from pathlib import Path
+from stat import filemode
+
+from warg.runtime import unload_modules
+
+if __name__ == "__main__":
+
+    def main():
+        # print(sys.modules)
+
+        import some_module
+
+        module_path = Path(some_module.__path__[0])
+
+        # print( os.access(module_path, os.F_OK | os.X_OK))
+        shutil.rmtree(module_path)
+
+        if True:
+            with os.scandir(module_path) as scandir_it:
+                entries = list(scandir_it)
+            for entry in entries:
+                fullname = entry.path
+                try:
+                    os.unlink(fullname)
+                except Exception as e:
+                    print(e)
+
+            os.rmdir(module_path)
+
+        failed = False
+        try:
+            os.unlink(module_path)
+        except PermissionError:
+            failed = True
+
+        if not failed:
+            raise Exception()
+
+        print(some_module.__doc__)
+
+        print(gc.get_stats())
+        # print(gc.get_referrers(some_module))
+        # print(gc.get_referents(some_module))
+        print(gc.is_tracked(some_module))
+        print(gc.is_finalized(some_module))
+        # print(gc.get_objects())
+
+        del some_module
+        unload_modules()
+
+        # print(sys.meta_path)
+        print(gc.garbage)
+        print(gc.callbacks)
+        # print(gc.get_objects())
+        # print(gc.is_finalized())
+        gc.collect(generation=2)
+
+        # folder_stat = os.stat(module_path)
+        # for mode in (folder_stat.st_mode,):
+        #  print(filemode(mode))
+
+        # fd = os.open(module_path, os.O_RDONLY)
+        # os.close(fd)
+
+        # print( os.lstat(module_path))
+
+        # print(os.statvfs(module_path))
+        # os.unlink(module_path)
+
+        # sys.audit("os.unlink", str(module_path))
+        # print(sys.exc_info())
+
+        # sys.audit("shutil.rmtree", str(module_path))
+
+        # remove_tree(module_path,dry_run=True)
+
+        # gc.is_finalized(some_module)
+
+        # print(sys.modules)
+
+    main()

warg/__init__.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-import logging
 
 
 __project__ = "Warg"

warg/data_structures/mappings.py

@@ -3,13 +3,7 @@
 from typing import Callable, Dict, Hashable, Iterable, Mapping, MutableMapping
 
 logger = logging.getLogger(__name__)
-__all__ = [
-    "invert_mapping",
-    "invert_dict",
-    "AppendingDict",
-    "pivot_dict_object",
-    "pivot_dict",
-]
+__all__ = ["invert_mapping", "invert_dict", "AppendingDict", "pivot_dict_object", "pivot_dict", "to_dict"]
 
 
 def append_to_dict(d: Dict, key, value) -> Dict:

warg/functions.py

@@ -422,7 +422,7 @@ def open_uri_resource(uri: str) -> str:
         webbrowser.open_new_tab(uri)
     else:
         try:
-            uri = f'https://{uri.lstrip("https://")}'
+            uri = f'https://{uri.replace("https://","")}'
             r = requests.head(uri)
             if r.ok:  # it is a boolean
                 try:

warg/iteration.py

@@ -1,14 +1,18 @@
 #!/usr/bin/env python3
 
 import logging
-from typing import Any, Callable, Iterable, List, Sequence, Tuple
+from itertools import pairwise
+from typing import Any, Callable, Iterable, List, Sequence, Sized, Tuple
 
 logger = logging.getLogger(__name__)
 __all__ = ["pairs", "chunks", "leaf_apply", "leaf_type_apply"]
 
 
-def pairs(s: Sequence) -> Tuple[Any, Any]:
+def pairs(s: Iterable) -> Tuple[Any, Any]:
     """
+
+    NOTE: Just use itertools.pairwise....
+
     Iterate over a list in overlapping pairs.
 
     Usage:
@@ -21,12 +25,14 @@ def pairs(s: Sequence) -> Tuple[Any, Any]:
     :param s: An iterable/list
     :return: Yields a pair of consecutive elements (lst[k], lst[k+1]) of lst. Last call yields (lst[-2], lst[-1]).
     """
-    i = iter(s)
-    prev = next(i)
+    yield from pairwise(s)
 
-    for item in i:
-        yield prev, item
-        prev = item
+    # i = iter(s)
+    # prev = next(i)
+    #
+    # for item in i:
+    #     yield prev, item
+    #     prev = item
 
 
 def leaf_apply(seq: Iterable, func: Callable) -> List:
@@ -49,7 +55,7 @@ def leaf_type_apply(seq: Iterable, func: Callable, leaf_type: type = tuple) -> L
     return sub
 
 
-def chunks(lst: Sequence, n: int) -> Any:
+def chunks(lst: Sized, n: int) -> Any:
     """
     Yield successive n-sized chunks from lst.
 
@@ -63,3 +69,4 @@ def chunks(lst: Sequence, n: int) -> Any:
 
 if __name__ == "__main__":
     logger.info(list(chunks(list(range(10)), 3)))
+    logger.warning(list(pairs(list(range(10)))))

warg/runtime.py

@@ -0,0 +1,40 @@
+__all__ = ["unload_modules"]
+
+import importlib
+import sys
+
+BUILT_IN = [*sys.builtin_module_names, *sys.stdlib_module_names]
+
+
+def unload_modules(*modules: str) -> None:
+    if not modules:
+        modules = sys.modules.copy()
+
+    for sm in modules.keys():
+        if sm in BUILT_IN:
+            continue
+
+        del sys.modules[sm]
+
+        for mod in modules.values():
+            try:
+                delattr(mod, sm)
+            except AttributeError:
+                pass
+
+    importlib.invalidate_caches()
+    # sys._clear_internal_caches()
+
+
+if __name__ == "__main__":
+
+    def main():
+        import argparse
+
+        print(argparse)
+
+        unload_modules()
+
+        print(sys.modules)
+
+    main()