Merge pull request #2730 from smilerz/orphan_file_cleanup

view and delete orphaned files

cookbook/helper/HelperFunctions.py

@@ -7,7 +7,7 @@ class Round(Func):
 
 
 def str2bool(v):
-    if type(v) == bool or v is None:
+    if isinstance(v, bool) or v is None:
         return v
     else:
         return v.lower() in ("yes", "true", "1")

cookbook/helper/recipe_search.py

@@ -309,7 +309,7 @@ def _recently_viewed(self, num_recent=None):
 
     def _favorite_recipes(self, times_cooked=None):
         if self._sort_includes('favorite') or times_cooked:
-            less_than = '-' in (times_cooked or []) and not self._sort_includes('-favorite')
+            less_than = '-' in (str(times_cooked) or []) and not self._sort_includes('-favorite')
             if less_than:
                 default = 1000
             else:

cookbook/templates/system.html

@@ -83,22 +83,50 @@ <h4 class="mt-3">{% trans 'Debug Mode' %} <span
         {% trans 'Everything is fine!' %}
     {% endif %}
 
-    <h4 class="mt-3">{% trans 'Database' %} <span
-            class="badge badge-{% if postgres %}warning{% else %}success{% endif %}">{% if postgres %}
-        {% trans 'Info' %}{% else %}{% trans 'Ok' %}{% endif %}</span></h4>
-    {% if postgres %}
-        {% blocktrans %}
-            This application is not running with a Postgres database backend. This is ok but not recommended as some
-            features only work with postgres databases.
-        {% endblocktrans %}
-    {% else %}
+    <h4 class="mt-3">{% trans 'Database' %} 
+        <span class="badge badge-{{postgres_status}}">
+            {% if postgres_status == 'warning' %}
+                {% trans 'Info' %}
+            {% elif postgres_status == 'danger'%}
+                {% trans 'Warning' %}
+            {% else %}
+                {% trans 'Ok' %}
+            {% endif %}
+        </span>
+    </h4>
+    {{postgres_message}}
+
+    <h4 class="mt-3">
+    {% trans 'Orphaned Files' %}
+
+    <span class="badge badge-{% if orphans|length == 0 %}success{% elif orphans|length <= 25 %}warning{% else %}danger{% endif %}">
+        {% if orphans|length == 0 %}{% trans 'Success' %}
+        {% elif orphans|length <= 25 %}{% trans 'Warning' %}
+        {% else %}{% trans 'Danger' %}
+        {% endif %}
+    </span>
+    </h4>
+
+    {% if orphans|length == 0 %}
         {% trans 'Everything is fine!' %}
+    {% else %}
+        {% blocktrans with orphan_count=orphans|length  %}
+        There are currently {{ orphan_count }} orphaned files.
+        {% endblocktrans %}
+        <br>
+        <button id="toggle-button" class="btn btn-info btn-sm"  onclick="toggleOrphans()">{% trans 'Show' %}</button>
+        <button class="btn btn-info btn-sm" onclick="deleteOrphans()">{% trans 'Delete' %}</button>
     {% endif %}
+    <textarea id="orphans-list" style="display:none;" class="form-control" rows="20">
+{% for orphan in orphans %}{{ orphan }}
+{% endfor %}
+    </textarea>
 
     <h4 class="mt-3">Debug</h4>
     <textarea class="form-control" rows="20">
 Gunicorn Media: {{ gunicorn_media }}
-Sqlite: {{ postgres }}
+Sqlite: {% if postgres %} {% trans 'False' %} {% else %} {% trans 'True' %} {% endif %}
+{% if postgres %}PostgreSQL: {{postgres_version}} {% endif %}
 Debug: {{ debug }}
 
 {% for key,value in request.META.items %}{% if key in 'SERVER_PORT,REMOTE_HOST,REMOTE_ADDR,SERVER_PROTOCOL' %}{{ key }}:{{ value }}
@@ -110,4 +138,30 @@ <h4 class="mt-3">Debug</h4>
     </textarea>
     <br/>
     <br/>
-{% endblock %}
+    <form method="POST" id="delete-form">
+        {% csrf_token %}
+        <input type="hidden" name="delete_orphans" value="false">
+    </form>
+{% block script %}
+<script>
+    function toggleOrphans() {
+        var orphansList = document.getElementById('orphans-list');
+        var button = document.getElementById('toggle-button');
+
+        if (orphansList.style.display === 'none') {
+            orphansList.style.display = 'block';
+            button.innerText = "{% trans 'Hide' %}";
+        } else {
+            orphansList.style.display = 'none';
+            button.innerText = "{% trans 'Show' %}";
+        }
+    }
+    function deleteOrphans() {
+        document.getElementById('delete-form').delete_orphans.value = 'true';
+        document.getElementById('delete-form').submit(); 
+    }
+</script>
+{% endblock script %}
+{% endblock %}
+
+

cookbook/version_info.py

@@ -1,3 +1,3 @@
 TANDOOR_VERSION = ""
-TANDOOR_REF = ""
-VERSION_INFO = []
+TANDOOR_REF = "abf8f791360b2bc4a5c7d011877668679bcbb3f2"
+VERSION_INFO = [{'name': 'Tandoor ', 'version': "commit abf8f791360b2bc4a5c7d011877668679bcbb3f2\nMerge: 4723a7ec fd028047\nAuthor: vabene1111 \nDate:   Sun Dec 3 14:10:28 2023 +0100\n\n    Merge branch 'develop'\n    \n    # Conflicts:\n    #       docs/faq.md\n", 'website': 'https://github.com/TandoorRecipes/recipes', 'commit_link': 'https://github.com/TandoorRecipes/recipes/commit/abf8f791360b2bc4a5c7d011877668679bcbb3f2', 'ref': 'abf8f791360b2bc4a5c7d011877668679bcbb3f2', 'branch': 'HEAD', 'tag': ''}]

cookbook/views/views.py

@@ -3,12 +3,14 @@
 from datetime import datetime
 from uuid import UUID
 
+from django.apps import apps
 from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.decorators import login_required
 from django.contrib.auth.models import Group
 from django.contrib.auth.password_validation import validate_password
 from django.core.exceptions import ValidationError
+from django.db import models
 from django.http import HttpResponseRedirect
 from django.shortcuts import get_object_or_404, redirect, render
 from django.urls import reverse, reverse_lazy
@@ -18,6 +20,7 @@
 
 from cookbook.forms import (CommentForm, Recipe, SearchPreferenceForm, SpaceCreateForm,
                             SpaceJoinForm, User, UserCreateForm, UserPreference)
+from cookbook.helper.HelperFunctions import str2bool
 from cookbook.helper.permission_helper import (group_required, has_group_permission,
                                                share_link_valid, switch_user_active_space)
 from cookbook.models import (Comment, CookLog, InviteLink, SearchFields, SearchPreference,
@@ -225,10 +228,10 @@ def shopping_settings(request):
                 if not sp:
                     sp = SearchPreferenceForm(user=request.user)
                 fields_searched = (
-                        len(search_form.cleaned_data['icontains'])
-                        + len(search_form.cleaned_data['istartswith'])
-                        + len(search_form.cleaned_data['trigram'])
-                        + len(search_form.cleaned_data['fulltext'])
+                    len(search_form.cleaned_data['icontains'])
+                    + len(search_form.cleaned_data['istartswith'])
+                    + len(search_form.cleaned_data['trigram'])
+                    + len(search_form.cleaned_data['fulltext'])
                 )
                 if search_form.cleaned_data['preset'] == 'fuzzy':
                     sp.search = SearchPreference.SIMPLE
@@ -314,17 +317,48 @@ def system(request):
     if not request.user.is_superuser:
         return HttpResponseRedirect(reverse('index'))
 
+    postgres_ver = None
     postgres = settings.DATABASES['default']['ENGINE'] == 'django.db.backends.postgresql'
 
+    if postgres:
+        postgres_current = 16  # will need to be updated as PostgreSQL releases new major versions
+        from decimal import Decimal
+
+        from django.db import connection
+
+        postgres_ver = Decimal(str(connection.pg_version).replace('00', '.'))
+        if postgres_ver >= postgres_current:
+            database_status = 'success'
+            database_message = _('Everything is fine!')
+        elif postgres_ver < postgres_current - 2:
+            database_status = 'danger'
+            database_message = _('PostgreSQL %(v)s is deprecated.  Upgrade to a fully supported version!') % {'v': postgres_ver}
+        else:
+            database_status = 'info'
+            database_message = _('You are running PostgreSQL %(v1)s.  PostgreSQL %(v2)s is recommended') % {'v1': postgres_ver, 'v2': postgres_current}
+    else:
+        database_status = 'info'
+        database_message = _('This application is not running with a Postgres database backend. This is ok but not recommended as some features only work with postgres databases.')
+
     secret_key = False if os.getenv('SECRET_KEY') else True
 
+    if request.method == "POST":
+        del_orphans = request.POST.get('delete_orphans')
+        orphans = get_orphan_files(delete_orphans=str2bool(del_orphans))
+    else:
+        orphans = get_orphan_files()
+
     return render(request, 'system.html', {
         'gunicorn_media': settings.GUNICORN_MEDIA,
         'debug': settings.DEBUG,
         'postgres': postgres,
+        'postgres_version': postgres_ver,
+        'postgres_status': database_status,
+        'postgres_message': database_message,
         'version_info': VERSION_INFO,
         'plugins': PLUGINS,
-        'secret_key': secret_key
+        'secret_key': secret_key,
+        'orphans': orphans
     })
 
 
@@ -448,3 +482,47 @@ def test(request):
 def test2(request):
     if not settings.DEBUG:
         return HttpResponseRedirect(reverse('index'))
+
+
+def get_orphan_files(delete_orphans=False):
+    # Get list of all image files in media folder
+    media_dir = settings.MEDIA_ROOT
+
+    def find_orphans():
+        image_files = []
+        for root, dirs, files in os.walk(media_dir):
+            for file in files:
+
+                if not file.lower().endswith(('.db')) and not root.lower().endswith(('@eadir')):
+                    full_path = os.path.join(root, file)
+                    relative_path = os.path.relpath(full_path, media_dir)
+                    image_files.append((relative_path, full_path))
+
+        # Get list of all image fields in models
+        image_fields = []
+        for model in apps.get_models():
+            for field in model._meta.get_fields():
+                if isinstance(field, models.ImageField) or isinstance(field, models.FileField):
+                    image_fields.append((model, field.name))
+
+        # get all images in the database
+        # TODO I don't know why, but this completely bypasses scope limitations
+        image_paths = []
+        for model, field in image_fields:
+            image_field_paths = model.objects.values_list(field, flat=True)
+            image_paths.extend(image_field_paths)
+
+        # Check each image file against model image fields
+        return [img for img in image_files if img[0] not in image_paths]
+    orphans = find_orphans()
+    if delete_orphans:
+        for f in [img[1] for img in orphans]:
+            try:
+                os.remove(f)
+            except FileNotFoundError:
+                print(f"File not found: {f}")
+            except Exception as e:
+                print(f"Error deleting file {f}: {e}")
+        orphans = find_orphans()
+
+    return [img[1] for img in orphans]

recipes/settings.py

@@ -438,7 +438,7 @@
     if p['bundle_name'] != '':
         WEBPACK_LOADER[p['bundle_name']] = {
             'CACHE': not DEBUG,
-            'BUNDLE_DIR_NAME': f'vue/',  # must end with slash
+            'BUNDLE_DIR_NAME': 'vue/',  # must end with slash
             'STATS_FILE': os.path.join(p["base_path"], 'vue', 'webpack-stats.json'),
             'POLL_INTERVAL': 0.1,
             'TIMEOUT': None,

version.py

@@ -13,12 +13,11 @@
 try:
     print('getting tandoor version')
     r = subprocess.check_output(['git', 'show', '-s'], cwd=BASE_DIR).decode()
-    tandoor_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=BASE_DIR).decode()
+    tandoor_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], cwd=BASE_DIR).decode().replace('\n', '')
     tandoor_hash = r.split('\n')[0].split(' ')[1]
     try:
-        tandoor_tag = subprocess.check_output(['git', 'describe', '--exact-match', tandoor_hash], cwd=BASE_DIR).decode().replace('\n', '')
-    except:
-
+        tandoor_tag = subprocess.check_output(['git', 'describe', '--exact-match', '--tags', tandoor_hash], cwd=BASE_DIR).decode().replace('\n', '')
+    except BaseException:
         pass
 
     version_info.append({
@@ -47,8 +46,9 @@
                     commit_hash = r.split('\n')[0].split(' ')[1]
                     try:
                         print('running describe')
-                        tag = subprocess.check_output(['git', 'describe', '--exact-match', commit_hash], cwd=os.path.join(BASE_DIR, 'recipes', 'plugins', d)).decode().replace('\n', '')
-                    except:
+                        tag = subprocess.check_output(['git', 'describe', '--exact-match', commit_hash],
+                                                      cwd=os.path.join(BASE_DIR, 'recipes', 'plugins', d)).decode().replace('\n', '')
+                    except BaseException:
                         tag = ''
 
                     version_info.append({
@@ -66,7 +66,7 @@
                     traceback.print_exc()
 except subprocess.CalledProcessError as e:
     print("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output))
-except:
+except BaseException:
     traceback.print_exc()
 
 with open('cookbook/version_info.py', 'w+', encoding='UTF-8') as f: