From cff0dc99fb95dcce79bbfd34c6ef2ea187618134 Mon Sep 17 00:00:00 2001 From: rup-narayan-rajbanshi Date: Tue, 3 Sep 2024 15:54:31 +0545 Subject: [PATCH 1/7] Secure files extending file name with uuid. --- .../0013_alter_flashgraphicmap_file.py | 19 +++++++++++++++++++ .../0014_alter_flashupdate_extracted_file.py | 19 +++++++++++++++++++ flash_update/models.py | 12 ++++++++++-- main/utils.py | 16 ++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 flash_update/migrations/0013_alter_flashgraphicmap_file.py create mode 100644 flash_update/migrations/0014_alter_flashupdate_extracted_file.py diff --git a/flash_update/migrations/0013_alter_flashgraphicmap_file.py b/flash_update/migrations/0013_alter_flashgraphicmap_file.py new file mode 100644 index 000000000..586af05b4 --- /dev/null +++ b/flash_update/migrations/0013_alter_flashgraphicmap_file.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.15 on 2024-09-03 09:37 + +from django.db import migrations, models +import flash_update.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('flash_update', '0012_auto_20230410_0720'), + ] + + operations = [ + migrations.AlterField( + model_name='flashgraphicmap', + name='file', + field=models.FileField(upload_to=flash_update.models.flash_map_upload_to, verbose_name='file'), + ), + ] diff --git a/flash_update/migrations/0014_alter_flashupdate_extracted_file.py b/flash_update/migrations/0014_alter_flashupdate_extracted_file.py new file mode 100644 index 000000000..08a63456d --- /dev/null +++ b/flash_update/migrations/0014_alter_flashupdate_extracted_file.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.15 on 2024-09-03 09:42 + +from django.db import migrations, models +import flash_update.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('flash_update', '0013_alter_flashgraphicmap_file'), + ] + + operations = [ + migrations.AlterField( + model_name='flashupdate', + name='extracted_file', + field=models.FileField(blank=True, null=True, upload_to=flash_update.models.flash_extracted_file_upload_to, verbose_name='extracted file'), + ), + ] diff --git a/flash_update/models.py b/flash_update/models.py index 3de787818..53a70ef04 100644 --- a/flash_update/models.py +++ b/flash_update/models.py @@ -1,3 +1,5 @@ +# import os +# from uuid import uuid4 import reversion from django.conf import settings from django.contrib.auth.models import Group @@ -6,6 +8,7 @@ from django.utils.translation import gettext_lazy as _ from tinymce.models import HTMLField +from main.utils import custom_upload_to from api.models import ( ActionCategory, ActionOrg, @@ -15,10 +18,15 @@ District, ) +def flash_map_upload_to(instance, filename): + return custom_upload_to('flash_update/images/')(instance, filename) + +def flash_extracted_file_upload_to(instance, filename): + return custom_upload_to('flash_update/pdf/')(instance, filename) @reversion.register() class FlashGraphicMap(models.Model): - file = models.FileField(verbose_name=_("file"), upload_to="flash_update/images/") + file = models.FileField(verbose_name=_("file"), upload_to=flash_map_upload_to) caption = models.CharField(max_length=225, blank=True, null=True) created_by = models.ForeignKey( settings.AUTH_USER_MODEL, @@ -116,7 +124,7 @@ class FlashShareWith(models.TextChoices): verbose_name=_("share with"), ) references = models.ManyToManyField(FlashReferences, blank=True, verbose_name=_("references")) - extracted_file = models.FileField(verbose_name=_("extracted file"), upload_to="flash_update/pdf/", blank=True, null=True) + extracted_file = models.FileField(verbose_name=_("extracted file"), upload_to=flash_extracted_file_upload_to, blank=True, null=True) extracted_at = models.DateTimeField(verbose_name=_("extracted at"), blank=True, null=True) class Meta: diff --git a/main/utils.py b/main/utils.py index a9ef3bba4..f51b0c220 100644 --- a/main/utils.py +++ b/main/utils.py @@ -1,6 +1,8 @@ +import os import datetime import json import typing +from uuid import uuid4 from collections import defaultdict from tempfile import NamedTemporaryFile, _TemporaryFileWrapper @@ -15,6 +17,20 @@ from reversion.revisions import _get_options +def custom_upload_to(directory): + """ + Rename file name with adding uuid + """ + def upload_to(instance, filename): + # Get the file extension + extension = filename.split('.')[-1] + old_file_name = filename.split('.')[0] + # Create a unique filename using uuid4 + new_filename = f"{old_file_name}-{uuid4().hex}.{extension}" + # Return the new file path + return os.path.join(directory, new_filename) + return upload_to + def is_tableau(request): """Checking the request for the 'tableau' parameter (used mostly for switching to the *TableauSerializers) From 67f200f7a22f2fc29ba51605cc53c5f5d5a71b03 Mon Sep 17 00:00:00 2001 From: rup-narayan-rajbanshi Date: Wed, 4 Sep 2024 11:53:57 +0545 Subject: [PATCH 2/7] Add custom SecureFileField class in FileField. --- api/models.py | 6 ++-- country_plan/models.py | 3 +- dref/models.py | 9 +++--- .../0013_alter_flashgraphicmap_file.py | 19 ------------ .../0014_alter_flashupdate_extracted_file.py | 19 ------------ flash_update/models.py | 11 ++----- main/fields.py | 30 +++++++++++++++++++ main/utils.py | 16 ---------- per/models.py | 5 ++-- 9 files changed, 47 insertions(+), 71 deletions(-) delete mode 100644 flash_update/migrations/0013_alter_flashgraphicmap_file.py delete mode 100644 flash_update/migrations/0014_alter_flashupdate_extracted_file.py create mode 100644 main/fields.py diff --git a/api/models.py b/api/models.py index afe4f0cc2..da0d09567 100644 --- a/api/models.py +++ b/api/models.py @@ -22,6 +22,8 @@ from django.utils.translation import gettext_lazy as _ from tinymce.models import HTMLField +from main.fields import SecureFileField + from .utils import validate_slug_number # is_user_ifrc, @@ -984,7 +986,7 @@ def sitrep_document_path(instance, filename): class SituationReport(models.Model): created_at = models.DateTimeField(verbose_name=_("created at"), auto_now_add=True) name = models.CharField(verbose_name=_("name"), max_length=100) - document = models.FileField(verbose_name=_("document"), null=True, blank=True, upload_to=sitrep_document_path) + document = SecureFileField(verbose_name=_("document"), null=True, blank=True, upload_to=sitrep_document_path) document_url = models.URLField(verbose_name=_("document url"), blank=True) event = models.ForeignKey(Event, verbose_name=_("event"), on_delete=models.CASCADE) @@ -1287,7 +1289,7 @@ class GeneralDocument(models.Model): name = models.CharField(verbose_name=_("name"), max_length=100) # Don't set `auto_now_add` so we can modify it on save created_at = models.DateTimeField(verbose_name=_("created at"), blank=True) - document = models.FileField(verbose_name=_("document"), null=True, blank=True, upload_to=general_document_path) + document = SecureFileField(verbose_name=_("document"), null=True, blank=True, upload_to=general_document_path) document_url = models.URLField(verbose_name=_("document url"), blank=True) class Meta: diff --git a/country_plan/models.py b/country_plan/models.py index 56dd05687..3fcb1f119 100644 --- a/country_plan/models.py +++ b/country_plan/models.py @@ -5,6 +5,7 @@ from django.utils.translation import gettext_lazy as _ from api.models import Country +from main.fields import SecureFileField def file_upload_to(instance, filename): @@ -63,7 +64,7 @@ def save(self, *args, **kwargs): class CountryPlan(CountryPlanAbstract): country = models.OneToOneField(Country, on_delete=models.CASCADE, related_name="country_plan", primary_key=True) - internal_plan_file = models.FileField( + internal_plan_file = SecureFileField( verbose_name=_("Internal Plan"), upload_to=pdf_upload_to, validators=[FileExtensionValidator(["pdf"])], diff --git a/dref/models.py b/dref/models.py index 11b5dd24b..f7023671a 100644 --- a/dref/models.py +++ b/dref/models.py @@ -12,6 +12,7 @@ from pdf2image import convert_from_bytes from api.models import Country, DisasterType, District, FieldReport +from main.fields import SecureFileField @reversion.register() @@ -536,7 +537,7 @@ class Status(models.IntegerChoices): verbose_name=_("budget file"), related_name="budget_file_dref", ) - budget_file_preview = models.FileField(verbose_name=_("budget file preview"), null=True, blank=True, upload_to="dref/images/") + budget_file_preview = SecureFileField(verbose_name=_("budget file preview"), null=True, blank=True, upload_to="dref/images/") assessment_report = models.ForeignKey( "DrefFile", on_delete=models.SET_NULL, @@ -648,7 +649,7 @@ def get_for(user): class DrefFile(models.Model): - file = models.FileField( + file = SecureFileField( verbose_name=_("file"), upload_to="dref/images/", ) @@ -750,7 +751,7 @@ class DrefOperationalUpdate(models.Model): verbose_name=_("budget file"), related_name="budget_file_dref_operational_update", ) - budget_file_preview = models.FileField( + budget_file_preview = SecureFileField( verbose_name=_("budget file preview"), null=True, blank=True, upload_to="dref-op-update/images/" ) assessment_report = models.ForeignKey( @@ -1316,7 +1317,7 @@ class DrefFinalReport(models.Model): verbose_name=_("financial report"), related_name="financial_report_dref_final_report", ) - financial_report_preview = models.FileField( + financial_report_preview = SecureFileField( verbose_name=_("financial preview"), null=True, blank=True, upload_to="dref/images/" ) num_assisted = models.IntegerField(verbose_name=_("number of assisted"), blank=True, null=True) diff --git a/flash_update/migrations/0013_alter_flashgraphicmap_file.py b/flash_update/migrations/0013_alter_flashgraphicmap_file.py deleted file mode 100644 index 586af05b4..000000000 --- a/flash_update/migrations/0013_alter_flashgraphicmap_file.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 4.2.15 on 2024-09-03 09:37 - -from django.db import migrations, models -import flash_update.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('flash_update', '0012_auto_20230410_0720'), - ] - - operations = [ - migrations.AlterField( - model_name='flashgraphicmap', - name='file', - field=models.FileField(upload_to=flash_update.models.flash_map_upload_to, verbose_name='file'), - ), - ] diff --git a/flash_update/migrations/0014_alter_flashupdate_extracted_file.py b/flash_update/migrations/0014_alter_flashupdate_extracted_file.py deleted file mode 100644 index 08a63456d..000000000 --- a/flash_update/migrations/0014_alter_flashupdate_extracted_file.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 4.2.15 on 2024-09-03 09:42 - -from django.db import migrations, models -import flash_update.models - - -class Migration(migrations.Migration): - - dependencies = [ - ('flash_update', '0013_alter_flashgraphicmap_file'), - ] - - operations = [ - migrations.AlterField( - model_name='flashupdate', - name='extracted_file', - field=models.FileField(blank=True, null=True, upload_to=flash_update.models.flash_extracted_file_upload_to, verbose_name='extracted file'), - ), - ] diff --git a/flash_update/models.py b/flash_update/models.py index 53a70ef04..967e21cfb 100644 --- a/flash_update/models.py +++ b/flash_update/models.py @@ -8,7 +8,6 @@ from django.utils.translation import gettext_lazy as _ from tinymce.models import HTMLField -from main.utils import custom_upload_to from api.models import ( ActionCategory, ActionOrg, @@ -17,16 +16,12 @@ DisasterType, District, ) +from main.fields import SecureFileField -def flash_map_upload_to(instance, filename): - return custom_upload_to('flash_update/images/')(instance, filename) - -def flash_extracted_file_upload_to(instance, filename): - return custom_upload_to('flash_update/pdf/')(instance, filename) @reversion.register() class FlashGraphicMap(models.Model): - file = models.FileField(verbose_name=_("file"), upload_to=flash_map_upload_to) + file = SecureFileField(verbose_name=_("file"), upload_to="flash_update/images") caption = models.CharField(max_length=225, blank=True, null=True) created_by = models.ForeignKey( settings.AUTH_USER_MODEL, @@ -124,7 +119,7 @@ class FlashShareWith(models.TextChoices): verbose_name=_("share with"), ) references = models.ManyToManyField(FlashReferences, blank=True, verbose_name=_("references")) - extracted_file = models.FileField(verbose_name=_("extracted file"), upload_to=flash_extracted_file_upload_to, blank=True, null=True) + extracted_file = SecureFileField(verbose_name=_("extracted file"), upload_to="flash_update/pdf/", blank=True, null=True) extracted_at = models.DateTimeField(verbose_name=_("extracted at"), blank=True, null=True) class Meta: diff --git a/main/fields.py b/main/fields.py new file mode 100644 index 000000000..5f1ad5671 --- /dev/null +++ b/main/fields.py @@ -0,0 +1,30 @@ +import datetime +import posixpath +from uuid import uuid4 + +from django.core.files.utils import validate_file_name +from django.db.models.fields.files import FileField + + +class SecureFileField(FileField): + def generate_filename(self, instance, filename): + """ + Apply (if callable) or prepend (if a string) upload_to to the filename, + then delegate further processing of the name to the storage backend. + Until the storage layer, all file paths are expected to be Unix style + (with forward slashes). + Add random uuid in the file name. + """ + extension = filename.split(".")[-1] + old_file_name = filename.split(".")[0] + # Create a unique filename using uuid4 + filename = f"{old_file_name}-{uuid4().hex}.{extension}" + + if callable(self.upload_to): + filename = self.upload_to(instance, filename) + else: + dirname = datetime.datetime.now().strftime(str(self.upload_to)) + filename = posixpath.join(dirname, filename) + filename = validate_file_name(filename, allow_relative_path=True) + + return self.storage.generate_filename(filename) diff --git a/main/utils.py b/main/utils.py index f51b0c220..a9ef3bba4 100644 --- a/main/utils.py +++ b/main/utils.py @@ -1,8 +1,6 @@ -import os import datetime import json import typing -from uuid import uuid4 from collections import defaultdict from tempfile import NamedTemporaryFile, _TemporaryFileWrapper @@ -17,20 +15,6 @@ from reversion.revisions import _get_options -def custom_upload_to(directory): - """ - Rename file name with adding uuid - """ - def upload_to(instance, filename): - # Get the file extension - extension = filename.split('.')[-1] - old_file_name = filename.split('.')[0] - # Create a unique filename using uuid4 - new_filename = f"{old_file_name}-{uuid4().hex}.{extension}" - # Return the new file path - return os.path.join(directory, new_filename) - return upload_to - def is_tableau(request): """Checking the request for the 'tableau' parameter (used mostly for switching to the *TableauSerializers) diff --git a/per/models.py b/per/models.py index df2355511..7af908e69 100644 --- a/per/models.py +++ b/per/models.py @@ -6,6 +6,7 @@ from api.models import Appeal, Country from deployments.models import SectorTag +from main.fields import SecureFileField class ProcessPhase(models.IntegerChoices): @@ -422,7 +423,7 @@ def save(self, *args, **kwargs): class PerFile(models.Model): - file = models.FileField( + file = SecureFileField( verbose_name=_("file"), upload_to="per/images/", ) @@ -738,7 +739,7 @@ def save(self, *args, **kwargs): class PerDocumentUpload(models.Model): - file = models.FileField( + file = SecureFileField( verbose_name=_("file"), upload_to="per/documents/", ) From f98ded00dde4126736e7a76879fc5ed0056dc778 Mon Sep 17 00:00:00 2001 From: rup-narayan-rajbanshi Date: Wed, 16 Oct 2024 17:55:06 +0545 Subject: [PATCH 3/7] Add test case for secure file field. --- api/test_views.py | 30 ++++++++++++++++++++++++++++++ main/fields.py | 25 ++++--------------------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/api/test_views.py b/api/test_views.py index ecb744887..ca7ab2dc7 100644 --- a/api/test_views.py +++ b/api/test_views.py @@ -1,6 +1,8 @@ import json +import uuid from django.contrib.auth.models import User +from django.core.files.uploadedfile import SimpleUploadedFile import api.models as models from api.factories.event import ( @@ -13,9 +15,37 @@ from api.factories.field_report import FieldReportFactory from api.models import Profile, VisibilityChoices from deployments.factories.user import UserFactory +from dref.models import DrefFile from main.test_case import APITestCase, SnapshotTestCase +class SecureFileFieldTest(APITestCase): + def is_valid_uuid(self, uuid_to_test): + try: + # Validate if the directory name is a valid UUID (hex) + uuid_obj = uuid.UUID(uuid_to_test, version=4) + except ValueError: + return False + return uuid_obj.hex == uuid_to_test + + def test_filefield_uuid_directory_and_filename(self): + # Mocking a file upload with SimpleUploadedFile + original_filename = "test_file.txt" + mock_file = SimpleUploadedFile(original_filename, b"file_content", content_type="text/plain") + # Create an instance of MyModel and save it with the mocked file + instance = DrefFile.objects.create(file=mock_file) + # Check the uploaded file name + uploaded_file_name = instance.file.name + # Example output: uploads/5f9d54c8b5a34d3e8fdc4e4f43e2f82a/test_file.txt + + # Extract UUID directory part and filename part + directory_name, file_name = uploaded_file_name.split("/")[-2:] + # Check that the directory name is a valid UUID (hexadecimal) + self.assertTrue(self.is_valid_uuid(directory_name)) + # Check that the file name retains the original name + self.assertEqual(file_name, original_filename) + + class GuestUserPermissionTest(APITestCase): def setUp(self): # Create guest user diff --git a/main/fields.py b/main/fields.py index 5f1ad5671..db3469e16 100644 --- a/main/fields.py +++ b/main/fields.py @@ -1,30 +1,13 @@ -import datetime -import posixpath from uuid import uuid4 -from django.core.files.utils import validate_file_name from django.db.models.fields.files import FileField class SecureFileField(FileField): def generate_filename(self, instance, filename): """ - Apply (if callable) or prepend (if a string) upload_to to the filename, - then delegate further processing of the name to the storage backend. - Until the storage layer, all file paths are expected to be Unix style - (with forward slashes). - Add random uuid in the file name. + Overwrites https://github.com/django/django/blob/main/django/db/models/fields/files.py#L345 """ - extension = filename.split(".")[-1] - old_file_name = filename.split(".")[0] - # Create a unique filename using uuid4 - filename = f"{old_file_name}-{uuid4().hex}.{extension}" - - if callable(self.upload_to): - filename = self.upload_to(instance, filename) - else: - dirname = datetime.datetime.now().strftime(str(self.upload_to)) - filename = posixpath.join(dirname, filename) - filename = validate_file_name(filename, allow_relative_path=True) - - return self.storage.generate_filename(filename) + # Append uuid4 path to the filename + filename = f"{uuid4().hex}/{filename}" + return super().generate_filename(instance, filename) # return self.storage.generate_filename(filename) From cac156a58d1d89d5975abecbc5324f77248b0e8a Mon Sep 17 00:00:00 2001 From: rup-narayan-rajbanshi Date: Thu, 17 Oct 2024 11:58:10 +0545 Subject: [PATCH 4/7] Add migration file for Secure File field. --- ...alter_generaldocument_document_and_more.py | 30 ++++++++++++++ ...08_alter_countryplan_internal_plan_file.py | 28 +++++++++++++ ...le_preview_alter_dreffile_file_and_more.py | 39 +++++++++++++++++++ ...013_alter_flashgraphicmap_file_and_more.py | 27 +++++++++++++ ...rdocumentupload_file_alter_perfile_file.py | 25 ++++++++++++ 5 files changed, 149 insertions(+) create mode 100644 api/migrations/0215_alter_generaldocument_document_and_more.py create mode 100644 country_plan/migrations/0008_alter_countryplan_internal_plan_file.py create mode 100644 dref/migrations/0075_alter_dref_budget_file_preview_alter_dreffile_file_and_more.py create mode 100644 flash_update/migrations/0013_alter_flashgraphicmap_file_and_more.py create mode 100644 per/migrations/0123_alter_perdocumentupload_file_alter_perfile_file.py diff --git a/api/migrations/0215_alter_generaldocument_document_and_more.py b/api/migrations/0215_alter_generaldocument_document_and_more.py new file mode 100644 index 000000000..267750956 --- /dev/null +++ b/api/migrations/0215_alter_generaldocument_document_and_more.py @@ -0,0 +1,30 @@ +# Generated by Django 4.2.16 on 2024-10-29 08:51 + +from django.db import migrations + +import api.models +import main.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("api", "0214_alter_profile_limit_access_to_guest"), + ] + + operations = [ + migrations.AlterField( + model_name="generaldocument", + name="document", + field=main.fields.SecureFileField( + blank=True, null=True, upload_to=api.models.general_document_path, verbose_name="document" + ), + ), + migrations.AlterField( + model_name="situationreport", + name="document", + field=main.fields.SecureFileField( + blank=True, null=True, upload_to=api.models.sitrep_document_path, verbose_name="document" + ), + ), + ] diff --git a/country_plan/migrations/0008_alter_countryplan_internal_plan_file.py b/country_plan/migrations/0008_alter_countryplan_internal_plan_file.py new file mode 100644 index 000000000..d4523aa85 --- /dev/null +++ b/country_plan/migrations/0008_alter_countryplan_internal_plan_file.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.16 on 2024-10-29 08:51 + +import django.core.validators +from django.db import migrations + +import country_plan.models +import main.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("country_plan", "0007_alter_membershipcoordination_sector_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="countryplan", + name="internal_plan_file", + field=main.fields.SecureFileField( + blank=True, + null=True, + upload_to=country_plan.models.pdf_upload_to, + validators=[django.core.validators.FileExtensionValidator(["pdf"])], + verbose_name="Internal Plan", + ), + ), + ] diff --git a/dref/migrations/0075_alter_dref_budget_file_preview_alter_dreffile_file_and_more.py b/dref/migrations/0075_alter_dref_budget_file_preview_alter_dreffile_file_and_more.py new file mode 100644 index 000000000..aa47b34bd --- /dev/null +++ b/dref/migrations/0075_alter_dref_budget_file_preview_alter_dreffile_file_and_more.py @@ -0,0 +1,39 @@ +# Generated by Django 4.2.16 on 2024-10-29 08:51 + +from django.db import migrations + +import main.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("dref", "0074_auto_20240129_0909"), + ] + + operations = [ + migrations.AlterField( + model_name="dref", + name="budget_file_preview", + field=main.fields.SecureFileField( + blank=True, null=True, upload_to="dref/images/", verbose_name="budget file preview" + ), + ), + migrations.AlterField( + model_name="dreffile", + name="file", + field=main.fields.SecureFileField(upload_to="dref/images/", verbose_name="file"), + ), + migrations.AlterField( + model_name="dreffinalreport", + name="financial_report_preview", + field=main.fields.SecureFileField(blank=True, null=True, upload_to="dref/images/", verbose_name="financial preview"), + ), + migrations.AlterField( + model_name="drefoperationalupdate", + name="budget_file_preview", + field=main.fields.SecureFileField( + blank=True, null=True, upload_to="dref-op-update/images/", verbose_name="budget file preview" + ), + ), + ] diff --git a/flash_update/migrations/0013_alter_flashgraphicmap_file_and_more.py b/flash_update/migrations/0013_alter_flashgraphicmap_file_and_more.py new file mode 100644 index 000000000..fa1ec0ab7 --- /dev/null +++ b/flash_update/migrations/0013_alter_flashgraphicmap_file_and_more.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.16 on 2024-10-29 08:51 + +from django.db import migrations + +import main.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("flash_update", "0012_auto_20230410_0720"), + ] + + operations = [ + migrations.AlterField( + model_name="flashgraphicmap", + name="file", + field=main.fields.SecureFileField(upload_to="flash_update/images", verbose_name="file"), + ), + migrations.AlterField( + model_name="flashupdate", + name="extracted_file", + field=main.fields.SecureFileField( + blank=True, null=True, upload_to="flash_update/pdf/", verbose_name="extracted file" + ), + ), + ] diff --git a/per/migrations/0123_alter_perdocumentupload_file_alter_perfile_file.py b/per/migrations/0123_alter_perdocumentupload_file_alter_perfile_file.py new file mode 100644 index 000000000..86a9ab26a --- /dev/null +++ b/per/migrations/0123_alter_perdocumentupload_file_alter_perfile_file.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.16 on 2024-10-29 08:51 + +from django.db import migrations + +import main.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ("per", "0122_opslearningcacheresponse_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="perdocumentupload", + name="file", + field=main.fields.SecureFileField(upload_to="per/documents/", verbose_name="file"), + ), + migrations.AlterField( + model_name="perfile", + name="file", + field=main.fields.SecureFileField(upload_to="per/images/", verbose_name="file"), + ), + ] From 56f5b6e32834578d398a5d2f64ff5e93962f834e Mon Sep 17 00:00:00 2001 From: Szabo Zoltan Date: Mon, 11 Nov 2024 12:40:45 +0100 Subject: [PATCH 5/7] Fix log noise + translation limit error beyond 50000 chars --- Dockerfile | 4 ++++ lang/translation.py | 38 +++++++++++++++++++++++++++++++++++++- main/settings.py | 2 ++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f24e11823..3592fff16 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,6 +39,10 @@ RUN perl -pi -e 's/ is not -1 / != 1 /' ${AZUREROOT}blob/baseblobservice.py RUN perl -pi -e "s/ is '' / == '' /" ${AZUREROOT}common/_connection.py RUN perl -pi -e "s/ is '' / == '' /" ${AZUREROOT}_connection.py +# To avoid dump of "Queue is full. Dropping telemetry." messages in log, 20241111: +ENV OPENCENSUSINIT=/usr/local/lib/python3.11/site-packages/opencensus/common/schedule/__init__.py +RUN perl -pi -e "s/logger.warning.*/pass/" ${OPENCENSUSINIT} 2>/dev/null + COPY main/nginx.conf /etc/nginx/sites-available/ RUN \ ln -s /etc/nginx/sites-available/nginx.conf /etc/nginx/sites-enabled; \ diff --git a/lang/translation.py b/lang/translation.py index ee3f8de8f..dd342caca 100644 --- a/lang/translation.py +++ b/lang/translation.py @@ -78,10 +78,43 @@ def __init__(self): def is_text_html(cls, text): return bool(BeautifulSoup(text, "html.parser").find()) + @classmethod + def find_last_slashtable(cls, text, limit): + tag = "" + truncate_here = text[:limit].rfind(tag) + if truncate_here != -1: + truncate_here += len(tag) + return truncate_here + + @classmethod + def find_last_slashp(cls, text, limit): + tag = "

" + truncate_here = text[:limit].rfind(tag) + if truncate_here != -1: + truncate_here += len(tag) + return truncate_here + def translate_text(self, text, dest_language, source_language=None): if settings.TESTING: # NOTE: Mocking for test purpose return self._fake_translation(text, dest_language, source_language) + + # A dirty workaround to handle oversized HTML+CSS texts, usually tables: + textTail = "" + if len(text) > settings.AZURE_TRANSL_LIMIT: + truncate_here = self.find_last_slashtable(text, settings.AZURE_TRANSL_LIMIT) + if truncate_here != -1: + textTail = text[truncate_here:] + text = text[:truncate_here] + else: + truncate_here = self.find_last_slashp(text, settings.AZURE_TRANSL_LIMIT) + if truncate_here != -1: + textTail = text[truncate_here:] + text = text[:truncate_here] + else: + textTail = text[settings.AZURE_TRANSL_LIMIT :] + text = text[: settings.AZURE_TRANSL_LIMIT] + payload = { "text": text, "from": source_language, @@ -96,7 +129,10 @@ def translate_text(self, text, dest_language, source_language=None): headers=self.headers, json=payload, ) - return response.json()[0]["translations"][0]["text"] + + # Not using == 200 – it would break tests with MagicMock name=requests.post() results + if response.status_code != 500: + return response.json()[0]["translations"][0]["text"] + textTail def get_translator_class(): diff --git a/main/settings.py b/main/settings.py index 61555ab5c..1c7d58690 100644 --- a/main/settings.py +++ b/main/settings.py @@ -702,3 +702,5 @@ def decode_base64(env_key, fallback_env_key): # Need to load this to overwrite modeltranslation module import main.translation # noqa: F401 E402 + +AZURE_TRANSL_LIMIT = 49990 From cd2fb5232e183cba760e1c76460c450852f9a3d2 Mon Sep 17 00:00:00 2001 From: Sushil Tiwari Date: Wed, 13 Nov 2024 16:27:32 +0545 Subject: [PATCH 6/7] Fix icrc ingestion operation scraping logic --- api/management/commands/ingest_icrc.py | 27 +++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/api/management/commands/ingest_icrc.py b/api/management/commands/ingest_icrc.py index 948257cc7..c5a0c2f31 100644 --- a/api/management/commands/ingest_icrc.py +++ b/api/management/commands/ingest_icrc.py @@ -39,7 +39,17 @@ def handle(self, *args, **kwargs): # Get countries information from "Where we work" page regions_list = soup.find("div", {"class": "js-select-country-list"}).find("ul").find_all("ul") - country_list = [] + # Holds the list of countries that are part of the key operations + key_operations_country_list = soup.find("div", {"class": "key-operations-content"}).find_all("div", class_="title") + + # NOTE: Mapping this Country, as it doesnot match with the name in the database + country_name_mapping = {"Syria": "Syrian Arab Republic", "Israel and the occupied territories": "Israel"} + + country_operations_list = [ + country.text.strip() for key_operation in key_operations_country_list for country in key_operation.find_all("a") + ] + + countries = [] for region in regions_list: for country in region.find_all("li"): name = country.text.strip() @@ -47,7 +57,8 @@ def handle(self, *args, **kwargs): country_url = icrc_url + href if href else None presence = bool(country_url) description = None - key_operation = False + # Check if country is part of the key operations + key_operation = name in country_operations_list if country_url: try: @@ -55,13 +66,12 @@ def handle(self, *args, **kwargs): country_page.raise_for_status() country_soup = BeautifulSoup(country_page.content, "html.parser") description_tag = country_soup.find("div", class_="description").find("div", class_="ck-text") - key_operation = bool(description_tag) description = description_tag.text.strip() if description_tag else None except Exception: pass # Append to list - country_list.append( + countries.append( { "Country": name, "ICRC presence": presence, @@ -72,7 +82,11 @@ def handle(self, *args, **kwargs): ) added = 0 - for data in country_list: + created_ns_presence_pk = [] + for data in countries: + # NOTE: mapping the country name + data["Country"] = country_name_mapping.get(data["Country"], data["Country"]) + country = Country.objects.filter(name__exact=data["Country"]).first() if country: country_icrc_presence, _ = CountryICRCPresence.objects.get_or_create(country=country) @@ -81,7 +95,10 @@ def handle(self, *args, **kwargs): country_icrc_presence.key_operation = data["Key operation"] country_icrc_presence.description = data["Description"] country_icrc_presence.save() + created_ns_presence_pk.append(country_icrc_presence.pk) added += 1 + # NOTE: Delete the CountryICRCPresence that are not in the source + CountryICRCPresence.objects.exclude(id__in=created_ns_presence_pk).delete() text_to_log = f"{added} ICRC added" logger.info(text_to_log) From ea68e4e788eb3537e80c4ef0fea2cb1466e6d515 Mon Sep 17 00:00:00 2001 From: Szabo Zoltan Date: Wed, 13 Nov 2024 13:39:26 +0100 Subject: [PATCH 7/7] Admin dark-mode issue fix --- api/templates/admin/base.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/templates/admin/base.html b/api/templates/admin/base.html index e3879d688..9b628cdde 100644 --- a/api/templates/admin/base.html +++ b/api/templates/admin/base.html @@ -4,6 +4,10 @@ {% block title %}{% endblock %} + {% block dark-mode-vars %} + + + {% endblock %} {% if LANGUAGE_BIDI %}{% endif %} {% block extrahead %}{% endblock %} {% block responsive %}