From 404b5aec5a14be9691edf857cd4aebc847bd33a3 Mon Sep 17 00:00:00 2001 From: Daniel Siepmann Date: Tue, 17 Sep 2024 10:46:23 +0200 Subject: [PATCH] Handle destination.one files without file name. It is possible to have files such as `https://dam.destination.one/2675868/3dc0a9dccd0dad46c73e669ece428c634ff8324ea3d01a4858a1c43169deed41/.jpg`. Those were not handled correctly yet. We now also handle those cases. We will generate a hash from the URL as file name in order to still use those files. Relates: #11396 --- .../FilesAssignment.php | 39 ++++++---- Documentation/Changelog/4.2.1.rst | 7 ++ .../ImportHandlesImageWithoutFileName.php | 42 ++++++++++ .../ResponseWithNewImageWithoutFileName.json | 77 +++++++++++++++++++ .../ImportHandlesImagesTest.php | 25 ++++++ 5 files changed, 177 insertions(+), 13 deletions(-) create mode 100644 Tests/Functional/Import/DestinationDataTest/Assertions/ImportHandlesImageWithoutFileName.php create mode 100644 Tests/Functional/Import/DestinationDataTest/Fixtures/ResponseWithNewImageWithoutFileName.json diff --git a/Classes/Service/DestinationDataImportService/FilesAssignment.php b/Classes/Service/DestinationDataImportService/FilesAssignment.php index a61c799..c261fbf 100644 --- a/Classes/Service/DestinationDataImportService/FilesAssignment.php +++ b/Classes/Service/DestinationDataImportService/FilesAssignment.php @@ -29,6 +29,7 @@ use TYPO3\CMS\Core\Log\LogManager; use TYPO3\CMS\Core\Resource\DuplicationBehavior; use TYPO3\CMS\Core\Resource\File; +use TYPO3\CMS\Core\Resource\Folder; use TYPO3\CMS\Core\Resource\Index\MetaDataRepository; use TYPO3\CMS\Core\Resource\ResourceFactory; use TYPO3\CMS\Core\Utility\GeneralUtility; @@ -67,27 +68,26 @@ public function getImages( } $fileUrl = urldecode((string)$mediaObject['url']); - $orgFileNameSanitized = $importFolder->getStorage()->sanitizeFileName(basename($fileUrl)); - - $this->logger->info('File attached.', [$fileUrl, $orgFileNameSanitized]); - - if ($importFolder->hasFile($orgFileNameSanitized)) { - $this->logger->info('File already exists.', [$orgFileNameSanitized]); - } elseif ($filename = $this->loadFile($fileUrl)) { - $this->logger->info('Adding file to FAL.', [$filename]); - $importFolder->addFile($filename, basename($fileUrl), DuplicationBehavior::REPLACE); + $fileName = $this->createFileName($fileUrl, $importFolder); + $this->logger->info('File attached.', [$fileUrl, $fileName]); + + if ($importFolder->hasFile($fileName)) { + $this->logger->info('File already exists.', [$fileName]); + } elseif ($tempFileName = $this->loadFile($fileUrl)) { + $this->logger->info('Adding file to FAL.', [$fileName]); + $importFolder->addFile($tempFileName, $fileName, DuplicationBehavior::REPLACE); } else { continue; } - if ($importFolder->hasFile($orgFileNameSanitized) === false) { - $this->logger->warning('Could not find file.', [$orgFileNameSanitized]); + if ($importFolder->hasFile($fileName) === false) { + $this->logger->warning('Could not find file.', [$fileName]); continue; } - $file = $importFolder->getStorage()->getFileInFolder($orgFileNameSanitized, $importFolder); + $file = $importFolder->getStorage()->getFileInFolder($fileName, $importFolder); if (!$file instanceof File) { - $this->logger->warning('Could not find file.', [$orgFileNameSanitized]); + $this->logger->warning('Could not find file.', [$fileName]); continue; } @@ -98,6 +98,19 @@ public function getImages( return $images; } + private function createFileName(string $url, Folder $importFolder): string + { + $extension = pathinfo($url, PATHINFO_EXTENSION); + $fileName = pathinfo($url, PATHINFO_FILENAME); + if ($fileName !== '' && $extension !== '') { + $fileName = basename($url); + } else { + $fileName = hash('sha256', $url) . '.' . $extension; + } + + return $importFolder->getStorage()->sanitizeFileName($fileName); + } + private function loadFile(string $fileUrl): string { $this->logger->info('Getting file.', [$fileUrl]); diff --git a/Documentation/Changelog/4.2.1.rst b/Documentation/Changelog/4.2.1.rst index 5f9b3ba..9557b91 100644 --- a/Documentation/Changelog/4.2.1.rst +++ b/Documentation/Changelog/4.2.1.rst @@ -28,6 +28,13 @@ Fixes back to `date_default_timezone_get()` call. That way it should be useful for most systems out of the box. +* Handle destination.one files without file name. + + It is possible to have files such as `https://dam.destination.one/2675868/3dc0a9dccd0dad46c73e669ece428c634ff8324ea3d01a4858a1c43169deed41/.jpg`. + Those were not handled correctly yet. + We now also handle those cases. + We will generate a hash from the URL as file name in order to still use those files. + Tasks ----- diff --git a/Tests/Functional/Import/DestinationDataTest/Assertions/ImportHandlesImageWithoutFileName.php b/Tests/Functional/Import/DestinationDataTest/Assertions/ImportHandlesImageWithoutFileName.php new file mode 100644 index 0000000..19d1a2a --- /dev/null +++ b/Tests/Functional/Import/DestinationDataTest/Assertions/ImportHandlesImageWithoutFileName.php @@ -0,0 +1,42 @@ + [ + [ + 'uid' => 1, + 'pid' => 0, + 'missing' => 0, + 'storage' => 1, + 'type' => File::FILETYPE_IMAGE, + 'identifier' => '/staedte/beispielstadt/events/bf126089c94f95031fa07bf9d7d9b10c3e58aafebdef31f0b60604da13019b8d.jpg', + 'extension' => 'jpg', + 'name' => 'bf126089c94f95031fa07bf9d7d9b10c3e58aafebdef31f0b60604da13019b8d.jpg', + ], + ], + 'sys_file_reference' => [ + [ + 'uid' => 1, + 'pid' => 2, + 'uid_local' => 1, + 'uid_foreign' => 1, + 'tablenames' => 'tx_events_domain_model_event', + 'fieldname' => 'images', + 'sorting_foreign' => 1, + 'title' => null, + 'description' => null, + 'alternative' => null, + ], + ], + 'sys_file_metadata' => [ + [ + 'uid' => 1, + 'pid' => 0, + 'file' => 1, + 'title' => null, + ], + ], +]; diff --git a/Tests/Functional/Import/DestinationDataTest/Fixtures/ResponseWithNewImageWithoutFileName.json b/Tests/Functional/Import/DestinationDataTest/Fixtures/ResponseWithNewImageWithoutFileName.json new file mode 100644 index 0000000..f058d13 --- /dev/null +++ b/Tests/Functional/Import/DestinationDataTest/Fixtures/ResponseWithNewImageWithoutFileName.json @@ -0,0 +1,77 @@ +{ + "status": "OK", + "count": 3, + "overallcount": 50, + "channels": [], + "facetGroups": [], + "items": [ + { + "global_id": "e_100347853", + "id": "100347853", + "title": "Allerlei Weihnachtliches (Heute mit Johannes Geißer)", + "type": "Event", + "categories": [ + "Weihnachten" + ], + "texts": [ + { + "rel": "details", + "type": "text/html", + "value": "Die Lichter sind entzündet, die Plätzchen duften, man rückt endlich wieder näher zusammen und lauscht den Geschichten. Vier Schauspieler*innen unseres Theaters überraschen mit ihren weihnachtlichen Texten, die sie für uns ausgewählt haben. Dazu plaudern sie über persönliche Anekdoten und erinnern sich an ihre schönsten und verrücktesten Weihnachtsfeste. Und da der Genuss in der Vorweihnachtszeit nicht fehlen darf, wird an jedem Adventssonntag eine andere weihnachtliche Spezialität serviert.
Eintritt: 10 € (inkl. Gedeck mit weihnachtlicher Schillerlocke)
Um Voranmeldung unter 03672-486470 oder schillerhaus@rudolstadt.de wird gebeten.
Es gilt die 2G-PLUS-Regel. 
" + }, + { + "rel": "details", + "type": "text/plain", + "value": "Die Lichter sind entzündet, die Plätzchen duften, man rückt endlich wieder näher zusammen und lauscht den Geschichten. Vier Schauspieler*innen unseres Theaters überraschen mit ihren weihnachtlichen Texten, die sie für uns ausgewählt haben. Dazu plaudern sie über persönliche Anekdoten und erinnern sich an ihre schönsten und verrücktesten Weihnachtsfeste. Und da der Genuss in der Vorweihnachtszeit nicht fehlen darf, wird an jedem Adventssonntag eine andere weihnachtliche Spezialität serviert.\nEintritt: 10 € (inkl. Gedeck mit weihnachtlicher Schillerlocke)\nUm Voranmeldung unter 03672-486470 oder schillerhaus@rudolstadt.de wird gebeten.\nEs gilt die 2G-PLUS-Regel." + }, + { + "rel": "teaser", + "type": "text/html" + }, + { + "rel": "teaser", + "type": "text/plain" + } + ], + "areas": [ + "Rudolstadt und Umgebung" + ], + "city": "Rudolstadt", + "zip": "07407", + "street": "Schillerstraße 25", + "phone": "+ 49 3672 / 486470", + "fax": "+ 49 3672 / 486475", + "web": "http://www.schillerhaus.rudolstadt.de/", + "email": "schillerhaus@rudolstadt.de", + "author": "support@hubermedia.de", + "media_objects": [ + { + "rel": "default", + "url": "https://dam.destination.one/849917/279ac45b3fc701a7197131f627164fffd9f8cc77bc75165e2fc2b864ed606920/.jpg", + "type": "image/jpeg", + "latitude": null, + "longitude": null, + "width": 1920, + "height": 1080, + "value": "" + } + ], + "timeIntervals": [ + { + "weekdays": [], + "start": "2022-12-19T15:00:00+01:00", + "end": "2022-12-19T16:30:00+01:00", + "tz": "Europe/Berlin", + "interval": 1 + } + ], + "name": "Schillerhaus Rudolstadt", + "created": "2022-10-31T12:29:00+00:00", + "changed": "2022-12-14T08:29:00+00:00", + "source": { + "url": "http://destination.one/", + "value": "destination.one" + } + } + ] +} diff --git a/Tests/Functional/Import/DestinationDataTest/ImportHandlesImagesTest.php b/Tests/Functional/Import/DestinationDataTest/ImportHandlesImagesTest.php index 44c54c6..aa0875c 100644 --- a/Tests/Functional/Import/DestinationDataTest/ImportHandlesImagesTest.php +++ b/Tests/Functional/Import/DestinationDataTest/ImportHandlesImagesTest.php @@ -60,6 +60,31 @@ public function addsNewImages(): void $this->assertEmptyLog(); } + #[Test] + public function addsNewImageWithoutFileName(): void + { + $this->setUpResponses([ + new Response(200, [], file_get_contents(__DIR__ . '/Fixtures/ResponseWithNewImageWithoutFileName.json') ?: ''), + new Response(200, [], file_get_contents(__DIR__ . '/Fixtures/ExampleImage.jpg') ?: ''), + ]); + + $this->executeCommand(); + + $this->assertPHPDataSet(__DIR__ . '/Assertions/ImportHandlesImageWithoutFileName.php'); + + $importedFiles = GeneralUtility::getFilesInDir($this->fileImportPath); + self::assertIsArray($importedFiles, 'Failed to retrieve imported files from filesystem.'); + self::assertSame( + [ + 'bf126089c94f95031fa07bf9d7d9b10c3e58aafebdef31f0b60604da13019b8d.jpg', + ], + array_values($importedFiles), + 'Got unexpected number of files' + ); + + $this->assertEmptyLog(); + } + #[Test] public function addsMultipleImagesToSingleEvent(): void {