From e8ab26a0f6151026437922a56704e9fda27b146b Mon Sep 17 00:00:00 2001 From: Jakub Tymejczyk Date: Tue, 29 Aug 2023 17:35:06 +0200 Subject: [PATCH 1/4] this fixes error with unpacking packages sometimes first layer of package did have different normalization that item.name and script couldn't open it also normalize content of archive after unpacking, same issue as in pull request #144 --- src/sync_drive.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/sync_drive.py b/src/sync_drive.py index 1c4243ff5..d6f1020e7 100644 --- a/src/sync_drive.py +++ b/src/sync_drive.py @@ -224,7 +224,18 @@ def process_file(item, destination_path, filters, ignore, files): download_file(item=item, local_file=local_file) if item_is_package: for f in Path(local_file).glob("**/*"): - files.add(str(f)) + f = str(f) + f_normalized = unicodedata.normalize("NFC", f) + try: + os.rename(f, f_normalized) + except Exception as e: + LOGGER.warning("Normalizing failed - " + str(e)) + f_dir = os.path.dirname(f) + # delete empty folder if any after normalization + with os.scandir(f_dir) as it: + if not any(it): + os.rmdir(f_dir) + files.add(f_normalized) return True From 0b5b8a4b3fec785d937b0ec110f2742e77af1307 Mon Sep 17 00:00:00 2001 From: Jakub Tymejczyk Date: Tue, 29 Aug 2023 17:48:08 +0200 Subject: [PATCH 2/4] add option to preserve album structure --- README.md | 7 +++++-- src/config_parser.py | 10 ++++++++++ src/sync_photos.py | 14 +++++++++++++- tests/test_config_parser.py | 18 ++++++++++++++++++ 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8f2a8174c..3fdebfd61 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,10 @@ photos: destination: "photos" remove_obsolete: false sync_interval: 500 - filters: # Optional, use it only if you want to download specific albums. Else, all photos are downloaded to `all` folder. + all_albums: false # Optional, default false. If true preserve album structure. If same photo is in multpile albums creates duplicates on filesystem + filters: + # if all_albums is false list of albums to download, if all_albums is true list of ignored albums + # if empty and all_albums is false download all photos to "all" folder. if empty and all_albums is true download all folders albums: - "album 1" - "album2" @@ -120,7 +123,7 @@ photos: - "original" # - "medium" # - "thumb" - extensions: #Optional, media extensions to be included in syncing iCloud Photos content + extensions: # Optional, media extensions to be included in syncing iCloud Photos content # - jpg # - heic # - png diff --git a/src/config_parser.py b/src/config_parser.py index ef89d4160..58a58f707 100644 --- a/src/config_parser.py +++ b/src/config_parser.py @@ -96,6 +96,16 @@ def get_photos_sync_interval(config): return sync_interval +def get_photos_all_albums(config): + """Return flag to download all albums from config.""" + download_all = False + config_path = ["photos", "all_albums"] + if traverse_config_path(config=config, config_path=config_path): + download_all = get_config_value(config=config, config_path=config_path) + LOGGER.info("Syncing all albums.") + return download_all + + def prepare_root_destination(config): """Prepare root destination.""" LOGGER.debug("Checking root destination ...") diff --git a/src/sync_photos.py b/src/sync_photos.py index d89aca5c3..ff723df05 100644 --- a/src/sync_photos.py +++ b/src/sync_photos.py @@ -144,7 +144,19 @@ def sync_photos(config, photos): destination_path = config_parser.prepare_photos_destination(config=config) filters = config_parser.get_photos_filters(config=config) files = set() - if filters["albums"]: + download_all = config_parser.get_photos_all_albums(config=config) + if download_all: + for album in photos.albums.keys(): + if album in iter(filters["albums"]): + continue + sync_album( + album=photos.albums[album], + destination_path=os.path.join(destination_path, album), + file_sizes=filters["file_sizes"], + extensions=filters["extensions"], + files=files, + ) + elif filters["albums"]: for album in iter(filters["albums"]): sync_album( album=photos.albums[album], diff --git a/tests/test_config_parser.py b/tests/test_config_parser.py index e1580d775..402bcc79f 100644 --- a/tests/test_config_parser.py +++ b/tests/test_config_parser.py @@ -399,3 +399,21 @@ def test_get_region_valid(self): config["app"]["region"] = "china" actual = config_parser.get_region(config=config) self.assertEqual(actual, "china") + + def test_get_all_albums_empty(self): + """Empty all_albums.""" + config = read_config(config_path=tests.CONFIG_PATH) + self.assertFalse(config_parser.get_photos_all_albums(config=config)) + + def test_get_all_albums_true(self): + """True all_albums.""" + config = read_config(config_path=tests.CONFIG_PATH) + config["photos"]["all_albums"] = True + self.assertTrue(config_parser.get_photos_all_albums(config=config)) + + def test_get_all_albums_false(self): + """False all_albums.""" + config = read_config(config_path=tests.CONFIG_PATH) + config["photos"]["all_albums"] = False + self.assertFalse(config_parser.get_photos_all_albums(config=config)) + From 0ebeb295dcf1c24bd48d3514a912aedf52104fba Mon Sep 17 00:00:00 2001 From: Mandar Patil Date: Wed, 4 Oct 2023 22:22:36 -0700 Subject: [PATCH 3/4] added test coverage --- src/sync_photos.py | 2 -- tests/data/__init__.py | 58 +++++++++++++++++++++++++++++++++++++++ tests/test_sync_photos.py | 28 +++++++++++++++++++ 3 files changed, 86 insertions(+), 2 deletions(-) diff --git a/src/sync_photos.py b/src/sync_photos.py index e12ec37b8..fbb864e75 100644 --- a/src/sync_photos.py +++ b/src/sync_photos.py @@ -147,8 +147,6 @@ def sync_photos(config, photos): download_all = config_parser.get_photos_all_albums(config=config) if download_all: for album in photos.albums.keys(): - if album in iter(filters["albums"]): - continue sync_album( album=photos.albums[album], destination_path=os.path.join(destination_path, album), diff --git a/tests/data/__init__.py b/tests/data/__init__.py index 9eebf723e..efac0ba0f 100644 --- a/tests/data/__init__.py +++ b/tests/data/__init__.py @@ -3834,6 +3834,64 @@ def request(self, method, url, **kwargs): "query?remapEnums=True&getCurrentSyncToken=True" ][0]["response"] ) + if ( + data.get("query").get("recordType") + == "CPLAssetAndMasterHiddenByAssetDate" + ): + return ResponseMock( + photos_data.DATA[ + "query?remapEnums=True&getCurrentSyncToken=True" + ][5]["response"] + ) + if ( + data.get("query").get("recordType") + == "CPLAssetAndMasterDeletedByExpungedDate" + ): + return ResponseMock( + photos_data.DATA[ + "query?remapEnums=True&getCurrentSyncToken=True" + ][5]["response"] + ) + if ( + data.get("query").get("recordType") + == "CPLBurstStackAssetAndMasterByAssetDate" + ): + return ResponseMock( + photos_data.DATA[ + "query?remapEnums=True&getCurrentSyncToken=True" + ][5]["response"] + ) + + if data.get("query").get("recordType") in ( + "CPLAssetAndMasterInSmartAlbumByAssetDate" + ): + if "filterBy" in data["query"] and data.get("query").get( + "filterBy" + )[2]["fieldValue"]["value"] in ( + "TIMELAPSE", + "LIVE", + "VIDEO", + "SLOMO", + "FAVORITE", + "SCREENSHOT", + "BURST", + "PANORAMA", + ): + return ResponseMock( + photos_data.DATA[ + "query?remapEnums=True&getCurrentSyncToken=True" + ][5]["response"] + ) + if ( + "filterBy" in data["query"] + and data.get("query").get("filterBy")[2]["fieldValue"]["value"] + == "VIDEO" + ): + return ResponseMock( + photos_data.DATA[ + "query?remapEnums=True&getCurrentSyncToken=True" + ][5]["response"] + ) if data.get("query").get("recordType") == "CPLAlbumByPositionLive": if ( "filterBy" in data["query"] diff --git a/tests/test_sync_photos.py b/tests/test_sync_photos.py index 15d74d781..d1821d887 100644 --- a/tests/test_sync_photos.py +++ b/tests/test_sync_photos.py @@ -50,6 +50,34 @@ def test_sync_photos_original( self.assertIsNone( sync_photos.sync_photos(config=config, photos=mock_service.photos) ) + album_2_path = os.path.join(self.destination_path, "album 2") + album_1_1_path = os.path.join(album_2_path, "album-1-1") + album_1_path = os.path.join(self.destination_path, "album-1") + self.assertTrue(os.path.isdir(album_2_path)) + self.assertTrue(os.path.isdir(album_1_path)) + self.assertTrue(os.path.isdir(album_1_1_path)) + self.assertTrue(len(os.listdir(album_2_path)) > 1) + self.assertTrue(len(os.listdir(album_1_path)) > 0) + + @patch(target="keyring.get_password", return_value=data.VALID_PASSWORD) + @patch( + target="src.config_parser.get_username", return_value=data.AUTHENTICATED_USER + ) + @patch("icloudpy.ICloudPyService") + @patch("src.read_config") + def test_sync_photos_all_albums( + self, mock_read_config, mock_service, mock_get_username, mock_get_password + ): + """Test for successful original photo size download.""" + mock_service = self.service + config = self.config.copy() + config["photos"]["destination"] = self.destination_path + config["photos"]["all_albums"] = True + mock_read_config.return_value = config + # Sync original photos + self.assertIsNone( + sync_photos.sync_photos(config=config, photos=mock_service.photos) + ) album_0_path = os.path.join( self.destination_path, config["photos"]["filters"]["albums"][0] ) From 3785c9095825bd85329dcbf6f188a10eed72bf95 Mon Sep 17 00:00:00 2001 From: Mandar Patil Date: Wed, 4 Oct 2023 22:25:33 -0700 Subject: [PATCH 4/4] added test coverage --- config.yaml | 3 +++ tests/data/test_config.yaml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/config.yaml b/config.yaml index 83697549a..3406030dd 100644 --- a/config.yaml +++ b/config.yaml @@ -45,7 +45,10 @@ photos: destination: "photos" remove_obsolete: false sync_interval: 500 + all_albums: false # Optional, default false. If true preserve album structure. If same photo is in multpile albums creates duplicates on filesystem filters: + # if all_albums is false list of albums to download, if all_albums is true list of ignored albums + # if empty and all_albums is false download all photos to "all" folder. if empty and all_albums is true download all folders albums: - "album 1" - "album2" diff --git a/tests/data/test_config.yaml b/tests/data/test_config.yaml index f29e3d3e7..bcc0e717d 100644 --- a/tests/data/test_config.yaml +++ b/tests/data/test_config.yaml @@ -53,7 +53,10 @@ photos: destination: photos remove_obsolete: false sync_interval: -1 + all_albums: false # Optional, default false. If true preserve album structure. If same photo is in multpile albums creates duplicates on filesystem filters: + # if all_albums is false list of albums to download, if all_albums is true list of ignored albums + # if empty and all_albums is false download all photos to "all" folder. if empty and all_albums is true download all folders albums: - "album 2" - album-1