Coverage for src/sync_photos.py: 100%
291 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-10-26 21:28 +0000
1"""Sync photos module.
3This module provides the main photo synchronization functionality,
4orchestrating the downloading of photos from iCloud to local storage.
5"""
7___author___ = "Mandar Patil <mandarons@pm.me>"
9import os
11import requests
12from icloudpy import exceptions
14from src import config_parser, configure_icloudpy_logging, get_logger, recycle_bin
15from src.album_sync_orchestrator import sync_album_photos
16from src.hardlink_registry import create_hardlink_registry
17from src.photo_cleanup_utils import remove_obsolete_files
19# Configure icloudpy logging immediately after import
20configure_icloudpy_logging()
22LOGGER = get_logger()
25# Legacy functions preserved for backward compatibility with existing tests
26# These functions are now implemented using the new modular architecture
29def get_max_threads(config):
30 """Get maximum number of threads for parallel downloads.
32 Legacy function - now delegates to config_parser.
34 Args:
35 config: Configuration dictionary
37 Returns:
38 Maximum number of threads to use for downloads
39 """
40 return config_parser.get_app_max_threads(config)
43def get_name_and_extension(photo, file_size):
44 """Extract filename and extension.
46 Legacy function - now delegates to photo_path_utils.
48 Args:
49 photo: Photo object from iCloudPy
50 file_size: File size variant
52 Returns:
53 Tuple of (name, extension)
54 """
55 from src.photo_path_utils import get_photo_name_and_extension
57 return get_photo_name_and_extension(photo, file_size)
60def photo_wanted(photo, extensions):
61 """Check if photo is wanted based on extension.
63 Legacy function - now delegates to photo_filter_utils.
65 Args:
66 photo: Photo object from iCloudPy
67 extensions: List of allowed extensions
69 Returns:
70 True if photo should be synced, False otherwise
71 """
72 from src.photo_filter_utils import is_photo_wanted
74 return is_photo_wanted(photo, extensions)
77def generate_file_name(photo, file_size, destination_path, folder_format):
78 """Generate full path to file.
80 Legacy function - now delegates to photo_download_manager.
82 Args:
83 photo: Photo object from iCloudPy
84 file_size: File size variant
85 destination_path: Base destination path
86 folder_format: Folder format string
88 Returns:
89 Full file path
90 """
91 from src.photo_download_manager import generate_photo_path
93 return generate_photo_path(photo, file_size, destination_path, folder_format)
96def photo_exists(photo, file_size, local_path):
97 """Check if photo exist locally.
99 Legacy function - now delegates to photo_file_utils.
101 Args:
102 photo: Photo object from iCloudPy
103 file_size: File size variant
104 local_path: Local file path to check
106 Returns:
107 True if photo exists with correct size, False otherwise
108 """
109 from src.photo_file_utils import check_photo_exists
111 return check_photo_exists(photo, file_size, local_path)
114def create_hardlink(source_path, destination_path):
115 """Create a hard link from source to destination.
117 Legacy function - now delegates to photo_file_utils.
119 Args:
120 source_path: Path to source file
121 destination_path: Path for new hardlink
123 Returns:
124 True if successful, False otherwise
125 """
126 from src.photo_file_utils import create_hardlink as create_hardlink_impl
128 return create_hardlink_impl(source_path, destination_path)
131def download_photo(photo, file_size, destination_path):
132 """Download photo from server.
134 Legacy function - now delegates to photo_file_utils.
136 Args:
137 photo: Photo object from iCloudPy
138 file_size: File size variant
139 destination_path: Where to save the photo
141 Returns:
142 True if successful, False otherwise
143 """
144 from src.photo_file_utils import download_photo_from_server
146 return download_photo_from_server(photo, file_size, destination_path)
149def process_photo(photo, file_size, destination_path, files, folder_format, hardlink_registry=None):
150 """Process photo details (legacy function for backward compatibility).
152 Args:
153 photo: Photo object from iCloudPy
154 file_size: File size variant
155 destination_path: Base destination path
156 files: Set to track downloaded files
157 folder_format: Folder format string
158 hardlink_registry: Registry for hardlinks (legacy dict format)
160 Returns:
161 True if photo was processed successfully, False otherwise
162 """
163 from src.photo_download_manager import collect_download_task, execute_download_task
165 # Convert legacy hardlink registry dict to new registry format if needed
166 converted_registry = None
167 if hardlink_registry is not None:
168 from src.hardlink_registry import HardlinkRegistry
170 converted_registry = HardlinkRegistry()
171 for key, path in hardlink_registry.items():
172 # Legacy format: photo_id_file_size -> path
173 if "_" in key:
174 parts = key.rsplit("_", 1)
175 if len(parts) == 2:
176 photo_id, file_sz = parts
177 converted_registry.register_photo_path(photo_id, file_sz, path)
179 # Collect download task
180 task_info = collect_download_task(
181 photo,
182 file_size,
183 destination_path,
184 files,
185 folder_format,
186 converted_registry,
187 )
189 if task_info is None:
190 return False
192 # Execute task
193 result = execute_download_task(task_info)
195 # Update legacy registry if provided
196 if result and hardlink_registry is not None:
197 photo_key = f"{photo.id}_{file_size}"
198 hardlink_registry[photo_key] = task_info.photo_path
200 return result
203def collect_photo_for_download(photo, file_size, destination_path, files, folder_format, hardlink_registry=None):
204 """Collect photo info for parallel download without immediately downloading.
206 Legacy function - now delegates to photo_download_manager.
208 Args:
209 photo: Photo object from iCloudPy
210 file_size: File size variant
211 destination_path: Base destination path
212 files: Set to track downloaded files
213 folder_format: Folder format string
214 hardlink_registry: Registry for hardlinks (legacy dict format)
216 Returns:
217 Download task info or None
218 """
219 from src.photo_download_manager import collect_download_task
221 # Convert legacy hardlink registry dict to new registry format if needed
222 converted_registry = None
223 if hardlink_registry is not None:
224 from src.hardlink_registry import HardlinkRegistry
226 converted_registry = HardlinkRegistry()
227 for key, path in hardlink_registry.items():
228 if "_" in key:
229 parts = key.rsplit("_", 1)
230 if len(parts) == 2:
231 photo_id, file_sz = parts
232 converted_registry.register_photo_path(photo_id, file_sz, path)
234 task_info = collect_download_task(
235 photo,
236 file_size,
237 destination_path,
238 files,
239 folder_format,
240 converted_registry,
241 )
243 if task_info is None:
244 return None
246 # Convert back to legacy format for compatibility
247 return {
248 "photo": task_info.photo,
249 "file_size": task_info.file_size,
250 "photo_path": task_info.photo_path,
251 "hardlink_source": task_info.hardlink_source,
252 "hardlink_registry": hardlink_registry,
253 }
256def download_photo_task(download_info):
257 """Download a single photo or create hardlink as part of parallel execution.
259 Legacy function - maintains original implementation for backward compatibility.
261 Args:
262 download_info: Dictionary with download task information
264 Returns:
265 True if successful, False otherwise
266 """
267 photo = download_info["photo"]
268 file_size = download_info["file_size"]
269 photo_path = download_info["photo_path"]
270 hardlink_source = download_info.get("hardlink_source")
271 hardlink_registry = download_info.get("hardlink_registry")
273 LOGGER.debug(f"[Thread] Starting processing of {photo_path}")
275 try:
276 # Try hardlink first if source exists
277 if hardlink_source:
278 if create_hardlink(hardlink_source, photo_path):
279 LOGGER.debug(f"[Thread] Created hardlink for {photo_path}")
280 return True
281 else:
282 # Fallback to download if hard link creation fails
283 LOGGER.warning(f"Hard link creation failed, downloading {photo_path} instead")
285 # Download the photo - this maintains the original function call for test compatibility
286 result = download_photo(photo, file_size, photo_path)
287 if result:
288 # Register for future hard links if enabled
289 if hardlink_registry is not None:
290 photo_key = f"{photo.id}_{file_size}"
291 hardlink_registry[photo_key] = photo_path
292 LOGGER.debug(f"[Thread] Completed download of {photo_path}")
293 return result
294 except Exception as e:
295 LOGGER.error(f"[Thread] Failed to process {photo_path}: {e!s}")
296 return False
299def sync_album(
300 album,
301 destination_path,
302 file_sizes,
303 extensions=None,
304 files=None,
305 folder_format=None,
306 hardlink_registry=None,
307 config=None,
308):
309 """Sync given album.
311 Legacy function - now delegates to album_sync_orchestrator with conversion
312 for legacy hardlink registry format.
314 Args:
315 album: Album object from iCloudPy
316 destination_path: Path where photos should be saved
317 file_sizes: List of file size variants to download
318 extensions: List of allowed file extensions
319 files: Set to track downloaded files
320 folder_format: Folder format string
321 hardlink_registry: Registry for hardlinks (legacy dict format)
322 config: Configuration dictionary
324 Returns:
325 True on success, None on invalid input
326 """
327 # Convert legacy hardlink registry dict to new registry format if needed
328 converted_registry = None
329 if hardlink_registry is not None:
330 from src.hardlink_registry import HardlinkRegistry
332 converted_registry = HardlinkRegistry()
333 for key, path in hardlink_registry.items():
334 if "_" in key:
335 parts = key.rsplit("_", 1)
336 if len(parts) == 2:
337 photo_id, file_sz = parts
338 converted_registry.register_photo_path(photo_id, file_sz, path)
340 result = sync_album_photos(
341 album=album,
342 destination_path=destination_path,
343 file_sizes=file_sizes,
344 extensions=extensions,
345 files=files,
346 folder_format=folder_format,
347 hardlink_registry=converted_registry,
348 config=config,
349 )
351 # Update legacy registry if provided and new registry was created
352 if hardlink_registry is not None and converted_registry is not None:
353 # This is a simplified approach - in practice, we'd need to track new entries
354 # But for legacy compatibility, we'll maintain the existing behavior
355 pass
357 return result
360def remove_obsolete(destination_path, files):
361 """Remove local obsolete file.
363 Legacy function - now delegates to photo_cleanup_utils.
365 Args:
366 destination_path: Path to search for obsolete files
367 files: Set of files that should be kept
369 Returns:
370 Set of removed file paths
371 """
372 return remove_obsolete_files(destination_path, files)
375def sync_photos(config, photos):
376 """Sync all photos.
378 Main orchestration function that coordinates the entire photo sync process.
379 This function has been refactored to use the new modular architecture while
380 maintaining backward compatibility.
382 Args:
383 config: Configuration dictionary
384 photos: Photos object from iCloudPy
386 Returns:
387 Tuple of (total_successful, total_failed) download counts
388 """
389 # Parse configuration using centralized config parser
390 destination_path = config_parser.prepare_photos_destination(config=config)
391 library_destinations = config_parser.get_photos_library_destinations(config=config)
392 # Apply the requested filename convention for this sync run. The module-level
393 # default in photo_path_utils flows down through generate_photo_path -> all
394 # downloads without requiring a per-call signature change.
395 from src.photo_path_utils import set_default_filename_format, set_file_format
396 set_default_filename_format(config_parser.get_photos_filename_format(config=config))
397 set_file_format(
398 config_parser.get_photos_file_format(config=config),
399 config_parser.get_photos_variant_separator(config=config),
400 )
401 filters = config_parser.get_photos_filters(config=config)
402 file_sizes = filters.get("file_sizes") or []
403 # preserve_originals_as_bak invariant: at least ONE visible file per
404 # photo must always land on disk. The suffix only hides ``original``
405 # behind ``.bak`` -- so if the edit (``full``) is not ALSO being
406 # downloaded, enabling the suffix would leave nothing visible. Gate the
407 # toggle on both ``preserve_originals_as_bak: true`` AND ``full`` in
408 # ``file_sizes``. ``original_alt`` is the RAW/JPEG pair alternate, not an
409 # edit, and no longer activates the toggle. If a photo is later removed
410 # from iCloud, the normal remove_obsolete sweep cascades to both the
411 # visible file AND the ``.bak`` sidecar (neither is in tracked_files, both
412 # get deleted) -- so there's no risk of orphan ``.bak`` files outliving
413 # their parent.
414 from src.photo_download_manager import set_fallback_to_original
415 from src.photo_path_utils import set_adjusted_suffix, set_preserve_originals_as_bak
417 set_preserve_originals_as_bak(
418 config_parser.get_photos_preserve_originals_as_bak(config=config)
419 and "full" in file_sizes,
420 )
421 # With both the original and the edit requested, a plain filename can
422 # resolve the two versions to one path; mark the edit so one cannot
423 # overwrite the other.
424 set_adjusted_suffix("original" in file_sizes and "full" in file_sizes)
425 # When ``original`` is itself requested it is downloaded on its own, so a
426 # fallback task for a missing size would target the same path and race that
427 # download. Only substitute when the original is not already in the list.
428 set_fallback_to_original(
429 bool(filters.get("fallback_to_original")) and "original" not in file_sizes,
430 )
431 files = set()
432 download_all = config_parser.get_photos_all_albums(config=config)
433 use_hardlinks = config_parser.get_photos_use_hardlinks(config=config)
434 libraries = filters["libraries"] if filters["libraries"] is not None else photos.libraries
435 folder_format = config_parser.get_photos_folder_format(config=config)
437 # Initialize hard link registry using new modular approach
438 hardlink_registry = create_hardlink_registry(use_hardlinks)
440 total_successful, total_failed = 0, 0
441 # Libraries we could not read. They must be barred from the cleanup
442 # below: a library that failed contributes nothing to ``files``, so
443 # cleaning its destination would read as "the server has none of these"
444 # and delete every local copy.
445 failed_libraries: set = set()
446 libraries = _listed_libraries(photos, libraries, failed_libraries)
447 # Libraries Apple is still indexing. They are synced -- the listing is
448 # usually complete long before the index says so -- but, like a failed
449 # library, kept out of cleanup: an incomplete listing would read as
450 # deletions.
451 indexing_libraries = _libraries_still_indexing(photos, libraries)
453 # Special handling for "All Photos" when hardlinks are enabled
454 if use_hardlinks and download_all:
455 sub_successful, sub_failed = _sync_all_photos_first_for_hardlinks(
456 photos,
457 libraries,
458 destination_path,
459 filters,
460 files,
461 folder_format,
462 hardlink_registry,
463 config,
464 library_destinations=library_destinations,
465 failed_libraries=failed_libraries,
466 )
467 total_successful += sub_successful
468 total_failed += sub_failed
470 # Sync albums based on configuration
471 sub_successful, sub_failed = _sync_albums_by_configuration(
472 photos,
473 libraries,
474 download_all,
475 destination_path,
476 filters,
477 files,
478 folder_format,
479 hardlink_registry,
480 config,
481 library_destinations=library_destinations,
482 failed_libraries=failed_libraries,
483 )
484 total_successful += sub_successful
485 total_failed += sub_failed
487 # Clean up obsolete files if enabled. When per-library destinations are
488 # configured we walk each library's subdir independently, otherwise the
489 # legacy single-destination walk preserves backward compatibility.
490 if config_parser.get_photos_remove_obsolete(config=config):
491 marker_filename = config_parser.get_mount_marker_filename(config=config)
492 exclude = {marker_filename}
493 limit = config_parser.get_photos_obsolete_delete_limit_percent(config=config)
494 bin_ = recycle_bin.from_config(config, "photos")
495 if library_destinations:
496 for library in libraries:
497 if library in failed_libraries:
498 LOGGER.warning(
499 f"Skipping obsolete-file cleanup for {library}: it could not "
500 f"be read this run, so its local files cannot be verified.",
501 )
502 continue
503 if library in indexing_libraries:
504 LOGGER.warning(
505 f"Skipping obsolete-file cleanup for {library}: Apple is still "
506 f"indexing it, or its index could not be checked, so its "
507 f"listing may be incomplete.",
508 )
509 continue
510 lib_dest = _library_destination(destination_path, library, library_destinations)
511 if lib_dest == destination_path:
512 # An unmapped library falls through to the shared root,
513 # which holds every other library's tree -- and anything
514 # else the user keeps there. Cleaning it on behalf of one
515 # library would delete all of them. Apple invents
516 # libraries (backend migration zones), so this is reached
517 # by doing nothing wrong.
518 LOGGER.warning(
519 f"Skipping obsolete-file cleanup for {library}: it has no "
520 f"library_destinations entry, so its destination is the "
521 f"shared root and cleaning it would affect other libraries.",
522 )
523 continue
524 remove_obsolete_files(
525 lib_dest,
526 files,
527 exclude_filenames=exclude,
528 limit_percent=limit,
529 recycle_bin=bin_ and bin_.within(_bin_subfolder(lib_dest, destination_path, library)),
530 )
531 elif failed_libraries:
532 # One shared destination: ``files`` cannot say which library a
533 # path came from, so a single failure makes the whole set
534 # untrustworthy for deletion.
535 LOGGER.warning(
536 "Skipping obsolete-file cleanup: "
537 f"{len(failed_libraries)} library(ies) could not be read this run.",
538 )
539 elif indexing_libraries:
540 # Same reasoning: one shared destination cannot be cleaned while
541 # any library's listing may be incomplete.
542 LOGGER.warning(
543 "Skipping obsolete-file cleanup: Apple is still indexing (or could "
544 f"not be checked for) {', '.join(sorted(indexing_libraries))}, so the "
545 "listing may be incomplete.",
546 )
547 else:
548 remove_obsolete_files(
549 destination_path,
550 files,
551 exclude_filenames=exclude,
552 limit_percent=limit,
553 recycle_bin=bin_,
554 )
556 return total_successful, total_failed
559def _bin_subfolder(lib_dest: str, destination_path: str, library: str) -> str:
560 """Where a library's files go inside the shared Photos bin.
562 The library's folder under the Photos destination, as it appears on
563 disk, so a restore is the same relative move. A mapping that points
564 outside the destination has no such folder, so the library's own name
565 stands in for it.
566 """
567 relative = os.path.relpath(lib_dest, destination_path)
568 if relative == os.pardir or relative.startswith(os.pardir + os.sep):
569 return library
570 return relative
573def _library_destination(base_destination: str, library: str, library_destinations: dict) -> str:
574 """Resolve the on-disk destination for a given iCloud photo library.
576 When ``library_destinations`` provides a mapping for ``library``, joins
577 the configured subdirectory under ``base_destination`` and ensures the
578 directory exists. Otherwise returns ``base_destination`` unchanged
579 (preserving mandarons' legacy single-destination behaviour).
581 Library-name matching has three rules, in priority order:
583 1. **Exact match.** ``library_destinations[library]`` if present.
584 2. **Role alias for `SharedLibrary`.** Apple's modern iCloud Shared
585 Photo Library is exposed by icloudpy under a GUID-based zone name
586 like ``SharedSync-<guid>``. A config
587 key of ``SharedLibrary`` matches any zone whose name starts with
588 ``SharedSync-`` so users don't need to discover and hardcode the
589 per-account GUID. (Configs that already use the literal current
590 Apple zone name still work via rule 1.)
591 3. **Fallthrough.** Returns ``base_destination`` unchanged.
592 """
593 if not library_destinations:
594 return base_destination
595 # ``get`` returns None only when the key is absent -- the config parser
596 # coerces every configured value to str, so an explicit "" stays "".
597 # Key on ``is None`` (not falsiness) so an explicitly-mapped library
598 # always wins over the SharedLibrary alias and the default, honouring
599 # the rule 1 > rule 2 > rule 3 priority documented above.
600 subdir = library_destinations.get(library)
601 if subdir is None and library.startswith("SharedSync-"):
602 subdir = library_destinations.get("SharedLibrary")
603 if subdir is None:
604 return base_destination
605 dest = os.path.join(base_destination, subdir)
606 os.makedirs(dest, exist_ok=True)
607 return dest
610# An account can be shown libraries it never created -- zones left behind
611# by Apple's own backend migrations -- and some of them answer every query
612# with ZONE_NOT_FOUND or BAD_REQUEST. One of those must not be able to
613# abort the sync for the libraries that work.
614_LIBRARY_FAULTS = (exceptions.ICloudPyAPIResponseException, requests.exceptions.RequestException)
617def _signal_library(action: str, library: str, **kwargs) -> None:
618 """Publish per-library progress for the dashboard, best effort.
620 A library can take an hour, so "which one is running right now" is
621 only knowable while it runs. Wrapped because the web-signal module is
622 optional and a dashboard write must never affect the sync.
623 """
624 try:
625 from src import web_signals
627 getattr(web_signals, action)(library, **kwargs)
628 except Exception as e: # noqa: BLE001 -- reporting must not break syncing
629 LOGGER.debug(f"web_signals: {action} for {library} raised: {e!s}")
632def _libraries_still_indexing(photos, libraries) -> set:
633 """Names of the libraries to sync that Apple has not finished indexing.
635 icloudpy opens such a library when asked to and records how far Apple
636 got as ``indexing_state``. Only an explicit ``FINISHED`` counts as
637 finished: a missing or empty state is treated as still indexing,
638 since a partial listing would read as deletions.
640 Fails closed: if the libraries cannot be listed here, every one is
641 returned, because a later attempt in the same cycle can succeed and
642 would otherwise let cleanup run on a library whose state was never
643 seen. A configured library that is not listed is returned for the
644 same reason.
645 """
646 try:
647 available = photos.libraries
648 except _LIBRARY_FAULTS as e:
649 LOGGER.warning(
650 f"Could not check whether Apple has finished indexing the photo "
651 f"libraries ({e!s}); obsolete-file cleanup is skipped this cycle.",
652 )
653 return set(libraries)
654 indexing = set()
655 for name in libraries:
656 if name not in available:
657 indexing.add(name)
658 continue
659 # Anything but an explicit FINISHED, a missing or empty state
660 # included, keeps the library out of cleanup.
661 state = getattr(available[name], "indexing_state", None)
662 still_indexing = state != "FINISHED"
663 _signal_library("record_library_indexing", name, state=str(state) if still_indexing else None)
664 if still_indexing:
665 LOGGER.warning(
666 f"Apple is still indexing {name} ({state}). Downloading what it "
667 f"lists; obsolete-file cleanup waits until the index finishes.",
668 )
669 indexing.add(name)
670 return indexing
673def _listed_libraries(photos, libraries, failed_libraries) -> list:
674 """The configured libraries Apple lists; the others are failed, not fatal.
676 ``filters.libraries`` can name a library Apple no longer lists, such as
677 a Shared Library that was left. Looking one up raises a KeyError outside
678 the per-library isolation, which stopped Photos for every library. Each
679 one dropped here is barred from cleanup like any failed library.
681 If the libraries cannot be listed, all are kept, and each fails on its
682 own when it is synced.
683 """
684 try:
685 available = photos.libraries
686 except _LIBRARY_FAULTS:
687 return list(libraries)
688 listed = []
689 for library in libraries:
690 if library in available:
691 listed.append(library)
692 continue
693 _note_library_failure(library, "Apple does not list this library", failed_libraries)
694 _signal_library("record_library_finished", library, ok=False, error="Apple does not list this library")
695 return listed
698def _note_library_failure(library, error, failed_libraries) -> None:
699 """Record a library we could not read, and say so in the log.
701 Never silent: the count is reported as failures by the caller, and the
702 library is barred from obsolete-file cleanup so an unreadable library
703 can never be mistaken for an empty one.
704 """
705 LOGGER.error(f"Library {library} could not be synced, skipping it: {error!s}")
706 if failed_libraries is not None:
707 failed_libraries.add(library)
710def _sync_all_photos_first_for_hardlinks(
711 photos,
712 libraries,
713 destination_path,
714 filters,
715 files,
716 folder_format,
717 hardlink_registry,
718 config,
719 library_destinations: dict | None = None,
720 failed_libraries: set | None = None,
721) -> tuple[int, int]:
722 """Sync 'All Photos' album first to populate hardlink registry.
724 Args:
725 photos: Photos object from iCloudPy
726 libraries: List of photo libraries to sync
727 destination_path: Base destination path
728 filters: Photo filters configuration
729 files: Set to track downloaded files
730 folder_format: Folder format string
731 hardlink_registry: Registry for tracking downloaded files
732 config: Configuration dictionary
734 Returns:
735 Tuple of (total_successful, total_failed) download counts
736 """
737 for library in libraries:
738 try:
739 if library == "PrimarySync" and "All Photos" in photos.libraries[library].albums:
740 LOGGER.info("Syncing 'All Photos' album first for hard link reference...")
741 lib_dest = _library_destination(destination_path, library, library_destinations or {})
742 result = sync_album_photos(
743 album=photos.libraries[library].albums["All Photos"],
744 destination_path=os.path.join(lib_dest, "All Photos"),
745 file_sizes=filters["file_sizes"],
746 extensions=filters["extensions"],
747 files=files,
748 folder_format=folder_format,
749 hardlink_registry=hardlink_registry,
750 config=config,
751 )
752 if hardlink_registry:
753 LOGGER.info(
754 f"'All Photos' sync complete. Hard link registry populated with "
755 f"{hardlink_registry.get_registry_size()} reference files.",
756 )
757 if result is not None:
758 return result
759 break
760 except _LIBRARY_FAULTS as e:
761 _note_library_failure(library, e, failed_libraries)
762 return 0, 0
765def _sync_albums_by_configuration(
766 photos,
767 libraries,
768 download_all,
769 destination_path,
770 filters,
771 files,
772 folder_format,
773 hardlink_registry,
774 config,
775 library_destinations: dict | None = None,
776 failed_libraries: set | None = None,
777) -> tuple[int, int]:
778 """Sync albums based on configuration settings.
780 Args:
781 photos: Photos object from iCloudPy
782 libraries: List of photo libraries to sync
783 download_all: Whether to download all albums
784 destination_path: Base destination path
785 filters: Photo filters configuration
786 files: Set to track downloaded files
787 folder_format: Folder format string
788 hardlink_registry: Registry for tracking downloaded files
789 config: Configuration dictionary
791 Returns:
792 Tuple of (total_successful, total_failed) aggregated across all libraries
793 """
794 total_successful, total_failed = 0, 0
795 for library in libraries:
796 try:
797 _signal_library("record_library_started", library)
798 lib_dest = _library_destination(destination_path, library, library_destinations or {})
799 if download_all and library == "PrimarySync":
800 sub_successful, sub_failed = _sync_all_albums_except_filtered(
801 photos,
802 library,
803 filters,
804 lib_dest,
805 files,
806 folder_format,
807 hardlink_registry,
808 config,
809 )
810 elif filters["albums"] and library == "PrimarySync":
811 sub_successful, sub_failed = _sync_filtered_albums(
812 photos,
813 library,
814 filters,
815 lib_dest,
816 files,
817 folder_format,
818 hardlink_registry,
819 config,
820 )
821 elif filters["albums"]:
822 sub_successful, sub_failed = _sync_filtered_albums_in_library(
823 photos,
824 library,
825 filters,
826 lib_dest,
827 files,
828 folder_format,
829 hardlink_registry,
830 config,
831 )
832 else:
833 sub_successful, sub_failed = _sync_all_photos_in_library(
834 photos,
835 library,
836 lib_dest,
837 filters,
838 files,
839 folder_format,
840 hardlink_registry,
841 config,
842 )
843 total_successful += sub_successful
844 total_failed += sub_failed
845 _signal_library(
846 "record_library_finished",
847 library,
848 ok=True,
849 downloaded=sub_successful,
850 skipped=sub_failed,
851 )
852 # Per-iteration by design: isolating one library from the next is
853 # the whole point, so the handler cannot be hoisted out of the loop.
854 except _LIBRARY_FAULTS as e: # noqa: PERF203
855 _note_library_failure(library, e, failed_libraries)
856 _signal_library("record_library_finished", library, ok=False, error=str(e))
857 return total_successful, total_failed
860def _sync_all_albums_except_filtered(
861 photos,
862 library,
863 filters,
864 destination_path,
865 files,
866 folder_format,
867 hardlink_registry,
868 config,
869) -> tuple[int, int]:
870 """Sync all albums except those in the filter exclusion list.
872 Args:
873 photos: Photos object from iCloudPy
874 library: Library name to sync
875 filters: Photo filters configuration
876 destination_path: Base destination path
877 files: Set to track downloaded files
878 folder_format: Folder format string
879 hardlink_registry: Registry for tracking downloaded files
880 config: Configuration dictionary
882 Returns:
883 Tuple of (total_successful, total_failed) aggregated across all synced albums
884 """
885 total_successful, total_failed = 0, 0
886 for album in photos.libraries[library].albums.keys():
887 # Skip All Photos if we already synced it first
888 if hardlink_registry and album == "All Photos":
889 continue
890 if filters["albums"] and album in iter(filters["albums"]):
891 continue
892 result = sync_album_photos(
893 album=photos.libraries[library].albums[album],
894 destination_path=os.path.join(destination_path, album),
895 file_sizes=filters["file_sizes"],
896 extensions=filters["extensions"],
897 files=files,
898 folder_format=folder_format,
899 hardlink_registry=hardlink_registry,
900 config=config,
901 )
902 if result is not None:
903 sub_successful, sub_failed = result
904 total_successful += sub_successful
905 total_failed += sub_failed
906 return total_successful, total_failed
909def _sync_filtered_albums(
910 photos,
911 library,
912 filters,
913 destination_path,
914 files,
915 folder_format,
916 hardlink_registry,
917 config,
918) -> tuple[int, int]:
919 """Sync only albums specified in filters.
921 Args:
922 photos: Photos object from iCloudPy
923 library: Library name to sync
924 filters: Photo filters configuration
925 destination_path: Base destination path
926 files: Set to track downloaded files
927 folder_format: Folder format string
928 hardlink_registry: Registry for tracking downloaded files
929 config: Configuration dictionary
931 Returns:
932 Tuple of (total_successful, total_failed) aggregated across all synced albums
933 """
934 total_successful, total_failed = 0, 0
935 for album in iter(filters["albums"]):
936 result = sync_album_photos(
937 album=photos.libraries[library].albums[album],
938 destination_path=os.path.join(destination_path, album),
939 file_sizes=filters["file_sizes"],
940 extensions=filters["extensions"],
941 files=files,
942 folder_format=folder_format,
943 hardlink_registry=hardlink_registry,
944 config=config,
945 )
946 if result is not None:
947 sub_successful, sub_failed = result
948 total_successful += sub_successful
949 total_failed += sub_failed
950 return total_successful, total_failed
953def _sync_filtered_albums_in_library(
954 photos,
955 library,
956 filters,
957 destination_path,
958 files,
959 folder_format,
960 hardlink_registry,
961 config,
962) -> tuple[int, int]:
963 """Sync filtered albums in a specific library.
965 Args:
966 photos: Photos object from iCloudPy
967 library: Library name to sync
968 filters: Photo filters configuration
969 destination_path: Base destination path
970 files: Set to track downloaded files
971 folder_format: Folder format string
972 hardlink_registry: Registry for tracking downloaded files
973 config: Configuration dictionary
975 Returns:
976 Tuple of (total_successful, total_failed) aggregated across all synced albums
977 """
978 total_successful, total_failed = 0, 0
979 for album in iter(filters["albums"]):
980 if album in photos.libraries[library].albums:
981 result = sync_album_photos(
982 album=photos.libraries[library].albums[album],
983 destination_path=os.path.join(destination_path, album),
984 file_sizes=filters["file_sizes"],
985 extensions=filters["extensions"],
986 files=files,
987 folder_format=folder_format,
988 hardlink_registry=hardlink_registry,
989 config=config,
990 )
991 if result is not None:
992 sub_successful, sub_failed = result
993 total_successful += sub_successful
994 total_failed += sub_failed
995 else:
996 LOGGER.warning(f"Album {album} not found in {library}. Skipping the album {album} ...")
997 return total_successful, total_failed
1000def _sync_all_photos_in_library(
1001 photos,
1002 library,
1003 destination_path,
1004 filters,
1005 files,
1006 folder_format,
1007 hardlink_registry,
1008 config,
1009) -> tuple[int, int]:
1010 """Sync all photos in a library.
1012 Args:
1013 photos: Photos object from iCloudPy
1014 library: Library name to sync
1015 destination_path: Base destination path
1016 filters: Photo filters configuration
1017 files: Set to track downloaded files
1018 folder_format: Folder format string
1019 hardlink_registry: Registry for tracking downloaded files
1020 config: Configuration dictionary
1022 Returns:
1023 Tuple of (total_successful, total_failed) download counts
1024 """
1025 result = sync_album_photos(
1026 album=photos.libraries[library].all,
1027 destination_path=os.path.join(destination_path, "all"),
1028 file_sizes=filters["file_sizes"],
1029 extensions=filters["extensions"],
1030 files=files,
1031 folder_format=folder_format,
1032 hardlink_registry=hardlink_registry,
1033 config=config,
1034 )
1035 if result is not None:
1036 return result
1037 return 0, 0