Coverage for src/photo_file_utils.py: 100%
154 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-10-26 21:28 +0000
1"""Photo file operations module.
3This module contains utilities for photo file operations including
4downloading, hardlink creation, and file existence checking.
5"""
7___author___ = "Mandar Patil <mandarons@pm.me>"
9import contextlib
10import json
11import os
12import shutil
13import threading
14from datetime import UTC
15from urllib.parse import urlencode
17from src import DEFAULT_REQUEST_TIMEOUT_SEC, get_logger
19LOGGER = get_logger()
21# Module-level lock to protect thread-safe mutation of photo._versions during retries
22_versions_refresh_lock = threading.Lock()
24# Consecutive download-URL refresh failures, and how often to escalate them to WARNING.
25# Downloads run in parallel, so the counter is guarded by its own lock.
26_refresh_failure_lock = threading.Lock()
27_consecutive_refresh_failures = 0
28_REFRESH_FAILURE_WARN_INTERVAL = 3
30# CloudKit fields to request when re-fetching a photo record for fresh download URLs.
31# Mirrors the desiredKeys list used by icloudpy's PhotoAlbum._list_query_gen().
32_DESIRED_KEYS = [
33 "resJPEGFullWidth",
34 "resJPEGFullHeight",
35 "resJPEGFullFileType",
36 "resJPEGFullFingerprint",
37 "resJPEGFullRes",
38 "resJPEGLargeWidth",
39 "resJPEGLargeHeight",
40 "resJPEGLargeFileType",
41 "resJPEGLargeFingerprint",
42 "resJPEGLargeRes",
43 "resJPEGMedWidth",
44 "resJPEGMedHeight",
45 "resJPEGMedFileType",
46 "resJPEGMedFingerprint",
47 "resJPEGMedRes",
48 "resJPEGThumbWidth",
49 "resJPEGThumbHeight",
50 "resJPEGThumbFileType",
51 "resJPEGThumbFingerprint",
52 "resJPEGThumbRes",
53 "resVidFullWidth",
54 "resVidFullHeight",
55 "resVidFullFileType",
56 "resVidFullFingerprint",
57 "resVidFullRes",
58 "resVidMedWidth",
59 "resVidMedHeight",
60 "resVidMedFileType",
61 "resVidMedFingerprint",
62 "resVidMedRes",
63 "resVidSmallWidth",
64 "resVidSmallHeight",
65 "resVidSmallFileType",
66 "resVidSmallFingerprint",
67 "resVidSmallRes",
68 "resSidecarWidth",
69 "resSidecarHeight",
70 "resSidecarFileType",
71 "resSidecarFingerprint",
72 "resSidecarRes",
73 "itemType",
74 "dataClassType",
75 "filenameEnc",
76 "originalOrientation",
77 "resOriginalWidth",
78 "resOriginalHeight",
79 "resOriginalFileType",
80 "resOriginalFingerprint",
81 "resOriginalRes",
82 "resOriginalAltWidth",
83 "resOriginalAltHeight",
84 "resOriginalAltFileType",
85 "resOriginalAltFingerprint",
86 "resOriginalAltRes",
87 "resOriginalVidComplWidth",
88 "resOriginalVidComplHeight",
89 "resOriginalVidComplFileType",
90 "resOriginalVidComplFingerprint",
91 "resOriginalVidComplRes",
92 "isDeleted",
93 "isExpunged",
94 "dateExpunged",
95 "remappedRef",
96 "recordName",
97 "recordType",
98 "recordChangeTag",
99 "masterRef",
100 "adjustmentRenderType",
101 "assetDate",
102 "addedDate",
103 "isFavorite",
104 "isHidden",
105 "orientation",
106 "duration",
107 "assetSubtype",
108 "assetSubtypeV2",
109 "assetHDRType",
110 "burstFlags",
111 "burstFlagsExt",
112 "burstId",
113 "captionEnc",
114 "extendedDescEnc",
115 "locationEnc",
116 "locationV2Enc",
117 "locationLatitude",
118 "locationLongitude",
119 "adjustmentType",
120 "timeZoneOffset",
121 "vidComplDurValue",
122 "vidComplDurScale",
123 "vidComplDispValue",
124 "vidComplDispScale",
125 "vidComplVisibilityState",
126 "customRenderedValue",
127 "containerId",
128 "itemId",
129 "position",
130 "isKeyAsset",
131 "importedByBundleIdentifierEnc",
132 "importedByDisplayNameEnc",
133 "importedBy",
134]
137def _note_refresh_success() -> None:
138 """Reset the consecutive refresh-failure streak after a successful refresh."""
139 global _consecutive_refresh_failures # noqa: PLW0603
140 with _refresh_failure_lock:
141 _consecutive_refresh_failures = 0
144def _note_refresh_failure(record_name: str, reason: str) -> None:
145 """Record a failed URL refresh, escalating to WARNING once failures repeat.
147 URL refresh is the last line of defence before a download is abandoned, so a
148 systematically broken refresh path (rather than an occasional miss) should be
149 visible without turning on debug logging. Individual failures stay at DEBUG;
150 every ``_REFRESH_FAILURE_WARN_INTERVAL`` consecutive failures emits a WARNING.
152 Args:
153 record_name: CloudKit recordName of the photo being refreshed
154 reason: Human-readable description of why the refresh failed
155 """
156 global _consecutive_refresh_failures # noqa: PLW0603
157 with _refresh_failure_lock:
158 _consecutive_refresh_failures += 1
159 failures = _consecutive_refresh_failures
161 if failures % _REFRESH_FAILURE_WARN_INTERVAL == 0:
162 LOGGER.warning(
163 f"Download URL refresh has failed {failures} times in a row - expired-URL (HTTP 410) "
164 f"recovery is not working, so affected photos will be reported as failed downloads. "
165 f"Most recent failure: {record_name} - {reason}",
166 )
167 else:
168 LOGGER.debug(f"Failed to refresh download URL for {record_name}: {reason}")
171def _refresh_photo_download_url(photo) -> bool:
172 """Re-fetch the photo's CloudKit records from iCloud for fresh download URLs.
174 iCloud download URLs are signed CDN tokens that expire after ~30–40 minutes.
175 When a URL expires (HTTP 410 Gone), clearing ``photo._versions`` alone is
176 insufficient because icloudpy re-parses the same stale records which still
177 contain the expired URL. This function makes a ``records/lookup`` API call
178 for the records that carry renditions -- the ``CPLMaster`` record and, for
179 an edited asset, the ``CPLAsset`` record whose ``resJPEGFull*`` fields hold
180 the edit's URL -- then updates them in place and clears ``_versions`` so
181 the next ``download()`` call uses the fresh URLs.
183 The two records have different ``recordName``s (the asset record links to
184 the master through ``masterRef``), so looking up only the master leaves the
185 ``full`` URL stale and an edited photo never recovers from a 410.
187 ``records/lookup`` is used rather than ``records/query`` because ``CPLMaster``
188 is not a query-indexable CloudKit type: querying it fails every time with
189 ``Type is not marked indexable: CPLMaster (BAD_REQUEST)``. Lookup fetches
190 records by name and returns the same ``{"records": [...]}`` shape.
192 Args:
193 photo: PhotoAsset object from icloudpy
195 Returns:
196 True if every record that carries a rendition was refreshed, False
197 otherwise.
198 """
199 try:
200 record_name = photo._master_record["recordName"] # noqa: SLF001
201 # The record's own zone. icloudpy gives every asset the photos
202 # service, whose zone is always the primary library, so a Shared
203 # Library asset looked up there is never found.
204 record_zone = photo._master_record.get("zoneID") # noqa: SLF001
205 except (AttributeError, KeyError, TypeError):
206 _note_refresh_failure("<unknown>", "photo missing _master_record or recordName")
207 return False
209 # The edited rendition lives on the asset (``CPLAsset``) record, which has
210 # its own ``recordName`` -- CloudKit links it to the master through
211 # ``masterRef``, and the two names differ. Refreshing only the master
212 # leaves a stale ``full`` URL, so the retry hits 410 again and the photo is
213 # reported as a failed download.
214 asset_record = getattr(photo, "_asset_record", None)
215 asset_name = asset_record.get("recordName") if isinstance(asset_record, dict) else None
216 asset_fields = (asset_record.get("fields") or {}) if isinstance(asset_record, dict) else {}
217 # Only ask for the asset record when it carries a rendition: an unedited
218 # asset's record holds dates and metadata, not download URLs.
219 needs_asset = bool(
220 asset_name
221 and asset_name != record_name
222 and any(key.endswith("Res") for key in asset_fields),
223 )
224 record_names = [record_name]
225 if needs_asset:
226 record_names.append(asset_name)
228 service = getattr(photo, "_service", None)
229 if service is None:
230 _note_refresh_failure(record_name, "photo missing _service")
231 return False
233 endpoint = getattr(service, "_service_endpoint", None)
234 session = getattr(service, "session", None)
235 params = getattr(service, "params", None)
236 zone_id = record_zone if isinstance(record_zone, dict) and record_zone else getattr(service, "zone_id", None)
238 if not all([endpoint, session, params, zone_id]):
239 _note_refresh_failure(record_name, "photo._service missing required attributes")
240 return False
242 try:
243 url = f"{endpoint}/records/lookup?{urlencode(params)}"
244 query = {
245 "records": [{"recordName": name} for name in record_names],
246 "desiredKeys": _DESIRED_KEYS,
247 "zoneID": zone_id,
248 }
249 request = session.post(
250 url,
251 data=json.dumps(query),
252 headers={"Content-type": "text/plain"},
253 )
254 response = request.json()
255 records = response.get("records", [])
257 # A failed lookup is not an HTTP error: CloudKit returns it as a
258 # record inside ``records`` carrying the requested recordName and a
259 # ``serverErrorCode``/``reason`` in place of ``fields``. Matching on
260 # the name alone therefore accepts an error payload as a fresh
261 # record -- the refresh reports success, and the retrying download
262 # raises KeyError('fields') deep inside icloudpy, which surfaces as an
263 # unexplained "Failed to download <path>: 'fields'" and is charged to
264 # the download rather than to the refresh.
265 #
266 # An empty ``fields`` is no better than a missing one: icloudpy builds
267 # ``versions`` by testing ``f"{prefix}Res" in fields``, so an empty
268 # dict yields no versions at all, ``download()`` returns None, and the
269 # caller dereferences ``.raw`` on it. Accepting either shape also
270 # discards the still-valid record we already hold, poisoning the asset
271 # for the rest of the sync.
272 #
273 # Failures are collected per record so a master that refreshed is not
274 # thrown away because the asset lookup failed for an unedited photo --
275 # and, conversely, a stale ``full`` URL never reports success.
276 refreshed = {}
277 failures = {}
278 for rec in records:
279 name = rec.get("recordName")
280 if name not in record_names:
281 continue
282 if rec.get("serverErrorCode"):
283 failures[name] = (
284 f"CloudKit error {rec['serverErrorCode']}: "
285 f"{rec.get('reason') or 'no reason given'}"
286 )
287 continue
288 if not rec.get("fields"):
289 failures[name] = "record came back with no usable fields"
290 continue
291 refreshed[name] = rec
293 if record_name not in refreshed:
294 _note_refresh_failure(
295 record_name,
296 failures.get(record_name, "record not found in iCloud response"),
297 )
298 return False
299 if needs_asset and asset_name is not None and asset_name not in refreshed:
300 _note_refresh_failure(
301 asset_name,
302 failures.get(asset_name, "record not found in iCloud response"),
303 )
304 return False
306 photo._master_record = refreshed[record_name] # noqa: SLF001
307 if asset_name in refreshed:
308 photo._asset_record = refreshed[asset_name] # noqa: SLF001
309 with _versions_refresh_lock:
310 photo._versions = None # noqa: SLF001
311 LOGGER.debug(f"Refreshed download URL for {record_name}")
312 _note_refresh_success()
313 return True
315 except Exception as e: # noqa: BLE001
316 _note_refresh_failure(record_name, str(e))
317 return False
320def check_photo_exists(photo, file_size: str, local_path: str) -> bool:
321 """Check if photo exists locally with correct size.
323 Args:
324 photo: Photo object from iCloudPy
325 file_size: File size variant (original, medium, thumb, etc.)
326 local_path: Local file path to check
328 Returns:
329 True if photo exists locally with correct size, False otherwise
330 """
331 if not (photo and local_path and os.path.isfile(local_path)):
332 return False
334 local_size = os.path.getsize(local_path)
335 remote_size = int(photo.versions[file_size]["size"])
337 if local_size == remote_size:
338 LOGGER.debug(f"No changes detected. Skipping the file {local_path} ...")
339 return True
340 else:
341 LOGGER.debug(f"Change detected: local_file_size is {local_size} and remote_file_size is {remote_size}.")
342 return False
345def create_hardlink(source_path: str, destination_path: str) -> bool:
346 """Create a hard link from source to destination.
348 Args:
349 source_path: Path to existing file to link from
350 destination_path: Path where hardlink should be created
352 Returns:
353 True if hardlink was created successfully, False otherwise
354 """
355 try:
356 # Ensure destination directory exists
357 os.makedirs(os.path.dirname(destination_path), exist_ok=True)
358 # Create hard link
359 os.link(source_path, destination_path)
360 LOGGER.info(f"Created hard link: {destination_path} (linked to existing file: {source_path})")
361 return True
362 except (OSError, FileNotFoundError) as e:
363 LOGGER.warning(f"Failed to create hard link {destination_path}: {e!s}")
364 return False
367def download_photo_from_server(
368 photo,
369 file_size: str,
370 destination_path: str,
371 max_retries: int = 1,
372 timeout: int = DEFAULT_REQUEST_TIMEOUT_SEC,
373) -> bool:
374 """Download photo from iCloud server to local path.
376 This function implements automatic retry logic for HTTP 410 (Gone) errors,
377 which occur when iCloud download URLs expire. When a 410 error is detected,
378 the function re-fetches the photo's master record from iCloud to obtain
379 fresh download URLs, then retries the download.
381 Args:
382 photo: Photo object from iCloudPy
383 file_size: File size variant (original, medium, thumb, etc.)
384 destination_path: Local path where photo should be saved
385 max_retries: Maximum number of retries on 410 errors (default: 1)
386 timeout: HTTP read timeout in seconds. Without one a stalled CDN
387 connection blocks the worker thread forever.
389 Returns:
390 True if download was successful, False otherwise
391 """
392 if not (photo and file_size and destination_path):
393 return False
395 LOGGER.info(f"Downloading {destination_path} ...")
397 max_retries = max(0, max_retries) # Clamp to minimum 0 for predictable behavior
398 attempt = 0
399 max_attempts = max_retries + 1 # Initial attempt + retries
401 # Download beside the target and move it into place only once it is
402 # complete. Writing straight to destination_path left a failed transfer's
403 # partial file where the real photo belongs, where anything reading the
404 # library (an Immich external library, a backup) takes it for the photo,
405 # and it also destroyed the previous good copy before the new one existed.
406 # A partial stranded by a killed process is untracked, so obsolete-file
407 # cleanup removes it.
408 partial = f"{destination_path}.part"
410 while attempt < max_attempts: # noqa: PERF203
411 try:
412 download = photo.download(file_size, timeout=timeout)
413 with open(partial, "wb") as file_out:
414 shutil.copyfileobj(download.raw, file_out)
416 # iCloud declares every version's size, and photo_exists already
417 # treats a mismatch as "not this file". Check it here too, so a
418 # body that is short or is not the photo at all (an error page)
419 # fails now instead of being recorded as downloaded.
420 expected = _declared_size(photo, file_size)
421 written = os.path.getsize(partial)
422 if expected is not None and written != expected:
423 _discard(partial)
424 LOGGER.error(
425 f"Incomplete download of {destination_path}: got {written} of {expected} bytes. "
426 "Will retry on the next sync.",
427 )
428 return False
429 os.replace(partial, destination_path)
431 # Set file modification time to photo's added date.
432 # iCloudPy returns added_date as an aware UTC datetime; replace() is a
433 # safe no-op here because tzinfo is already UTC. If it ever returns a
434 # naive datetime, replace(tzinfo=utc) correctly treats it as UTC.
435 local_modified_time = photo.added_date.replace(tzinfo=UTC).timestamp()
436 os.utime(destination_path, (local_modified_time, local_modified_time))
438 return True
440 except Exception as e: # noqa: PERF203
441 _discard(partial)
442 # Enhanced error logging with file path context
443 # This catches all exceptions including iCloudPy errors like ObjectNotFoundException
444 error_msg = str(e)
446 # Check for HTTP 410 Gone error - download URL has expired
447 # The iCloudPy library raises exceptions with "Gone (410)" in the message
448 # when the download URL has expired (typically after 30-40 minutes)
449 if "Gone (410)" in error_msg:
450 attempt += 1
451 if attempt < max_attempts:
452 LOGGER.warning(
453 f"Download URL expired (410) for {destination_path}. "
454 f"Refreshing URL and retrying (attempt {attempt}/{max_attempts})...",
455 )
456 # Re-fetch the master record from iCloud to obtain fresh download URLs.
457 # Simply clearing _versions is insufficient because icloudpy re-parses
458 # the same stale _master_record which still contains the expired URL.
459 _refresh_photo_download_url(photo)
460 continue
461 else:
462 LOGGER.error(
463 f"Failed to download {destination_path} after {max_retries} retries: {error_msg}",
464 )
465 return False
467 # Handle other errors
468 if "ObjectNotFoundException" in error_msg or "NOT_FOUND" in error_msg:
469 LOGGER.error(f"Photo not found in iCloud Photos - {destination_path}: {error_msg}")
470 else:
471 LOGGER.error(f"Failed to download {destination_path}: {error_msg}")
472 return False
474 # This line should never be reached due to the logic above, but is kept as defensive programming
475 return False # pragma: no cover
478def _declared_size(photo, file_size: str) -> int | None:
479 """The size iCloud declares for this version, or None if it does not say."""
480 try:
481 return int(photo.versions[file_size]["size"])
482 except (AttributeError, KeyError, TypeError, ValueError):
483 return None
486def _discard(path: str) -> None:
487 """Remove a partial download if one was written."""
488 with contextlib.suppress(FileNotFoundError):
489 os.remove(path)
492def rename_legacy_file_if_exists(old_path: str, new_path: str) -> None:
493 """Rename legacy file format to new format if it exists.
495 Args:
496 old_path: Path to legacy file format
497 new_path: Path to new file format
498 """
499 if os.path.isfile(old_path):
500 os.rename(old_path, new_path)