Coverage for src/photo_file_utils.py: 100%
137 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-08 21:05 +0000
1"""Photo file operations module.
3This module contains utilities for photo file operations including
4downloading, hardlink creation, and file existence checking.
5"""
7___author___ = "Mandar Patil <mandarons@pm.me>"
9import contextlib
10import json
11import os
12import shutil
13import threading
14from datetime import timezone
15from urllib.parse import urlencode
17from src import DEFAULT_REQUEST_TIMEOUT_SEC, get_logger
19LOGGER = get_logger()
21# Module-level lock to protect thread-safe mutation of photo._versions during retries
22_versions_refresh_lock = threading.Lock()
24# Consecutive download-URL refresh failures, and how often to escalate them to WARNING.
25# Downloads run in parallel, so the counter is guarded by its own lock.
26_refresh_failure_lock = threading.Lock()
27_consecutive_refresh_failures = 0
28_REFRESH_FAILURE_WARN_INTERVAL = 3
30# CloudKit fields to request when re-fetching a photo record for fresh download URLs.
31# Mirrors the desiredKeys list used by icloudpy's PhotoAlbum._list_query_gen().
32_DESIRED_KEYS = [
33 "resJPEGFullWidth",
34 "resJPEGFullHeight",
35 "resJPEGFullFileType",
36 "resJPEGFullFingerprint",
37 "resJPEGFullRes",
38 "resJPEGLargeWidth",
39 "resJPEGLargeHeight",
40 "resJPEGLargeFileType",
41 "resJPEGLargeFingerprint",
42 "resJPEGLargeRes",
43 "resJPEGMedWidth",
44 "resJPEGMedHeight",
45 "resJPEGMedFileType",
46 "resJPEGMedFingerprint",
47 "resJPEGMedRes",
48 "resJPEGThumbWidth",
49 "resJPEGThumbHeight",
50 "resJPEGThumbFileType",
51 "resJPEGThumbFingerprint",
52 "resJPEGThumbRes",
53 "resVidFullWidth",
54 "resVidFullHeight",
55 "resVidFullFileType",
56 "resVidFullFingerprint",
57 "resVidFullRes",
58 "resVidMedWidth",
59 "resVidMedHeight",
60 "resVidMedFileType",
61 "resVidMedFingerprint",
62 "resVidMedRes",
63 "resVidSmallWidth",
64 "resVidSmallHeight",
65 "resVidSmallFileType",
66 "resVidSmallFingerprint",
67 "resVidSmallRes",
68 "resSidecarWidth",
69 "resSidecarHeight",
70 "resSidecarFileType",
71 "resSidecarFingerprint",
72 "resSidecarRes",
73 "itemType",
74 "dataClassType",
75 "filenameEnc",
76 "originalOrientation",
77 "resOriginalWidth",
78 "resOriginalHeight",
79 "resOriginalFileType",
80 "resOriginalFingerprint",
81 "resOriginalRes",
82 "resOriginalAltWidth",
83 "resOriginalAltHeight",
84 "resOriginalAltFileType",
85 "resOriginalAltFingerprint",
86 "resOriginalAltRes",
87 "resOriginalVidComplWidth",
88 "resOriginalVidComplHeight",
89 "resOriginalVidComplFileType",
90 "resOriginalVidComplFingerprint",
91 "resOriginalVidComplRes",
92 "isDeleted",
93 "isExpunged",
94 "dateExpunged",
95 "remappedRef",
96 "recordName",
97 "recordType",
98 "recordChangeTag",
99 "masterRef",
100 "adjustmentRenderType",
101 "assetDate",
102 "addedDate",
103 "isFavorite",
104 "isHidden",
105 "orientation",
106 "duration",
107 "assetSubtype",
108 "assetSubtypeV2",
109 "assetHDRType",
110 "burstFlags",
111 "burstFlagsExt",
112 "burstId",
113 "captionEnc",
114 "extendedDescEnc",
115 "locationEnc",
116 "locationV2Enc",
117 "locationLatitude",
118 "locationLongitude",
119 "adjustmentType",
120 "timeZoneOffset",
121 "vidComplDurValue",
122 "vidComplDurScale",
123 "vidComplDispValue",
124 "vidComplDispScale",
125 "vidComplVisibilityState",
126 "customRenderedValue",
127 "containerId",
128 "itemId",
129 "position",
130 "isKeyAsset",
131 "importedByBundleIdentifierEnc",
132 "importedByDisplayNameEnc",
133 "importedBy",
134]
137def _note_refresh_success() -> None:
138 """Reset the consecutive refresh-failure streak after a successful refresh."""
139 global _consecutive_refresh_failures # noqa: PLW0603
140 with _refresh_failure_lock:
141 _consecutive_refresh_failures = 0
144def _note_refresh_failure(record_name: str, reason: str) -> None:
145 """Record a failed URL refresh, escalating to WARNING once failures repeat.
147 URL refresh is the last line of defence before a download is abandoned, so a
148 systematically broken refresh path (rather than an occasional miss) should be
149 visible without turning on debug logging. Individual failures stay at DEBUG;
150 every ``_REFRESH_FAILURE_WARN_INTERVAL`` consecutive failures emits a WARNING.
152 Args:
153 record_name: CloudKit recordName of the photo being refreshed
154 reason: Human-readable description of why the refresh failed
155 """
156 global _consecutive_refresh_failures # noqa: PLW0603
157 with _refresh_failure_lock:
158 _consecutive_refresh_failures += 1
159 failures = _consecutive_refresh_failures
161 if failures % _REFRESH_FAILURE_WARN_INTERVAL == 0:
162 LOGGER.warning(
163 f"Download URL refresh has failed {failures} times in a row - expired-URL (HTTP 410) "
164 f"recovery is not working, so affected photos will be reported as failed downloads. "
165 f"Most recent failure: {record_name} - {reason}",
166 )
167 else:
168 LOGGER.debug(f"Failed to refresh download URL for {record_name}: {reason}")
171def _refresh_photo_download_url(photo) -> bool:
172 """Re-fetch the photo's master record from iCloud to obtain fresh download URLs.
174 iCloud download URLs are signed CDN tokens that expire after ~30–40 minutes.
175 When a URL expires (HTTP 410 Gone), clearing ``photo._versions`` alone is
176 insufficient because icloudpy re-parses the same stale ``_master_record``
177 which still contains the expired URL. This function makes a new
178 ``records/lookup`` API call to get an updated master record with fresh URLs,
179 then updates ``photo._master_record`` in place and clears ``_versions`` so
180 the next ``download()`` call uses the fresh URL.
182 ``records/lookup`` is used rather than ``records/query`` because ``CPLMaster``
183 is not a query-indexable CloudKit type: querying it fails every time with
184 ``Type is not marked indexable: CPLMaster (BAD_REQUEST)``. Lookup fetches
185 records by name and returns the same ``{"records": [...]}`` shape.
187 Args:
188 photo: PhotoAsset object from icloudpy
190 Returns:
191 True if the master record was successfully refreshed, False otherwise.
192 """
193 try:
194 record_name = photo._master_record["recordName"] # noqa: SLF001
195 # The record's own zone. icloudpy gives every asset the photos
196 # service, whose zone is always the primary library, so a Shared
197 # Library asset looked up there is never found.
198 record_zone = photo._master_record.get("zoneID") # noqa: SLF001
199 except (AttributeError, KeyError, TypeError):
200 _note_refresh_failure("<unknown>", "photo missing _master_record or recordName")
201 return False
203 service = getattr(photo, "_service", None)
204 if service is None:
205 _note_refresh_failure(record_name, "photo missing _service")
206 return False
208 endpoint = getattr(service, "_service_endpoint", None)
209 session = getattr(service, "session", None)
210 params = getattr(service, "params", None)
211 zone_id = record_zone if isinstance(record_zone, dict) and record_zone else getattr(service, "zone_id", None)
213 if not all([endpoint, session, params, zone_id]):
214 _note_refresh_failure(record_name, "photo._service missing required attributes")
215 return False
217 try:
218 url = f"{endpoint}/records/lookup?{urlencode(params)}"
219 query = {
220 "records": [{"recordName": record_name}],
221 "desiredKeys": _DESIRED_KEYS,
222 "zoneID": zone_id,
223 }
224 request = session.post(
225 url,
226 data=json.dumps(query),
227 headers={"Content-type": "text/plain"},
228 )
229 response = request.json()
230 records = response.get("records", [])
232 for rec in records:
233 if rec.get("recordName") != record_name:
234 continue
235 # A failed lookup is not an HTTP error: CloudKit returns it as a
236 # record inside ``records`` carrying the requested recordName and a
237 # ``serverErrorCode``/``reason`` in place of ``fields``. Matching on
238 # the name alone therefore accepts an error payload as a fresh
239 # master record -- the refresh reports success, and the retrying
240 # download raises KeyError('fields') deep inside icloudpy, which
241 # surfaces as an unexplained "Failed to download <path>: 'fields'"
242 # and is charged to the download rather than to the refresh.
243 #
244 # An empty ``fields`` is no better than a missing one: icloudpy
245 # builds ``versions`` by testing ``f"{prefix}Res" in fields``, so
246 # an empty dict yields no versions at all, ``download()`` returns
247 # None, and the caller dereferences ``.raw`` on it. Accepting
248 # either shape also discards the still-valid master record we
249 # already hold, poisoning the asset for the rest of the sync.
250 # Two distinct failures, reported separately so a refresh that
251 # keeps failing in the field says which one it is.
252 if rec.get("serverErrorCode"):
253 _note_refresh_failure(
254 record_name,
255 f"CloudKit error {rec['serverErrorCode']}: {rec.get('reason') or 'no reason given'}",
256 )
257 return False
258 if not rec.get("fields"):
259 _note_refresh_failure(record_name, "record came back with no usable fields")
260 return False
261 photo._master_record = rec # noqa: SLF001
262 with _versions_refresh_lock:
263 photo._versions = None # noqa: SLF001
264 LOGGER.debug(f"Refreshed download URL for {record_name}")
265 _note_refresh_success()
266 return True
268 _note_refresh_failure(record_name, "record not found in iCloud response")
269 return False
271 except Exception as e: # noqa: BLE001
272 _note_refresh_failure(record_name, str(e))
273 return False
276def check_photo_exists(photo, file_size: str, local_path: str) -> bool:
277 """Check if photo exists locally with correct size.
279 Args:
280 photo: Photo object from iCloudPy
281 file_size: File size variant (original, medium, thumb, etc.)
282 local_path: Local file path to check
284 Returns:
285 True if photo exists locally with correct size, False otherwise
286 """
287 if not (photo and local_path and os.path.isfile(local_path)):
288 return False
290 local_size = os.path.getsize(local_path)
291 remote_size = int(photo.versions[file_size]["size"])
293 if local_size == remote_size:
294 LOGGER.debug(f"No changes detected. Skipping the file {local_path} ...")
295 return True
296 else:
297 LOGGER.debug(f"Change detected: local_file_size is {local_size} and remote_file_size is {remote_size}.")
298 return False
301def create_hardlink(source_path: str, destination_path: str) -> bool:
302 """Create a hard link from source to destination.
304 Args:
305 source_path: Path to existing file to link from
306 destination_path: Path where hardlink should be created
308 Returns:
309 True if hardlink was created successfully, False otherwise
310 """
311 try:
312 # Ensure destination directory exists
313 os.makedirs(os.path.dirname(destination_path), exist_ok=True)
314 # Create hard link
315 os.link(source_path, destination_path)
316 LOGGER.info(f"Created hard link: {destination_path} (linked to existing file: {source_path})")
317 return True
318 except (OSError, FileNotFoundError) as e:
319 LOGGER.warning(f"Failed to create hard link {destination_path}: {e!s}")
320 return False
323def download_photo_from_server(
324 photo,
325 file_size: str,
326 destination_path: str,
327 max_retries: int = 1,
328 timeout: int = DEFAULT_REQUEST_TIMEOUT_SEC,
329) -> bool:
330 """Download photo from iCloud server to local path.
332 This function implements automatic retry logic for HTTP 410 (Gone) errors,
333 which occur when iCloud download URLs expire. When a 410 error is detected,
334 the function re-fetches the photo's master record from iCloud to obtain
335 fresh download URLs, then retries the download.
337 Args:
338 photo: Photo object from iCloudPy
339 file_size: File size variant (original, medium, thumb, etc.)
340 destination_path: Local path where photo should be saved
341 max_retries: Maximum number of retries on 410 errors (default: 1)
342 timeout: HTTP read timeout in seconds. Without one a stalled CDN
343 connection blocks the worker thread forever.
345 Returns:
346 True if download was successful, False otherwise
347 """
348 if not (photo and file_size and destination_path):
349 return False
351 LOGGER.info(f"Downloading {destination_path} ...")
353 max_retries = max(0, max_retries) # Clamp to minimum 0 for predictable behavior
354 attempt = 0
355 max_attempts = max_retries + 1 # Initial attempt + retries
357 # Download beside the target and move it into place only once it is
358 # complete. Writing straight to destination_path left a failed transfer's
359 # partial file where the real photo belongs, where anything reading the
360 # library (an Immich external library, a backup) takes it for the photo,
361 # and it also destroyed the previous good copy before the new one existed.
362 # A partial stranded by a killed process is untracked, so obsolete-file
363 # cleanup removes it.
364 partial = f"{destination_path}.part"
366 while attempt < max_attempts: # noqa: PERF203
367 try:
368 download = photo.download(file_size, timeout=timeout)
369 with open(partial, "wb") as file_out:
370 shutil.copyfileobj(download.raw, file_out)
372 # iCloud declares every version's size, and photo_exists already
373 # treats a mismatch as "not this file". Check it here too, so a
374 # body that is short or is not the photo at all (an error page)
375 # fails now instead of being recorded as downloaded.
376 expected = _declared_size(photo, file_size)
377 written = os.path.getsize(partial)
378 if expected is not None and written != expected:
379 _discard(partial)
380 LOGGER.error(
381 f"Incomplete download of {destination_path}: got {written} of {expected} bytes. "
382 "Will retry on the next sync.",
383 )
384 return False
385 os.replace(partial, destination_path)
387 # Set file modification time to photo's added date.
388 # iCloudPy returns added_date as an aware UTC datetime; replace() is a
389 # safe no-op here because tzinfo is already UTC. If it ever returns a
390 # naive datetime, replace(tzinfo=utc) correctly treats it as UTC.
391 local_modified_time = photo.added_date.replace(tzinfo=timezone.utc).timestamp()
392 os.utime(destination_path, (local_modified_time, local_modified_time))
394 return True
396 except Exception as e: # noqa: PERF203
397 _discard(partial)
398 # Enhanced error logging with file path context
399 # This catches all exceptions including iCloudPy errors like ObjectNotFoundException
400 error_msg = str(e)
402 # Check for HTTP 410 Gone error - download URL has expired
403 # The iCloudPy library raises exceptions with "Gone (410)" in the message
404 # when the download URL has expired (typically after 30-40 minutes)
405 if "Gone (410)" in error_msg:
406 attempt += 1
407 if attempt < max_attempts:
408 LOGGER.warning(
409 f"Download URL expired (410) for {destination_path}. "
410 f"Refreshing URL and retrying (attempt {attempt}/{max_attempts})...",
411 )
412 # Re-fetch the master record from iCloud to obtain fresh download URLs.
413 # Simply clearing _versions is insufficient because icloudpy re-parses
414 # the same stale _master_record which still contains the expired URL.
415 _refresh_photo_download_url(photo)
416 continue
417 else:
418 LOGGER.error(
419 f"Failed to download {destination_path} after {max_retries} retries: {error_msg}",
420 )
421 return False
423 # Handle other errors
424 if "ObjectNotFoundException" in error_msg or "NOT_FOUND" in error_msg:
425 LOGGER.error(f"Photo not found in iCloud Photos - {destination_path}: {error_msg}")
426 else:
427 LOGGER.error(f"Failed to download {destination_path}: {error_msg}")
428 return False
430 # This line should never be reached due to the logic above, but is kept as defensive programming
431 return False # pragma: no cover
434def _declared_size(photo, file_size: str) -> int | None:
435 """The size iCloud declares for this version, or None if it does not say."""
436 try:
437 return int(photo.versions[file_size]["size"])
438 except (AttributeError, KeyError, TypeError, ValueError):
439 return None
442def _discard(path: str) -> None:
443 """Remove a partial download if one was written."""
444 with contextlib.suppress(FileNotFoundError):
445 os.remove(path)
448def rename_legacy_file_if_exists(old_path: str, new_path: str) -> None:
449 """Rename legacy file format to new format if it exists.
451 Args:
452 old_path: Path to legacy file format
453 new_path: Path to new file format
454 """
455 if os.path.isfile(old_path):
456 os.rename(old_path, new_path)