Coverage for src/drive_package_processing.py: 100%
86 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-08 21:05 +0000
1"""Package processing utilities.
3This module provides package extraction and processing functionality,
4separating archive handling logic from sync operations per SRP.
5"""
7__author__ = "Mandar Patil (mandarons@pm.me)"
9import gzip
10import os
11import unicodedata
12import zipfile
13from shutil import copyfileobj
15import magic
17from src import configure_icloudpy_logging, get_logger
19# Configure icloudpy logging immediately after import
20configure_icloudpy_logging()
22LOGGER = get_logger()
24# Guard against a crafted archive chain (gzip wrapping gzip wrapping ...)
25# recursing without bound. Real iCloud packages are one archive deep (gzip of
26# a zip, at most). Past this depth we stop unpacking and keep the bytes as-is
27# rather than risk a RecursionError on hostile input. gzip streams -- unlike
28# ZIP entries, which zipfile sanitises -- carry no path info, so the only
29# defence against a decompression-bomb chain is a depth ceiling.
30_MAX_PACKAGE_DEPTH = 8
33def process_package(
34 local_file: str, flatten: bool = False, _depth: int = 0,
35) -> str | None:
36 """Process and extract a downloaded package file.
38 This function handles different archive types (ZIP, gzip) and extracts them
39 to the appropriate location.
41 Args:
42 local_file: Path to the downloaded package file
43 flatten: When True, skip unpacking entirely and keep the package
44 on disk as a single binary file (the gzip / zip bytes). Useful
45 for backup-style deployments (NAS / cold storage) where bundle-
46 directory semantics aren't needed and single-file storage is
47 preferred for simpler dedup, restoration, and inode footprint.
48 Opt-in via ``drive.flatten_packages: true`` in config.yaml.
50 Returns:
51 Path to the processed file/directory. The local file path is also
52 returned when the mime type is not a recognised archive (the
53 bytes are preserved on disk as a single-file bundle — Apple's
54 iWork formats ``.key``/``.pages``/``.numbers``, JMG ``.jmb``,
55 etc. report as ``application/octet-stream`` and don't need
56 unpacking to be usable by their target application). Returns
57 ``None`` only on hard processing errors that leave the file in
58 an unusable state.
59 """
60 archive_file = local_file
62 if flatten:
63 # The downloaded bytes are already at local_file; nothing to do.
64 # Skip libmagic entirely -- flatten mode doesn't care what the
65 # bytes are, and libmagic can fail on truncated or unusual
66 # downloads where we still want to keep the bytes on disk.
67 LOGGER.info(
68 f"flatten_packages enabled -- keeping {local_file} as "
69 f"single-file bundle; skipping unpack.",
70 )
71 return local_file
73 if _depth > _MAX_PACKAGE_DEPTH:
74 LOGGER.warning(
75 f"Package nesting exceeded {_MAX_PACKAGE_DEPTH} levels for {local_file}; "
76 f"stopping unpack and keeping the current bytes as a single-file bundle.",
77 )
78 return local_file
80 # zipfile.is_zipfile() rather than libmagic's MIME string: libmagic reports
81 # "application/octet-stream" for many of Apple's packageDownload zips even though
82 # they begin with PK\x03\x04 and open fine with zipfile. is_zipfile() reads the End
83 # of Central Directory record and is the authority on whether ZipFile() will
84 # succeed, which is all _process_zip_package() needs to know.
85 if zipfile.is_zipfile(local_file):
86 return _process_zip_package(local_file, archive_file)
88 magic_object = magic.Magic(mime=True)
89 file_mime_type = magic_object.from_file(filename=local_file)
91 if file_mime_type == "application/gzip":
92 return _process_gzip_package(local_file, archive_file, _depth=_depth)
93 else:
94 # NOT an error — many iCloud Drive "package" downloads are flat
95 # binary bundles (Apple iWork .key/.pages/.numbers,
96 # JMG's third-party .jmb, etc) that report as application/octet-stream
97 # but don't need to be unpacked to be usable by their target
98 # application. The bytes are already on disk at ``local_file``;
99 # treat that as the canonical local representation.
100 LOGGER.info(
101 f"Package format not recognised for unpacking ({file_mime_type}); "
102 f"keeping as single-file bundle: {local_file}",
103 )
104 return local_file
107def _zip_entries_self_prefixed(zf: zipfile.ZipFile, bundle_basename: str) -> str | None:
108 """Heuristic classifier for the three zip layouts iCloud Drive serves.
110 Returns one of:
111 - ``"self"`` — every entry starts with ``<bundle_basename>/``;
112 extract into parent_dir, safety boundary is
113 parent_dir.
114 - ``"traversal"`` — every entry starts with ``../<bundle_basename>/``
115 (gzip-wrapped bundles do this). The traversal
116 resolves UP one level, so extracting into
117 parent_dir lands files in *grandparent_dir*.
118 Safety boundary must be grandparent_dir too.
119 - ``None`` — bare-rooted (or empty); extract into a bundle-
120 named subdir to avoid sibling collisions.
122 Differentiating ``self`` vs ``traversal`` matters for the zip-slip
123 safety check: ``traversal`` is a benign-but-escape-the-extract-dir
124 pattern, so the safety boundary needs to be set to the dir the
125 ``..`` lands in, not the dir we passed to ``extractall``.
127 The three layouts in the wild:
129 1. **self** — e.g. a GarageBand ``Project.band`` whose zip contains
130 entries like ``Project.band/Alternatives/000/...``. Extracting
131 into the parent dir reconstructs the bundle correctly.
133 2. **traversal** — gzip-wrapped bundles whose inner ZIP has
134 entries like ``../<basename>/...``. The ``..`` resolves to
135 the parent of the extract target, so when we extract into the
136 parent dir the files end up in the grandparent. End-state is
137 analogous to ``self`` but one level up.
139 3. **None (bare-rooted)** — e.g. a Numbers ``Untitled.numbers``
140 whose zip contains generic entries like ``Data/Document.iwa``,
141 ``Metadata/buildVersion.plist``. Two iWork files in the same
142 folder both extract ``Data/...`` and clobber each other; for
143 this case we extract into a bundle-named subdir.
144 """
145 names = [n for n in zf.namelist() if n and n != bundle_basename + "/"]
146 if not names:
147 return None
148 prefix = bundle_basename + "/"
149 traversal_prefix = "../" + prefix
150 if all(n.startswith(prefix) for n in names):
151 return "self"
152 if all(n.startswith((prefix, traversal_prefix)) for n in names):
153 return "traversal"
154 return None
157def _safe_extractall(
158 zf: zipfile.ZipFile, extract_dir: str, safety_boundary: str,
159) -> None:
160 """Path-validating wrapper around ``ZipFile.extractall``.
162 Defends against Zip Slip (CWE-22): a malicious zip can embed entries
163 with absolute paths (``/etc/passwd``) or ``..`` traversal segments
164 that resolve outside the intended extract directory. iCloud Drive
165 package bytes are untrusted network input -- if Apple's CloudKit
166 were ever compromised, or an attacker-controlled iCloud account
167 shared a poisoned package with the user, ``extractall`` would
168 happily write anywhere on the filesystem.
170 For each entry, resolves the final target path with ``os.path.realpath``
171 and verifies it stays under ``safety_boundary`` (the directory we're
172 extracting into, OR -- in the self-prefixed-with-``../`` case -- the
173 parent dir whose nesting structure the zip relies on). Entries that
174 escape are skipped with a WARNING.
176 Args:
177 zf: Open ZipFile.
178 extract_dir: Path passed to ``extractall``.
179 safety_boundary: Absolute directory the extracted tree must
180 remain under. For self-prefixed bundles this is the parent
181 of the bundle; for bare-rooted bundles this equals
182 ``extract_dir``.
183 """
184 # realpath canonicalises a path that does not exist yet (the bare-rooted
185 # case creates the bundle subdir just before this, but nothing here
186 # relies on that) -- POSIX realpath resolves what exists and appends
187 # the rest literally, which is what the containment check needs.
188 boundary_abs = os.path.realpath(safety_boundary)
189 safe_members = []
190 for member in zf.infolist():
191 target = os.path.realpath(os.path.join(extract_dir, member.filename))
192 # The boundary check uses ``commonpath`` so we don't get tricked
193 # by prefixes like ``/var/data2/`` matching ``/var/data`` via
194 # naive ``startswith``.
195 try:
196 common = os.path.commonpath([boundary_abs, target])
197 except ValueError: # pragma: no cover -- Windows-only (different drives)
198 common = ""
199 if common != boundary_abs:
200 LOGGER.warning(
201 f"Zip Slip blocked: refused to extract {member.filename!r} "
202 f"-> {target!r} (outside safety boundary {boundary_abs!r})",
203 )
204 continue
205 safe_members.append(member)
206 # Pass the filtered member list to extractall so vetoed entries are
207 # never touched.
208 zf.extractall(path=extract_dir, members=safe_members)
211def _process_zip_package(local_file: str, archive_file: str) -> str:
212 """Process a ZIP package file.
214 Args:
215 local_file: Original file path
216 archive_file: Archive file path
218 Returns:
219 Path to the processed file
220 """
221 archive_file += ".zip"
222 os.rename(local_file, archive_file)
223 parent_dir = os.path.dirname(archive_file)
224 bundle_basename = os.path.basename(local_file)
226 with zipfile.ZipFile(archive_file) as zf:
227 layout = _zip_entries_self_prefixed(zf, bundle_basename)
228 if layout == "self":
229 # Entries are ``<bundle>/...``; extract into parent_dir and
230 # the bundle dir takes shape at ``<parent_dir>/<bundle>/``.
231 extract_dir = parent_dir
232 safety_boundary = parent_dir
233 elif layout == "traversal":
234 # Entries are ``../<bundle>/...``. Extracting into parent_dir
235 # makes the ``..`` resolve to grandparent_dir, so files land
236 # at ``<grandparent_dir>/<bundle>/...``. The safety boundary
237 # must include grandparent_dir for the legitimate traversal
238 # to be allowed -- otherwise the zip-slip guard rejects every
239 # entry. Real bundles in the wild (gzip-wrapped GarageBand
240 # exports etc.) use this layout.
241 extract_dir = parent_dir
242 safety_boundary = os.path.dirname(parent_dir) or parent_dir
243 else:
244 # Bare-rooted: entries are ``Data/Document.iwa`` etc.
245 # Extract into a bundle-named subdir so siblings don't
246 # collide on shared internal names.
247 extract_dir = local_file
248 os.makedirs(extract_dir, exist_ok=True)
249 safety_boundary = extract_dir
250 LOGGER.info(f"Unpacking {archive_file} to {extract_dir}")
251 _safe_extractall(zf, extract_dir, safety_boundary)
253 # The package stays at the NFC path it was asked for. Renaming it to the
254 # NFD form of the whole path broke on Linux, where the two forms are
255 # different names: a non-ASCII parent folder made the rename fail, and a
256 # non-ASCII package name left it where the next sync (which looks for the
257 # NFC path) could not find it, so it was downloaded again every time.
259 os.remove(archive_file)
260 LOGGER.info(f"Successfully unpacked the package {archive_file}.")
261 _warn_about_a_leftover_nfd_copy(local_file)
262 return local_file
265def _warn_about_a_leftover_nfd_copy(local_file: str) -> None:
266 """Point out a copy an earlier version left at the NFD form of the path.
268 Those versions renamed non-ASCII packages to NFD; the package is now
269 downloaded again under its NFC name, and the old copy is never looked
270 at again. Obsolete cleanup removes it, but only when
271 ``drive.remove_obsolete`` is on, which is not the default -- so say so
272 rather than let it sit there at double the size. Fires when the package
273 is downloaded, which for an unchanged package is once. On a filesystem
274 where both forms name the same file (macOS), there is nothing to report.
275 """
276 leftover = unicodedata.normalize("NFD", local_file)
277 if leftover == local_file or not os.path.exists(leftover) or os.path.samefile(leftover, local_file):
278 return
279 LOGGER.warning(
280 f"An older copy of {local_file} is still at {leftover!r}, left by an earlier "
281 f"version that renamed packages to NFD. It is no longer synced: delete it, "
282 f"or set drive.remove_obsolete: true to have it cleaned up.",
283 )
286def _process_gzip_package(
287 local_file: str, archive_file: str, _depth: int = 0,
288) -> str | None:
289 """Process a gzip package file.
291 Args:
292 local_file: Original file path
293 archive_file: Archive file path
295 Returns:
296 Path to the processed file, or None if processing failed
297 """
298 archive_file += ".gz"
299 os.rename(local_file, archive_file)
300 LOGGER.info(f"Unpacking {archive_file} to {os.path.dirname(local_file)}")
302 with gzip.GzipFile(filename=archive_file, mode="rb") as gz_file:
303 with open(file=local_file, mode="wb") as package_file:
304 copyfileobj(gz_file, package_file)
306 os.remove(archive_file)
308 # Recursively process the extracted file (might be another archive).
309 # Thread the depth counter so a hostile gzip chain can't recurse forever.
310 return process_package(local_file=local_file, _depth=_depth + 1)