From 698cbe145a76fe03f9f23469bad96d25f6774027 Mon Sep 17 00:00:00 2001 From: "Oliver Surke (Gitea)" Date: Tue, 12 May 2026 09:46:12 +0200 Subject: [PATCH] feat: async iCloud prefetch + keep-local for Mackup folder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lib/icloud.sh (new): ssr::icloud::prefetch — spawn background open -g requests for all .icloud stubs; returns immediately so other work can run in parallel. ssr::icloud::wait [sec] — poll until stubs are gone or timeout (default 180s). Logs progress every 5s. ssr::icloud::keep_local — after sync, removes com.apple.icloud.evictable xattr from every real file + updates access-time (LRU delay), preventing iCloud from re-creating stubs before the next run. Best-effort (no public pin API on macOS CLI). bin/ssr-sync: - prefetch Mackup folder immediately after cloud is available (before brew update/upgrade) so downloads proceed in the background. - ssr::icloud::wait called just before mackup backup. - ssr::icloud::keep_local called after sync completes. - SSR_ICLOUD_WAIT_TIMEOUT (default 180) and SSR_ICLOUD_KEEP_LOCAL (default true) config toggles added. bin/ssr-restore: - prefetch Mackup folder right after cloud is validated, before the brew bundle install step, maximising download time available. config/ssr.conf.example: Added SSR_ICLOUD_WAIT_TIMEOUT and SSR_ICLOUD_KEEP_LOCAL keys. Co-authored-by: Cursor --- bin/ssr-restore | 7 +++ bin/ssr-sync | 18 +++++++ config/ssr.conf.example | 12 +++++ lib/icloud.sh | 109 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 146 insertions(+) create mode 100644 lib/icloud.sh diff --git a/bin/ssr-restore b/bin/ssr-restore index 917bc09..fa55cc9 100755 --- a/bin/ssr-restore +++ b/bin/ssr-restore @@ -21,6 +21,8 @@ source "$SSR_ROOT/lib/macos.sh" source "$SSR_ROOT/lib/macos-prefs.sh" # shellcheck source=../lib/restore-state.sh source "$SSR_ROOT/lib/restore-state.sh" +# shellcheck source=../lib/icloud.sh +source "$SSR_ROOT/lib/icloud.sh" ssr::require_macos ssr::load_config @@ -74,9 +76,14 @@ ssr::log "Using Brewfile: $brewfile" # Initialise state (archives previous state file if present). ssr::rs::init "$brewfile" +# Trigger async iCloud prefetch for the Mackup folder as early as possible +# so files have time to download while brew bundle install runs. +ssr::icloud::prefetch "$mackup_dir" + # --- Step helpers ------------------------------------------------------------ # Groups mackup install + configure + restore into a single resumable step. +# ssr::mackup::restore already calls ssr::icloud::wait internally. _ssr_mackup_restore_step() { ssr::mackup::ensure_installed ssr::mackup::configure "$mackup_dir" diff --git a/bin/ssr-sync b/bin/ssr-sync index 54f61f4..d2d8b4c 100755 --- a/bin/ssr-sync +++ b/bin/ssr-sync @@ -15,6 +15,8 @@ source "$SSR_ROOT/lib/mackup.sh" source "$SSR_ROOT/lib/apps.sh" # shellcheck source=../lib/macos-prefs.sh source "$SSR_ROOT/lib/macos-prefs.sh" +# shellcheck source=../lib/icloud.sh +source "$SSR_ROOT/lib/icloud.sh" ssr::require_macos ssr::load_config @@ -28,12 +30,22 @@ mackup_dir="$(ssr::cloud::mackup_dir)" ssr::ensure_dir "$backup_dir" ssr::ensure_dir "$mackup_dir" +# Kick off async iCloud download for the Mackup folder immediately so that +# files have maximum time to download while brew update/upgrade runs. +# ssr::icloud::wait is called just before mackup backup below. +ssr::icloud::prefetch "$mackup_dir" + ssr::brew::ensure_installed ssr::mackup::ensure_installed ssr::mackup::configure "$mackup_dir" ssr::brew::update ssr::brew::upgrade + +# Wait for iCloud downloads triggered above before handing off to mackup. +# Timeout (seconds) is configurable; default 180s. +ssr::icloud::wait "$mackup_dir" "${SSR_ICLOUD_WAIT_TIMEOUT:-180}" + ssr::mackup::backup ssr::brew::dump "$backup_dir" ssr::brew::cleanup @@ -67,5 +79,11 @@ state_dir="${SSR_STATE_DIR:-$HOME/.local/state/ssr}" ssr::ensure_dir "$state_dir" date '+%Y-%m-%dT%H:%M:%S%z' > "$state_dir/last-sync" +# Mark Mackup folder files as keep-local so iCloud doesn't evict them +# before the next sync run, avoiding slow on-demand downloads next time. +if [[ "${SSR_ICLOUD_KEEP_LOCAL:-true}" == "true" ]]; then + ssr::icloud::keep_local "$mackup_dir" +fi + ssr::ok "ssr sync completed → $backup_dir" ssr::syslog "sync completed" diff --git a/config/ssr.conf.example b/config/ssr.conf.example index 307c799..e28e0d7 100644 --- a/config/ssr.conf.example +++ b/config/ssr.conf.example @@ -24,6 +24,18 @@ SSR_BREW_CLEANUP=true # --------------------------------------------------------------------------- # Unmanaged-app scan during sync +# --------------------------------------------------------------------------- +# --------------------------------------------------------------------------- +# iCloud prefetch & keep-local +# --------------------------------------------------------------------------- +# Trigger async iCloud download at sync/restore start so files are available +# before Mackup runs. Wait up to this many seconds before proceeding. +# SSR_ICLOUD_WAIT_TIMEOUT=180 + +# After sync, mark Mackup folder files as keep-local to prevent iCloud from +# re-evicting them before the next run (LRU touch + xattr, best-effort). +SSR_ICLOUD_KEEP_LOCAL=true + # --------------------------------------------------------------------------- # Lists .app bundles installed manually (not via brew cask, not via MAS). # Writes /unmanaged-apps.txt for restore-time reference. diff --git a/lib/icloud.sh b/lib/icloud.sh new file mode 100644 index 0000000..dd3ed50 --- /dev/null +++ b/lib/icloud.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# lib/icloud.sh — iCloud Drive prefetch, wait, and keep-local helpers. +# +# All functions are no-ops when the target directory is not under iCloud +# (i.e. does not contain "Mobile Documents" in the path). + +# --- Guard: only operate on iCloud paths ------------------------------------ +_ssr::icloud::is_icloud() { + case "$1" in *Mobile\ Documents*) return 0 ;; esac + return 1 +} + +# --------------------------------------------------------------------------- +# ssr::icloud::prefetch +# +# Triggers background download of every .icloud placeholder file found under +# . Returns immediately (async). Use ssr::icloud::wait to poll for +# completion before running tools that need the real file content. +# --------------------------------------------------------------------------- +ssr::icloud::prefetch() { + local dir="$1" + [[ -d "$dir" ]] || return 0 + _ssr::icloud::is_icloud "$dir" || return 0 + + local count=0 stub d base real + while IFS= read -r stub; do + [[ -n "$stub" ]] || continue + d="$(dirname "$stub")" + base="$(basename "$stub")" + # .icloud stubs are named ..icloud — strip leading dot + suffix. + real="${base#.}"; real="${real%.icloud}" + # open -g: open in background without bringing any app to the front. + # This is the supported way to request iCloud Drive file materialisation + # since brctl download was removed in macOS Sonoma. + open -g "$d/$real" 2>/dev/null & + count=$(( count + 1 )) + done < <(find "$dir" -name '*.icloud' -type f 2>/dev/null) + + if [[ $count -gt 0 ]]; then + ssr::log "iCloud prefetch: triggered download of $count placeholder(s) in $(basename "$dir") (async)" + fi +} + +# --------------------------------------------------------------------------- +# ssr::icloud::wait [timeout_seconds] +# +# Polls until all .icloud placeholder files under are gone (fully +# downloaded) or until (default 180) elapses. +# Returns 0 on full completion, 1 on timeout (non-fatal). +# --------------------------------------------------------------------------- +ssr::icloud::wait() { + local dir="$1" timeout="${2:-180}" interval=5 + [[ -d "$dir" ]] || return 0 + _ssr::icloud::is_icloud "$dir" || return 0 + + local elapsed=0 remaining + while [[ $elapsed -lt $timeout ]]; do + remaining="$(find "$dir" -name '*.icloud' -type f 2>/dev/null | wc -l | tr -d ' ')" + [[ "$remaining" -eq 0 ]] && { + ssr::ok "iCloud sync complete (all files local)" + return 0 + } + ssr::log "iCloud: $remaining file(s) still downloading… (${elapsed}s/${timeout}s)" + sleep "$interval" + elapsed=$(( elapsed + interval )) + done + + remaining="$(find "$dir" -name '*.icloud' -type f 2>/dev/null | wc -l | tr -d ' ')" + ssr::warn "iCloud wait timed out after ${timeout}s — $remaining placeholder(s) still pending; continuing anyway" + return 1 +} + +# --------------------------------------------------------------------------- +# ssr::icloud::keep_local +# +# Best-effort: marks every real (non-stub) file under as non-evictable +# so iCloud Drive does not replace them with .icloud placeholders between +# runs. Two complementary techniques: +# +# 1. Remove com.apple.icloud.evictable — iCloud-internal xattr that, when +# absent (or set to 0), prevents on-demand eviction. This xattr is +# written by iCloudDrive when eviction is allowed; removing it pins +# the file locally. (Internal API — best-effort.) +# +# 2. Update the access-time (touch -a) — iCloud uses an LRU policy; files +# accessed recently are the last to be evicted. +# +# Note: Apple provides no *public* CLI to permanently pin iCloud files. +# For a guaranteed "Keep Downloaded" equivalent the Finder UI must be used, +# or NSFileManager.setUbiquitous(false, …) from a native app. +# --------------------------------------------------------------------------- +ssr::icloud::keep_local() { + local dir="$1" + [[ -d "$dir" ]] || return 0 + _ssr::icloud::is_icloud "$dir" || return 0 + + local count=0 f + while IFS= read -r f; do + [[ -n "$f" ]] || continue + # 1) Remove the evictable attribute (ignore errors on files that don't have it). + xattr -d "com.apple.icloud.evictable" "$f" 2>/dev/null || true + # 2) Touch access time so LRU keeps this file at the back of the eviction queue. + touch -a "$f" 2>/dev/null || true + count=$(( count + 1 )) + done < <(find "$dir" -type f -not -name '*.icloud' 2>/dev/null) + + [[ $count -gt 0 ]] && ssr::log "iCloud keep-local: marked $count file(s) in $(basename "$dir") (access-time + xattr)" + return 0 +}