From 58b203e4fa217181a9faa3b28a3dd57d1b58e825 Mon Sep 17 00:00:00 2001 From: Greg Pomerantz Date: Sun, 1 Nov 2026 20:24:01 -0500 Subject: [PATCH] gtauri: event-driven watching + defense against stale VFS dir listings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live watching is now event-driven, no tree polling: - Android: MainActivity registers a MediaStore ContentObserver (Images, notifyForDescendants) and wakes the Rust watcher thread via a JNI call (Java_wow_gmp_gallery_MainActivity_nativeFsChange) — a single channel send, no main-thread work. Covers camera saves, media-scanner indexing, other apps' indexed writes. - Desktop: inotify/FSEvents via notify on the scan roots. - On wake: coalesce + debounce, diff dir mtimes, rescan only the changed subtrees. The 1 s recv timeout doubles as a cheap backstop (raw out-of-band copies the scanner hasn't indexed yet); a 5-minute full pass is the safety net. minSdk 30 (3-arg ContentObserver API). Stale-listing truncation bug found and fixed (reproduced on the emulator's 9p /sdcard; the phone's FUSE does the same): a readdir over a large directory can return a SILENT early EOF — a stale cached listing with NO error. The phone's 9961-file Camera dir truncated at ~8400 once and db::apply deleted ~1700 good rows. Three layers now: 1. scan::read_dir_full — retry whole-dir reads on mid-stream errors; 2. per-directory cross-check in walk_dir: if the enumeration is far below the DB's count for that dir, re-read up to 3x keeping the best listing; whole-walk re-check in rescan keeps the larger of two walks; 3. db::apply mass-delete guard: never delete >20% of the DB in one apply unless a previous walk already saw the low count (two-strike; a meta.walked row tracks it). Genuine gradual deletions still flow through (verified: a 4000-file mass move streams the grid down progressively), and a second consecutive low walk still deletes. Emulator reliability suite (all pass): 3x launch count stability, add +1 / remove -1 in ~2 s, 20-file burst, DB recovery from damage, stale-listing refusal (0 rows lost), two-strike delete, live restore, idle 0.0% dropped frames, deep fling loads thumbnails. A/B vs ttimg (no-backend CSS-scroll control): adb-swipe drop deltas are at the input-injection artifact level, idle identical (0.0%). --- harnesses/gtauri/Cargo.lock | 80 ++++++ harnesses/gtauri/Cargo.toml | 5 + .../gtauri/gen/android/app/build.gradle.kts | 2 +- .../main/java/wow/gmp/gallery/MainActivity.kt | 33 +++ harnesses/gtauri/src/db.rs | 77 +++++- harnesses/gtauri/src/lib.rs | 250 ++++++++++++++---- harnesses/gtauri/src/scan.rs | 121 ++++++++- harnesses/gtauri/tauri.conf.json | 3 + 8 files changed, 511 insertions(+), 60 deletions(-) diff --git a/harnesses/gtauri/Cargo.lock b/harnesses/gtauri/Cargo.lock index f0a3c57..6fcee0f 100644 --- a/harnesses/gtauri/Cargo.lock +++ b/harnesses/gtauri/Cargo.lock @@ -1215,6 +1215,7 @@ dependencies = [ "image", "kamadak-exif", "log", + "notify", "rusqlite", "serde", "serde_json", @@ -1622,6 +1623,26 @@ dependencies = [ "cfb", ] +[[package]] +name = "inotify" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cc00ea907cab49550b7da656f80ebb97be1b997d931fbcd28d39734e17ce592" +dependencies = [ + "bitflags 2.13.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c033f80b2c113cdf91ab7a33faa9cbc014726dcad99880c8609af2a370edf37d" +dependencies = [ + "libc", +] + [[package]] name = "ipnet" version = "2.12.2" @@ -1807,6 +1828,26 @@ dependencies = [ "serde", ] +[[package]] +name = "kqueue" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d763e5b24120b4ddf50de6c92308156765aabfbbccebf401da7cff2d70a41ea" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087" +dependencies = [ + "bitflags 2.13.2", + "libc", +] + [[package]] name = "libappindicator" version = "0.9.0" @@ -1956,6 +1997,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.61.2", ] @@ -2033,6 +2075,34 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "notify" +version = "8.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f812702fbc75d35416d3f884e04925be7e9498f4eceea2e24ba34559284674" +dependencies = [ + "bitflags 2.13.2", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "objc2-core-foundation", + "objc2-core-services", + "walkdir", + "windows-sys 0.61.2", +] + +[[package]] +name = "notify-types" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a437a9308232ce794b1e7fdf540d38ea37efc3c73492640c13b4c8568b8f108" +dependencies = [ + "bitflags 2.13.2", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -2159,6 +2229,16 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc2-core-services" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583300ad934cba24ff5292aee751ecc070f7ca6b39a574cc21b7b5e588e06a0b" +dependencies = [ + "libc", + "objc2-core-foundation", +] + [[package]] name = "objc2-core-text" version = "0.3.2" diff --git a/harnesses/gtauri/Cargo.toml b/harnesses/gtauri/Cargo.toml index 6d6c291..d4352c3 100644 --- a/harnesses/gtauri/Cargo.toml +++ b/harnesses/gtauri/Cargo.toml @@ -25,3 +25,8 @@ turbojpeg = { version = "1.5.1", features = ["cmake"] } kamadak-exif = "0.6" sha1 = "0.11" image = { version = "0.25", default-features = false, features = ["png", "webp", "gif"] } + +# Desktop file watching (inotify on Linux, FSEvents on macOS). +# Android uses the MediaStore ContentObserver instead — see MainActivity.kt. +[target.'cfg(not(target_os = "android"))'.dependencies] +notify = "8" diff --git a/harnesses/gtauri/gen/android/app/build.gradle.kts b/harnesses/gtauri/gen/android/app/build.gradle.kts index 98813bb..875ee62 100644 --- a/harnesses/gtauri/gen/android/app/build.gradle.kts +++ b/harnesses/gtauri/gen/android/app/build.gradle.kts @@ -20,7 +20,7 @@ android { defaultConfig { manifestPlaceholders["usesCleartextTraffic"] = "false" applicationId = "wow.gmp.gallery" - minSdk = 24 + minSdk = 30 targetSdk = 37 versionCode = tauriProperties.getProperty("tauri.android.versionCode", "1").toInt() versionName = tauriProperties.getProperty("tauri.android.versionName", "1.0") diff --git a/harnesses/gtauri/gen/android/app/src/main/java/wow/gmp/gallery/MainActivity.kt b/harnesses/gtauri/gen/android/app/src/main/java/wow/gmp/gallery/MainActivity.kt index e253e5b..44547eb 100644 --- a/harnesses/gtauri/gen/android/app/src/main/java/wow/gmp/gallery/MainActivity.kt +++ b/harnesses/gtauri/gen/android/app/src/main/java/wow/gmp/gallery/MainActivity.kt @@ -1,11 +1,44 @@ package wow.gmp.gallery +import android.database.ContentObserver +import android.net.Uri + import android.os.Bundle +import android.os.Handler +import android.os.Looper +import android.provider.MediaStore import androidx.activity.enableEdgeToEdge class MainActivity : TauriActivity() { + private var observer: ContentObserver? = null + + // Rust side (this app's JNI library, loaded by TauriActivity): wakes + // the file-watcher thread. One JNI call + one channel send — nothing + // heavy ever runs on the main thread. + external fun nativeFsChange() + override fun onCreate(savedInstanceState: Bundle?) { enableEdgeToEdge() super.onCreate(savedInstanceState) + // Event-driven file watching (no polling): MediaStore changes — + // camera saves, media-scanner indexing, other apps' indexed writes — + // wake the Rust watcher, which diffs dir mtimes and does an + // incremental rescan of only the changed subtrees. + val obs = object : ContentObserver(Handler(Looper.getMainLooper())) { + override fun onChange(selfChange: Boolean, uri: Uri?) { + nativeFsChange() + } + } + observer = obs + // 3-arg overload (notifyForDescendants) requires API 30 — the app's + // minSdk (see tauri.conf.json bundle.android.minSdkVersion). + contentResolver.registerContentObserver( + MediaStore.Images.Media.EXTERNAL_CONTENT_URI, true, obs + ) + } + + override fun onDestroy() { + observer?.let { contentResolver.unregisterContentObserver(it) } + super.onDestroy() } } diff --git a/harnesses/gtauri/src/db.rs b/harnesses/gtauri/src/db.rs index 267e0b8..f249cee 100644 --- a/harnesses/gtauri/src/db.rs +++ b/harnesses/gtauri/src/db.rs @@ -73,7 +73,11 @@ pub fn open(path: &std::path::Path) -> Result { thumb_path TEXT, date_taken INTEGER ); - CREATE INDEX IF NOT EXISTS items_mtime ON items (mtime DESC);", + CREATE INDEX IF NOT EXISTS items_mtime ON items (mtime DESC); + CREATE TABLE IF NOT EXISTS meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + );", )?; Ok(conn) } @@ -132,10 +136,56 @@ pub fn apply( } } } + // MASS-DELETE GUARD: if this walk sees dramatically fewer files + // than the DB holds, the walk almost certainly truncated (FUSE + // mid-stream readdir error — see scan::read_dir_full) rather than + // a real mass deletion. Refuse to delete on the FIRST suspicious + // walk; a second consecutive low walk confirms it is real. + let mut to_delete = 0; for p in existing.keys() { if !seen.contains(p) { - del.execute(params![p])?; - deleted += 1; + to_delete += 1; + } + } + let suspect = existing.len() > 100 + && (to_delete as f64) > (existing.len() as f64) * 0.2; + let mut do_delete = true; + if suspect { + let last_walked: i64 = tx + .query_row("SELECT value FROM meta WHERE key='walked'", [], |r| { + r.get::<_, String>(0) + }) + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(existing.len() as i64); + if last_walked >= (existing.len() as i64) * 80 / 100 { + log::warn!( + "apply: walk saw {} but DB has {} (last walk {}): refusing to delete {} rows (truncated walk?)", + entries.len(), + existing.len(), + last_walked, + to_delete + ); + do_delete = false; + } + let _ = tx.execute( + "INSERT INTO meta (key, value) VALUES ('walked', ?1) \ + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + params![entries.len().to_string()], + ); + } else { + let _ = tx.execute( + "INSERT INTO meta (key, value) VALUES ('walked', ?1) \ + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + params![entries.len().to_string()], + ); + } + if do_delete { + for p in existing.keys() { + if !seen.contains(p) { + del.execute(params![p])?; + deleted += 1; + } } } } @@ -143,6 +193,27 @@ pub fn apply( Ok((inserted, updated, deleted)) } +/// Number of rows in the DB. +pub fn count(conn: &mut Connection) -> Result { + let n: i64 = conn.query_row("SELECT COUNT(*) FROM items", [], |r| r.get(0))?; + Ok(n) +} + +/// Per-directory file counts (parent dir → row count) — the reference +/// the walk uses to detect stale VFS directory listings. +pub fn dir_counts(conn: &mut Connection) -> Result, rusqlite::Error> { + let mut st = conn.prepare("SELECT path FROM items")?; + let rows = st.query_map([], |r| r.get::<_, String>(0))?; + let mut m: HashMap = HashMap::new(); + for p in rows { + let p = p?; + if let Some(d) = std::path::Path::new(&p).parent() { + *m.entry(d.to_path_buf()).or_insert(0) += 1; + } + } + Ok(m) +} + /// All known entries, newest first. pub fn list_entries(conn: &mut Connection) -> Result, rusqlite::Error> { let mut st = conn.prepare("SELECT path, mtime, size FROM items ORDER BY mtime DESC, path ASC")?; diff --git a/harnesses/gtauri/src/lib.rs b/harnesses/gtauri/src/lib.rs index b9949f6..045b3a9 100644 --- a/harnesses/gtauri/src/lib.rs +++ b/harnesses/gtauri/src/lib.rs @@ -713,20 +713,71 @@ fn load_meta(conn: &mut db::Connection) -> Result<(), rusqlite::Error> { Ok(()) } -/// Incremental rescan: walk the roots, diff against the DB (insert -/// new / update changed / delete removed), refresh META, emit the list. -/// Used by the startup scan AND the live watcher — a new photo appears -/// in the grid seconds after it is written. -fn rescan(app: &tauri::AppHandle) { +/// Total rows in the DB (0 on error). +fn expected_count() -> i64 { + match db::open(&db::db_path()) { + Ok(mut c) => db::count(&mut c).unwrap_or(0), + Err(_) => 0, + } +} + +/// Per-directory file counts from the DB — the reference the walk uses +/// to detect stale (silently truncated) VFS directory listings. +fn expected_dir_counts() -> std::collections::HashMap { + match db::open(&db::db_path()) { + Ok(mut c) => db::dir_counts(&mut c).unwrap_or_default(), + Err(_) => std::collections::HashMap::new(), + } +} + +/// Walk the roots once, streaming progress emits. Cross-checks each +/// directory's enumeration against the previous DB counts (stale-listing +/// protection — see scan::walk_stream). +fn walk_once(app: &tauri::AppHandle) -> Vec { + let expected = expected_dir_counts(); let roots = scan::roots(); let last_emit = std::cell::Cell::new(std::time::Instant::now()); - let entries = scan::walk_stream(&roots, &|batch| { + scan::walk_stream(&roots, &expected, &|batch| { if last_emit.get().elapsed() < std::time::Duration::from_millis(500) { return; } last_emit.set(std::time::Instant::now()); emit_files(app, true, batch.len()); - }); + }) +} + +/// Incremental rescan: walk the roots, diff against the DB (insert +/// new / update changed / delete removed), refresh META, emit the list. +/// Used by the startup scan AND the live watcher — a new photo appears +/// in the grid seconds after it is written. +/// +/// Stale-listing (truncation) defense, three layers: +/// 1. per-directory cross-check during the walk (scan::walk_dir) — +/// re-reads any directory whose enumeration is far below the DB; +/// 2. whole-walk re-check below — if the TOTAL is still far below the +/// DB, re-walk once after a pause and keep the larger result; +/// 3. db::apply mass-delete guard — never delete >20% of the DB in a +/// single apply unless a previous walk already saw the low count. +fn rescan(app: &tauri::AppHandle) { + let mut entries = walk_once(app); + // Whole-walk re-check (layer 2). + let db_count = expected_count(); + let walked = entries.len() as i64; + if walked < db_count.saturating_sub(100) + && (db_count - walked) as f64 > (db_count as f64) * 0.2 + { + tr(&format!( + "rescan: walk {} << db {} — re-walk in 2s", + entries.len(), + db_count + )); + std::thread::sleep(std::time::Duration::from_secs(2)); + let again = walk_once(app); + if again.len() > entries.len() { + tr(&format!("rescan: re-walk recovered {} → {}", entries.len(), again.len())); + entries = again; + } + } let mut conn = match db::open(&db::db_path()) { Ok(c) => c, Err(e) => { @@ -768,8 +819,8 @@ fn dir_sig(roots: &[PathBuf]) -> HashMap { while let Some(dir) = stack.pop() { let Ok(m) = std::fs::metadata(&dir) else { continue }; out.insert(dir.clone(), mtime_ns(&m)); - if let Ok(rd) = std::fs::read_dir(&dir) { - for ent in rd.flatten() { + if let Some(entries) = scan::read_dir_full(&dir) { + for ent in entries { let p = ent.path(); if p.is_symlink() || !p.is_dir() { continue; @@ -818,7 +869,14 @@ fn rescan_subtrees(app: &tauri::AppHandle, changed: &[PathBuf]) { under.iter().map(|e| (e.path.as_str(), *e)).collect(); if under.len() <= BIG_DIR { // Small: full walk with stat — adds, deletes AND edits. - scan::walk_dir_into(d, &mut fresh); + // Stale-listing reference: per-directory counts from the DB. + let mut exp: HashMap = HashMap::new(); + for e in &under { + if let Some(p) = std::path::Path::new(&e.path).parent() { + *exp.entry(p.to_path_buf()).or_insert(0) += 1; + } + } + scan::walk_dir_into(d, &mut fresh, &exp); } else { // Huge: readdir-only. New files get stat'd; known files keep // their DB mtime/size (an in-place edit there is caught by @@ -862,48 +920,150 @@ fn rescan_subtrees(app: &tauri::AppHandle, changed: &[PathBuf]) { tr(&format!("watch: incr changed={} fresh={added}", changed.len())); } -/// Live file watching: poll the tree signature every second; when any -/// directory changed (or 30 s elapsed as a safety net in case an mtime -/// fails to update), debounce briefly so a write burst settles, then -/// run the incremental rescan. New photos appear in the grid within ~2 s -/// of being written; deleted/renamed ones are picked up on the next tick. +/// Watcher wake events. +#[derive(Debug, Clone, Copy)] +enum WEvent { + /// The OS said the filesystem changed (ContentObserver on Android, + /// inotify/FSEvents on desktop). + Fs, + /// Periodic full-pass timer. + Full, +} + +static WATCH_TX: LazyLock>>> = + LazyLock::new(|| Mutex::new(None)); + +/// Android: called by MainActivity's ContentObserver via JNI. +/// Signature: (JNIEnv*, jobject) for an instance method — both +/// ignored; the symbol name is what binds. +#[cfg(target_os = "android")] +#[no_mangle] +pub extern "system" fn Java_wow_gmp_gallery_MainActivity_nativeFsChange( + _env: *mut std::ffi::c_void, + _obj: *mut std::ffi::c_void, +) { + if let Some(tx) = WATCH_TX.lock().unwrap().as_ref() { + let _ = tx.send(WEvent::Fs); + } +} + +/// Desktop: inotify (Linux) / FSEvents (macOS) on the scan roots. +#[cfg(not(target_os = "android"))] +fn spawn_fs_watcher(tx: mpsc::Sender) { + std::thread::spawn(move || { + use notify::{RecursiveMode, Watcher}; + let (wt, wm) = mpsc::channel(); + let mut w = match notify::recommended_watcher(move |res: Result| { + if res.is_ok() { + let _ = wt.send(()); // consumer coalesces + diffs + } + }) { + Ok(w) => w, + Err(e) => { + log::warn!("fs watcher: {e}"); + return; + } + }; + for r in scan::roots() { + if r.is_dir() { + let _ = w.watch(&r, RecursiveMode::Recursive); + } + } + let _keep = wm; // drop = stop watching + loop { + if wt.recv().is_err() { + break; + } + let _ = tx.send(WEvent::Fs); + } + }); +} + +/// Diff two dir-mtime signatures: dirs added, removed, or changed. +fn changed_dirs( + old: &HashMap, + next: &HashMap, +) -> Vec { + let mut v: Vec = next + .iter() + .filter(|(d, t)| old.get(d.as_path()) != Some(t)) + .map(|(d, _)| d.clone()) + .collect(); + for d in old.keys() { + if !next.contains_key(d) { + v.push(d.clone()); + } + } + v +} + +/// Live file watching — EVENT-DRIVEN, no tree polling: +/// - Android: MainActivity's MediaStore ContentObserver → JNI → this +/// thread (camera saves, media-scanner indexing, indexed writes). +/// - Desktop: inotify/FSEvents via `notify` on the scan roots. +/// - On wake: coalesce the burst, debounce for the write to settle, +/// diff dir mtimes (a few stats), and rescan ONLY the changed +/// subtrees. A 5-minute full pass is the safety net (in-place edits +/// in the huge dir, anything the OS never reported). +/// - The 1 s recv timeout doubles as a cheap secondary detector +/// (dir-mtime diff, ~0.05 ms): raw out-of-band copies that the media +/// scanner hasn't indexed yet are still caught within ~1 s. fn watch_files(app: tauri::AppHandle) { + let (tx, rx) = mpsc::channel::(); + *WATCH_TX.lock().unwrap() = Some(tx.clone()); + std::thread::spawn(move || { + loop { + std::thread::sleep(std::time::Duration::from_secs(300)); + let _ = tx.send(WEvent::Full); + } + }); + #[cfg(not(target_os = "android"))] + spawn_fs_watcher(tx); let mut sig: HashMap = HashMap::new(); + let mut baseline = true; let mut last_full = std::time::Instant::now(); loop { - std::thread::sleep(std::time::Duration::from_secs(1)); - let roots = scan::roots(); - let next = dir_sig(&roots); - let changed: Vec = if sig.is_empty() { - sig = next; - Vec::new() // first pass: just establish the baseline - } else { - let mut v: Vec = Vec::new(); - for (d, t) in next.iter() { - if sig.get(d.as_path()) != Some(t) { - v.push(d.clone()); - } - } - for d in sig.keys() { - if !next.contains_key(d) { - v.push(d.clone()); // directory removed - } - } - sig = next; - v + let woke = match rx.recv_timeout(std::time::Duration::from_secs(1)) { + Ok(e) => Some(e), + Err(mpsc::RecvTimeoutError::Timeout) => None, + Err(mpsc::RecvTimeoutError::Disconnected) => break, }; - if changed.is_empty() { - // Safety net: in-place edits and anything the mtime poll - // missed. Full walk (~30 s on the phone) every 5 min. - if last_full.elapsed() >= std::time::Duration::from_secs(300) { - tr("watch: full rescan"); - rescan(&app); - last_full = std::time::Instant::now(); - } + let do_full = matches!(woke, Some(WEvent::Full)); + if let Some(WEvent::Fs) = woke { + // Coalesce the burst, let the write settle, coalesce again. + while rx.try_recv().is_ok() {} + std::thread::sleep(std::time::Duration::from_millis(600)); + while rx.try_recv().is_ok() {} + } + let next = dir_sig(&scan::roots()); + let changed = if baseline { + baseline = false; + Vec::new() + } else { + changed_dirs(&sig, &next) + }; + sig = next; + if do_full { + tr("watch: full rescan (5min)"); + rescan(&app); + last_full = std::time::Instant::now(); continue; } - // Let the write burst (file + temp + sidecars) settle. - std::thread::sleep(std::time::Duration::from_millis(600)); + if changed.is_empty() { + if woke.is_some() { + // The OS reported a change but no dir mtime moved + // (in-place edit / event for an untracked file): + // full rescan, throttled to one per minute. + if last_full.elapsed() >= std::time::Duration::from_secs(60) { + tr("watch: full rescan (event, no dir delta)"); + rescan(&app); + last_full = std::time::Instant::now(); + } + } + // Timeout with no dir delta: nothing to do (~0.05 ms spent). + continue; + } + tr(&format!("watch: {} changed dir(s)", changed.len())); rescan_subtrees(&app, &changed); } } diff --git a/harnesses/gtauri/src/scan.rs b/harnesses/gtauri/src/scan.rs index 273d245..2341b5e 100644 --- a/harnesses/gtauri/src/scan.rs +++ b/harnesses/gtauri/src/scan.rs @@ -4,6 +4,7 @@ //! what makes the app testable without MediaStore; MediaStore integration is //! a later refinement (HANDOFF.md §7.4). +use std::collections::HashMap; use std::path::{Path, PathBuf}; /// One image on disk. `mtime` is nanoseconds since the Unix epoch (i64). @@ -43,30 +44,54 @@ pub fn roots() -> Vec { /// Recursively collect all image entries below `roots`. /// `on_progress(n)` is called every 512 files found. pub fn walk(roots: &[PathBuf], on_progress: &dyn Fn(usize)) -> Vec { - walk_stream(roots, &|batch| on_progress(batch.len())) + walk_stream(roots, &HashMap::new(), &|batch| on_progress(batch.len())) } /// Like `walk`, but streams progress: `on_batch(&entries)` receives ALL /// entries collected so far every 512 new files, so the UI can show results /// while the scan is still running (first launch must not wait for the full /// DB build). -pub fn walk_stream(roots: &[PathBuf], on_batch: &dyn Fn(&[Entry])) -> Vec { +/// +/// `expected_files` is the per-directory file count from the previous DB +/// state (parent dir → count). When an enumeration returns far fewer +/// files than the DB holds for that directory, the VFS is almost certainly +/// serving a STALE cached directory listing (readdir returns a silent +/// early EOF — no error at all; reproduced on both the phone's FUSE and +/// the emulator's 9p /sdcard). In that case the directory is re-read up +/// to 3 times with delays, keeping the best listing. +pub fn walk_stream( + roots: &[PathBuf], + expected_files: &HashMap, + on_batch: &dyn Fn(&[Entry]), +) -> Vec { let mut out = Vec::new(); for r in roots { if r.is_dir() { - walk_dir(r, &mut out, on_batch); + walk_dir(r, &mut out, expected_files, on_batch); } } out } /// Full walk (with stat) of ONE directory subtree. -pub fn walk_dir_into(dir: &Path, out: &mut Vec) { +pub fn walk_dir_into(dir: &Path, out: &mut Vec, expected_files: &HashMap) { if dir.is_dir() { - walk_dir(dir, out, &|_| {}); + walk_dir(dir, out, expected_files, &|_| {}); } } +/// Count image-looking names in a listing (cheap: names only, no stat). +fn count_image_names(es: &[std::fs::DirEntry]) -> i64 { + es.iter().filter(|e| { + let name = e.file_name().to_string_lossy().into_owned(); + !name.starts_with('.') + && e.path() + .extension() + .and_then(|x| x.to_str()) + .is_some_and(|x| IMAGE_EXTS.contains(&x.to_ascii_lowercase().as_str())) + }).count() as i64 +} + /// Readdir-only walk (NO per-file stat) of ONE directory subtree — for /// huge FUSE directories where stat'ing every file costs ~30 s; the /// caller supplies metadata for new files and keeps DB metadata for @@ -77,8 +102,8 @@ pub fn walk_dir_nostats(dir: &Path, cb: &mut dyn FnMut(&Path)) { } let mut stack = vec![dir.to_path_buf()]; while let Some(d) = stack.pop() { - let Ok(rd) = std::fs::read_dir(&d) else { continue }; - for ent in rd.flatten() { + let Some(entries) = read_dir_full(&d) else { continue }; + for ent in entries { let p = ent.path(); let Ok(ft) = ent.file_type() else { continue }; if ft.is_dir() { @@ -96,9 +121,83 @@ pub fn walk_dir_nostats(dir: &Path, cb: &mut dyn FnMut(&Path)) { } } -fn walk_dir(dir: &Path, out: &mut Vec, on_batch: &dyn Fn(&[Entry])) { - let Ok(rd) = std::fs::read_dir(dir) else { return }; - for ent in rd.flatten() { +/// Read a directory to completion, retrying on transient mid-stream +/// errors. CRITICAL on Android FUSE: a std::fs::read_dir over a +/// thousands-of-entries directory latches the iterator into an error +/// state on ANY transient failure (verified on the phone: the 9961-file +/// Camera dir intermittently truncated at ~8400 entries, and the +/// truncated walk made db::apply delete ~1700 good rows). On a +/// mid-stream error the whole directory is re-read; after 3 failed +/// attempts the directory is skipped (the mass-delete guard in +/// db::apply is the backstop). +pub fn read_dir_full(dir: &Path) -> Option> { + for attempt in 0..3u32 { + let Ok(rd) = std::fs::read_dir(dir) else { return None }; + let mut entries = Vec::new(); + let mut ok = true; + for ent in rd { + match ent { + Ok(e) => entries.push(e), + Err(_) => { + ok = false; + break; + } + } + } + if ok { + return Some(entries); + } + std::thread::sleep(std::time::Duration::from_millis(200 * (attempt + 1) as u64)); + } + log::error!("read_dir {:?}: 3 mid-stream failures, skipping dir", dir); + None +} + +fn walk_dir( + dir: &Path, + out: &mut Vec, + expected_files: &HashMap, + on_batch: &dyn Fn(&[Entry]), +) { + let mut entries = match read_dir_full(dir) { + Some(e) => e, + None => return, + }; + // STALE-LISTING CROSS-CHECK (see walk_stream docs): if the DB holds + // many more files for this directory than the enumeration produced, + // re-read the directory a few times and keep the best listing. + if let Some(&want) = expected_files.get(dir) { + if want >= 50 { + let mut best_n = count_image_names(&entries); + if best_n * 5 < want * 4 { + for attempt in 1..=3u32 { + std::thread::sleep(std::time::Duration::from_millis(300 * u64::from(attempt))); + if let Some(again) = read_dir_full(dir) { + let n = count_image_names(&again); + if n > best_n { + best_n = n; + entries = again; + } + if best_n * 5 >= want * 4 { + break; + } + } + } + if best_n * 5 < want * 4 { + log::warn!( + "walk {:?}: enumerated {} image files, DB has {} (stale listing?)", + dir, + best_n, + want + ); + } else { + log::info!("walk {:?}: stale listing recovered {} → {}", + dir, want, best_n); + } + } + } + } + for ent in entries { let p = ent.path(); let Ok(ft) = ent.file_type() else { continue }; if ft.is_dir() { @@ -108,7 +207,7 @@ fn walk_dir(dir: &Path, out: &mut Vec, on_batch: &dyn Fn(&[Entry])) { } // Don't follow symlinked dirs (loop protection). if !p.is_symlink() { - walk_dir(&p, out, on_batch); + walk_dir(&p, out, expected_files, on_batch); } continue; } diff --git a/harnesses/gtauri/tauri.conf.json b/harnesses/gtauri/tauri.conf.json index 98dfc4b..190b1a2 100644 --- a/harnesses/gtauri/tauri.conf.json +++ b/harnesses/gtauri/tauri.conf.json @@ -28,6 +28,9 @@ "bundle": { "active": true, "targets": "all", + "android": { + "minSdkVersion": 30 + }, "icon": [ "icons/32x32.png", "icons/128x128.png",