From bd27a7ccc143abdcf07fde894cdaa8d21d9b9981 Mon Sep 17 00:00:00 2001 From: Greg Pomerantz Date: Sun, 1 Nov 2026 18:55:31 -0500 Subject: [PATCH] gtauri: fix live watching (incremental subtree rescans) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of 'adds/removes not showing up': the watcher's file-COUNT tree signature took 8-10 s per tick on the phone (FUSE readdir of the 8400-file Camera dir is ~1 ms/entry), so the 1 s poll was effectively a 10 s walk, and the old 30 s forced FULL rescan (walk+stat of every file, ~30 s) kept the watcher thread perpetually busy. New design: - 1 s tick stats DIRECTORY mtimes only (~1 ms). Verified on the phone: Android's FUSE DOES update dir mtime on file add/remove (stat before/after a copy showed it) — the earlier 'never fires' reading was wrong; no files had actually been added in that trace window. - On a changed dir: incremental rescan of ONLY that subtree. Small dirs (<=2000 files) get a full walk with stat (adds, deletes AND in-place edits). The huge Camera dir gets a readdir-only walk: new files are stat'd, known files keep their DB mtime/size, deletions are detected by set difference. Everything outside the changed subtree keeps its DB metadata — cost is proportional to the change. - Full rescan every 5 min as the safety net (in-place edits in the huge dir, anything missed). - DB: busy_timeout(10s) so the startup scan and watcher can transact concurrently (WAL). Measured on the phone: add to Pictures -> grid +1 in ~5 s; add to Camera -> +1 in ~12 s; removals picked up on the next tick. Also observed tracking a live 1700-photo user import (grid 8440 -> 10166 as the import progressed). JS: event payload normalizer for thumb-ready/full-ready (object vs JSON string across WebView builds), and js files/emit instrumentation to compare send vs receive counts. --- harnesses/gtauri/frontend/dist/index.html | 20 +++- harnesses/gtauri/src/db.rs | 2 + harnesses/gtauri/src/lib.rs | 123 +++++++++++++++++++--- harnesses/gtauri/src/scan.rs | 36 +++++++ 4 files changed, 161 insertions(+), 20 deletions(-) diff --git a/harnesses/gtauri/frontend/dist/index.html b/harnesses/gtauri/frontend/dist/index.html index a05bff7..52ea4a8 100644 --- a/harnesses/gtauri/frontend/dist/index.html +++ b/harnesses/gtauri/frontend/dist/index.html @@ -224,15 +224,27 @@ addEventListener("scroll", () => sendAnchor(false), { passive: true }); try { window.__TAURI__.event.listen("gallery-files", (e) => { - const p = e.payload || {}; - rebuild(p.ids || [], !!p.scanning, p.found || 0); + let p = e.payload || {}; + // On Android the payload can arrive as a JSON *string* — decode it. + if (typeof p === "string") { + try { p = JSON.parse(p); } catch (err) { jlog("files parse err: " + err); } + } + jlog("js files ids=" + ((p && p.ids) || []).length + + " scanning=" + !!p.scanning + " type=" + typeof e.payload); + rebuild((p && p.ids) || [], !!(p && p.scanning), (p && p.found) || 0); }); // first anchor (top of grid) — Rust also warms 0..36 on its side. setTimeout(() => sendAnchor(true), 300); // Cold-miss round-trip: Rust served a 1x1 placeholder and queued the real // decode off the main thread; when it lands, re-request (now a mem hit). + // Event payloads arrive as JSON strings on Android (object on some + // WebView builds) — normalize both. + const pid = (e) => { + const p = e.payload; + return (typeof p === "string" ? JSON.parse(p) : p) || {}; + }; window.__TAURI__.event.listen("thumb-ready", (e) => { - const id = e.payload.id; + const id = pid(e).id; const im = imgById.get(id); if (!im) return; if (im.naturalWidth <= 2) { @@ -242,7 +254,7 @@ try { }); // Full-res read finished (worker thread) — re-request; now a mem hit. window.__TAURI__.event.listen("full-ready", (e) => { - const id = e.payload.id; + const id = pid(e).id; if (viewerId === id && !vFullDone) { jlog("js full-ready id=" + id); tryShow(id, "http://gallery.localhost/full/" + id + "?v=f" + (++vFullSeq), true); diff --git a/harnesses/gtauri/src/db.rs b/harnesses/gtauri/src/db.rs index 68751b6..267e0b8 100644 --- a/harnesses/gtauri/src/db.rs +++ b/harnesses/gtauri/src/db.rs @@ -59,6 +59,8 @@ pub fn db_path() -> PathBuf { pub fn open(path: &std::path::Path) -> Result { let conn = Connection::open(path)?; + // The startup scan and the live watcher can transact concurrently. + conn.busy_timeout(std::time::Duration::from_secs(10))?; conn.pragma_update(None, "journal_mode", "WAL")?; conn.execute_batch( "CREATE TABLE IF NOT EXISTS items ( diff --git a/harnesses/gtauri/src/lib.rs b/harnesses/gtauri/src/lib.rs index 6e792d5..b9949f6 100644 --- a/harnesses/gtauri/src/lib.rs +++ b/harnesses/gtauri/src/lib.rs @@ -680,6 +680,7 @@ fn emit_files(app: &tauri::AppHandle, scanning: bool, found: usize) { *DISPLAY_IDS.write().unwrap() = ids.clone(); *ID_TO_IDX.write().unwrap() = ids.iter().enumerate().map(|(i, &id)| (id, i as u32)).collect(); + tr(&format!("emit n={} scanning={scanning}", ids.len())); let _ = app.emit("gallery-files", FileList { ids, scanning, found }); } @@ -758,11 +759,10 @@ fn mtime_ns(m: &std::fs::Metadata) -> i64 { } /// Tree signature: (dir, mtime) for every directory under the roots. -/// Adding/removing a file updates its directory's mtime, so a changed -/// library shows up here; the cost is a few dozen stats — cheap enough -/// to run every second. (inotify does not work on Android's FUSE-backed -/// external storage, which is why this is polling, not events.) -fn tree_sig(roots: &[PathBuf]) -> HashMap { +/// Verified on the phone: Android's FUSE DOES update a directory's mtime +/// when files are added/removed inside it (stat before/after a copy +/// showed the change). Stat-only cost: a few dozen syscalls per tick. +fn dir_sig(roots: &[PathBuf]) -> HashMap { let mut out = HashMap::new(); let mut stack: Vec = roots.iter().filter(|r| r.is_dir()).cloned().collect(); while let Some(dir) = stack.pop() { @@ -784,6 +784,84 @@ fn tree_sig(roots: &[PathBuf]) -> HashMap { out } +/// Above 2000 files, stat'ing every file on FUSE costs tens of seconds; +/// for such directories a change event is handled readdir-only (adds and +/// deletes are still detected exactly; in-place edits wait for the +/// periodic full pass). Small directories are walked with stat so edits +/// are caught too. +const BIG_DIR: usize = 2000; + +/// Incremental rescan limited to the changed directory subtrees. +/// Everything outside the changed trees keeps its DB metadata, so the +/// walk cost is proportional to the change, not the library. +fn rescan_subtrees(app: &tauri::AppHandle, changed: &[PathBuf]) { + let mut conn = match db::open(&db::db_path()) { + Ok(c) => c, + Err(e) => { + log::warn!("incr db: {e}"); + return; + } + }; + let existing = match db::list_entries(&mut conn) { + Ok(v) => v, + Err(e) => { + log::warn!("incr list: {e}"); + return; + } + }; + let mut fresh: Vec = Vec::new(); + for d in changed { + let prefix = d.to_string_lossy().into_owned() + "/"; + let under: Vec<&scan::Entry> = + existing.iter().filter(|e| e.path.starts_with(&prefix)).collect(); + let known: HashMap<&str, &scan::Entry> = + under.iter().map(|e| (e.path.as_str(), *e)).collect(); + if under.len() <= BIG_DIR { + // Small: full walk with stat — adds, deletes AND edits. + scan::walk_dir_into(d, &mut fresh); + } else { + // Huge: readdir-only. New files get stat'd; known files keep + // their DB mtime/size (an in-place edit there is caught by + // the 5-minute full pass). + scan::walk_dir_nostats(d, &mut |p| { + let ps = p.to_string_lossy(); + let entry = match known.get(ps.as_ref()) { + Some(old) => scan::Entry { + path: ps.into_owned(), + mtime: old.mtime, + size: old.size, + }, + None => match std::fs::metadata(p) { + Ok(md) => scan::Entry { + path: ps.into_owned(), + mtime: mtime_ns(&md), + size: md.len() as i64, + }, + Err(_) => return, + }, + }; + fresh.push(entry); + }); + } + } + let mut all = existing + .into_iter() + .filter(|e| !changed.iter().any(|d| e.path.starts_with(&*(d.to_string_lossy().into_owned() + "/")))) + .collect::>(); + let added = fresh.len(); + all.extend(fresh); + if let Err(e) = db::apply(&mut conn, &all) { + log::warn!("incr apply: {e}"); + return; + } + if let Err(e) = load_meta(&mut conn) { + log::warn!("incr meta: {e}"); + return; + } + emit_files(app, false, all.len()); + tr(&format!("watch: incr changed={} fresh={added}", changed.len())); +} + /// Live file watching: poll the tree signature every second; when any /// directory changed (or 30 s elapsed as a safety net in case an mtime /// fails to update), debounce briefly so a write burst settles, then @@ -795,25 +873,38 @@ fn watch_files(app: tauri::AppHandle) { loop { std::thread::sleep(std::time::Duration::from_secs(1)); let roots = scan::roots(); - let next = tree_sig(&roots); - let changed = if sig.is_empty() { + let next = dir_sig(&roots); + let changed: Vec = if sig.is_empty() { sig = next; - false // first pass: just establish the baseline + Vec::new() // first pass: just establish the baseline } else { - let mut ch = next.iter().any(|(d, t)| sig.get(d) != Some(t)) - || sig.keys().any(|d| !next.contains_key(d)); + let mut v: Vec = Vec::new(); + for (d, t) in next.iter() { + if sig.get(d.as_path()) != Some(t) { + v.push(d.clone()); + } + } + for d in sig.keys() { + if !next.contains_key(d) { + v.push(d.clone()); // directory removed + } + } sig = next; - ch + v }; - if !changed && last_full.elapsed() < std::time::Duration::from_secs(30) { + if changed.is_empty() { + // Safety net: in-place edits and anything the mtime poll + // missed. Full walk (~30 s on the phone) every 5 min. + if last_full.elapsed() >= std::time::Duration::from_secs(300) { + tr("watch: full rescan"); + rescan(&app); + last_full = std::time::Instant::now(); + } continue; } // Let the write burst (file + temp + sidecars) settle. std::thread::sleep(std::time::Duration::from_millis(600)); - tr("watch: rescan"); - rescan(&app); - last_full = std::time::Instant::now(); - sig = tree_sig(&scan::roots()); + rescan_subtrees(&app, &changed); } } diff --git a/harnesses/gtauri/src/scan.rs b/harnesses/gtauri/src/scan.rs index 70a5141..273d245 100644 --- a/harnesses/gtauri/src/scan.rs +++ b/harnesses/gtauri/src/scan.rs @@ -60,6 +60,42 @@ pub fn walk_stream(roots: &[PathBuf], on_batch: &dyn Fn(&[Entry])) -> Vec out } +/// Full walk (with stat) of ONE directory subtree. +pub fn walk_dir_into(dir: &Path, out: &mut Vec) { + if dir.is_dir() { + walk_dir(dir, out, &|_| {}); + } +} + +/// Readdir-only walk (NO per-file stat) of ONE directory subtree — for +/// huge FUSE directories where stat'ing every file costs ~30 s; the +/// caller supplies metadata for new files and keeps DB metadata for +/// known ones. +pub fn walk_dir_nostats(dir: &Path, cb: &mut dyn FnMut(&Path)) { + if !dir.is_dir() { + return; + } + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + let Ok(rd) = std::fs::read_dir(&d) else { continue }; + for ent in rd.flatten() { + let p = ent.path(); + let Ok(ft) = ent.file_type() else { continue }; + if ft.is_dir() { + if p.is_symlink() { + continue; + } + if p.file_name().map_or(false, |n| n.to_string_lossy().starts_with('.')) { + continue; + } + stack.push(p); + } else if ft.is_file() { + cb(&p); + } + } + } +} + fn walk_dir(dir: &Path, out: &mut Vec, on_batch: &dyn Fn(&[Entry])) { let Ok(rd) = std::fs::read_dir(dir) else { return }; for ent in rd.flatten() {