gtauri: event-driven watching + defense against stale VFS dir listings

Live watching is now event-driven, no tree polling:
- Android: MainActivity registers a MediaStore ContentObserver
  (Images, notifyForDescendants) and wakes the Rust watcher thread via
  a JNI call (Java_wow_gmp_gallery_MainActivity_nativeFsChange) — a
  single channel send, no main-thread work. Covers camera saves,
  media-scanner indexing, other apps' indexed writes.
- Desktop: inotify/FSEvents via notify on the scan roots.
- On wake: coalesce + debounce, diff dir mtimes, rescan only the
  changed subtrees. The 1 s recv timeout doubles as a cheap backstop
  (raw out-of-band copies the scanner hasn't indexed yet); a 5-minute
  full pass is the safety net. minSdk 30 (3-arg ContentObserver API).

Stale-listing truncation bug found and fixed (reproduced on the
emulator's 9p /sdcard; the phone's FUSE does the same): a readdir over
a large directory can return a SILENT early EOF — a stale cached
listing with NO error. The phone's 9961-file Camera dir truncated at
~8400 once and db::apply deleted ~1700 good rows. Three layers now:
1. scan::read_dir_full — retry whole-dir reads on mid-stream errors;
2. per-directory cross-check in walk_dir: if the enumeration is far
   below the DB's count for that dir, re-read up to 3x keeping the
   best listing; whole-walk re-check in rescan keeps the larger of two
   walks;
3. db::apply mass-delete guard: never delete >20% of the DB in one
   apply unless a previous walk already saw the low count (two-strike;
   a meta.walked row tracks it). Genuine gradual deletions still flow
   through (verified: a 4000-file mass move streams the grid down
   progressively), and a second consecutive low walk still deletes.

Emulator reliability suite (all pass): 3x launch count stability,
add +1 / remove -1 in ~2 s, 20-file burst, DB recovery from damage,
stale-listing refusal (0 rows lost), two-strike delete, live restore,
idle 0.0% dropped frames, deep fling loads thumbnails. A/B vs ttimg
(no-backend CSS-scroll control): adb-swipe drop deltas are at the
input-injection artifact level, idle identical (0.0%).
This commit is contained in:
Greg Pomerantz 2026-11-01 20:24:01 -05:00
parent bd27a7ccc1
commit 58b203e4fa
8 changed files with 511 additions and 60 deletions

View File

@ -1215,6 +1215,7 @@ dependencies = [
"image", "image",
"kamadak-exif", "kamadak-exif",
"log", "log",
"notify",
"rusqlite", "rusqlite",
"serde", "serde",
"serde_json", "serde_json",
@ -1622,6 +1623,26 @@ dependencies = [
"cfb", "cfb",
] ]
[[package]]
name = "inotify"
version = "0.11.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cc00ea907cab49550b7da656f80ebb97be1b997d931fbcd28d39734e17ce592"
dependencies = [
"bitflags 2.13.2",
"inotify-sys",
"libc",
]
[[package]]
name = "inotify-sys"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c033f80b2c113cdf91ab7a33faa9cbc014726dcad99880c8609af2a370edf37d"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "ipnet" name = "ipnet"
version = "2.12.2" version = "2.12.2"
@ -1807,6 +1828,26 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "kqueue"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8d763e5b24120b4ddf50de6c92308156765aabfbbccebf401da7cff2d70a41ea"
dependencies = [
"kqueue-sys",
"libc",
]
[[package]]
name = "kqueue-sys"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087"
dependencies = [
"bitflags 2.13.2",
"libc",
]
[[package]] [[package]]
name = "libappindicator" name = "libappindicator"
version = "0.9.0" version = "0.9.0"
@ -1956,6 +1997,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee"
dependencies = [ dependencies = [
"libc", "libc",
"log",
"wasi", "wasi",
"windows-sys 0.61.2", "windows-sys 0.61.2",
] ]
@ -2033,6 +2075,34 @@ version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
[[package]]
name = "notify"
version = "8.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82f812702fbc75d35416d3f884e04925be7e9498f4eceea2e24ba34559284674"
dependencies = [
"bitflags 2.13.2",
"inotify",
"kqueue",
"libc",
"log",
"mio",
"notify-types",
"objc2-core-foundation",
"objc2-core-services",
"walkdir",
"windows-sys 0.61.2",
]
[[package]]
name = "notify-types"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a437a9308232ce794b1e7fdf540d38ea37efc3c73492640c13b4c8568b8f108"
dependencies = [
"bitflags 2.13.2",
]
[[package]] [[package]]
name = "num-conv" name = "num-conv"
version = "0.2.2" version = "0.2.2"
@ -2159,6 +2229,16 @@ dependencies = [
"objc2-foundation", "objc2-foundation",
] ]
[[package]]
name = "objc2-core-services"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "583300ad934cba24ff5292aee751ecc070f7ca6b39a574cc21b7b5e588e06a0b"
dependencies = [
"libc",
"objc2-core-foundation",
]
[[package]] [[package]]
name = "objc2-core-text" name = "objc2-core-text"
version = "0.3.2" version = "0.3.2"

View File

@ -25,3 +25,8 @@ turbojpeg = { version = "1.5.1", features = ["cmake"] }
kamadak-exif = "0.6" kamadak-exif = "0.6"
sha1 = "0.11" sha1 = "0.11"
image = { version = "0.25", default-features = false, features = ["png", "webp", "gif"] } image = { version = "0.25", default-features = false, features = ["png", "webp", "gif"] }
# Desktop file watching (inotify on Linux, FSEvents on macOS).
# Android uses the MediaStore ContentObserver instead — see MainActivity.kt.
[target.'cfg(not(target_os = "android"))'.dependencies]
notify = "8"

View File

@ -20,7 +20,7 @@ android {
defaultConfig { defaultConfig {
manifestPlaceholders["usesCleartextTraffic"] = "false" manifestPlaceholders["usesCleartextTraffic"] = "false"
applicationId = "wow.gmp.gallery" applicationId = "wow.gmp.gallery"
minSdk = 24 minSdk = 30
targetSdk = 37 targetSdk = 37
versionCode = tauriProperties.getProperty("tauri.android.versionCode", "1").toInt() versionCode = tauriProperties.getProperty("tauri.android.versionCode", "1").toInt()
versionName = tauriProperties.getProperty("tauri.android.versionName", "1.0") versionName = tauriProperties.getProperty("tauri.android.versionName", "1.0")

View File

@ -1,11 +1,44 @@
package wow.gmp.gallery package wow.gmp.gallery
import android.database.ContentObserver
import android.net.Uri
import android.os.Bundle import android.os.Bundle
import android.os.Handler
import android.os.Looper
import android.provider.MediaStore
import androidx.activity.enableEdgeToEdge import androidx.activity.enableEdgeToEdge
class MainActivity : TauriActivity() { class MainActivity : TauriActivity() {
private var observer: ContentObserver? = null
// Rust side (this app's JNI library, loaded by TauriActivity): wakes
// the file-watcher thread. One JNI call + one channel send — nothing
// heavy ever runs on the main thread.
external fun nativeFsChange()
override fun onCreate(savedInstanceState: Bundle?) { override fun onCreate(savedInstanceState: Bundle?) {
enableEdgeToEdge() enableEdgeToEdge()
super.onCreate(savedInstanceState) super.onCreate(savedInstanceState)
// Event-driven file watching (no polling): MediaStore changes —
// camera saves, media-scanner indexing, other apps' indexed writes —
// wake the Rust watcher, which diffs dir mtimes and does an
// incremental rescan of only the changed subtrees.
val obs = object : ContentObserver(Handler(Looper.getMainLooper())) {
override fun onChange(selfChange: Boolean, uri: Uri?) {
nativeFsChange()
}
}
observer = obs
// 3-arg overload (notifyForDescendants) requires API 30 — the app's
// minSdk (see tauri.conf.json bundle.android.minSdkVersion).
contentResolver.registerContentObserver(
MediaStore.Images.Media.EXTERNAL_CONTENT_URI, true, obs
)
}
override fun onDestroy() {
observer?.let { contentResolver.unregisterContentObserver(it) }
super.onDestroy()
} }
} }

View File

@ -73,7 +73,11 @@ pub fn open(path: &std::path::Path) -> Result<Connection, rusqlite::Error> {
thumb_path TEXT, thumb_path TEXT,
date_taken INTEGER date_taken INTEGER
); );
CREATE INDEX IF NOT EXISTS items_mtime ON items (mtime DESC);", CREATE INDEX IF NOT EXISTS items_mtime ON items (mtime DESC);
CREATE TABLE IF NOT EXISTS meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);",
)?; )?;
Ok(conn) Ok(conn)
} }
@ -132,10 +136,56 @@ pub fn apply(
} }
} }
} }
// MASS-DELETE GUARD: if this walk sees dramatically fewer files
// than the DB holds, the walk almost certainly truncated (FUSE
// mid-stream readdir error — see scan::read_dir_full) rather than
// a real mass deletion. Refuse to delete on the FIRST suspicious
// walk; a second consecutive low walk confirms it is real.
let mut to_delete = 0;
for p in existing.keys() { for p in existing.keys() {
if !seen.contains(p) { if !seen.contains(p) {
del.execute(params![p])?; to_delete += 1;
deleted += 1; }
}
let suspect = existing.len() > 100
&& (to_delete as f64) > (existing.len() as f64) * 0.2;
let mut do_delete = true;
if suspect {
let last_walked: i64 = tx
.query_row("SELECT value FROM meta WHERE key='walked'", [], |r| {
r.get::<_, String>(0)
})
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(existing.len() as i64);
if last_walked >= (existing.len() as i64) * 80 / 100 {
log::warn!(
"apply: walk saw {} but DB has {} (last walk {}): refusing to delete {} rows (truncated walk?)",
entries.len(),
existing.len(),
last_walked,
to_delete
);
do_delete = false;
}
let _ = tx.execute(
"INSERT INTO meta (key, value) VALUES ('walked', ?1) \
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
params![entries.len().to_string()],
);
} else {
let _ = tx.execute(
"INSERT INTO meta (key, value) VALUES ('walked', ?1) \
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
params![entries.len().to_string()],
);
}
if do_delete {
for p in existing.keys() {
if !seen.contains(p) {
del.execute(params![p])?;
deleted += 1;
}
} }
} }
} }
@ -143,6 +193,27 @@ pub fn apply(
Ok((inserted, updated, deleted)) Ok((inserted, updated, deleted))
} }
/// Number of rows in the DB.
pub fn count(conn: &mut Connection) -> Result<i64, rusqlite::Error> {
let n: i64 = conn.query_row("SELECT COUNT(*) FROM items", [], |r| r.get(0))?;
Ok(n)
}
/// Per-directory file counts (parent dir → row count) — the reference
/// the walk uses to detect stale VFS directory listings.
pub fn dir_counts(conn: &mut Connection) -> Result<HashMap<PathBuf, i64>, rusqlite::Error> {
let mut st = conn.prepare("SELECT path FROM items")?;
let rows = st.query_map([], |r| r.get::<_, String>(0))?;
let mut m: HashMap<PathBuf, i64> = HashMap::new();
for p in rows {
let p = p?;
if let Some(d) = std::path::Path::new(&p).parent() {
*m.entry(d.to_path_buf()).or_insert(0) += 1;
}
}
Ok(m)
}
/// All known entries, newest first. /// All known entries, newest first.
pub fn list_entries(conn: &mut Connection) -> Result<Vec<Entry>, rusqlite::Error> { pub fn list_entries(conn: &mut Connection) -> Result<Vec<Entry>, rusqlite::Error> {
let mut st = conn.prepare("SELECT path, mtime, size FROM items ORDER BY mtime DESC, path ASC")?; let mut st = conn.prepare("SELECT path, mtime, size FROM items ORDER BY mtime DESC, path ASC")?;

View File

@ -713,20 +713,71 @@ fn load_meta(conn: &mut db::Connection) -> Result<(), rusqlite::Error> {
Ok(()) Ok(())
} }
/// Incremental rescan: walk the roots, diff against the DB (insert /// Total rows in the DB (0 on error).
/// new / update changed / delete removed), refresh META, emit the list. fn expected_count() -> i64 {
/// Used by the startup scan AND the live watcher — a new photo appears match db::open(&db::db_path()) {
/// in the grid seconds after it is written. Ok(mut c) => db::count(&mut c).unwrap_or(0),
fn rescan(app: &tauri::AppHandle) { Err(_) => 0,
}
}
/// Per-directory file counts from the DB — the reference the walk uses
/// to detect stale (silently truncated) VFS directory listings.
fn expected_dir_counts() -> std::collections::HashMap<std::path::PathBuf, i64> {
match db::open(&db::db_path()) {
Ok(mut c) => db::dir_counts(&mut c).unwrap_or_default(),
Err(_) => std::collections::HashMap::new(),
}
}
/// Walk the roots once, streaming progress emits. Cross-checks each
/// directory's enumeration against the previous DB counts (stale-listing
/// protection — see scan::walk_stream).
fn walk_once(app: &tauri::AppHandle) -> Vec<scan::Entry> {
let expected = expected_dir_counts();
let roots = scan::roots(); let roots = scan::roots();
let last_emit = std::cell::Cell::new(std::time::Instant::now()); let last_emit = std::cell::Cell::new(std::time::Instant::now());
let entries = scan::walk_stream(&roots, &|batch| { scan::walk_stream(&roots, &expected, &|batch| {
if last_emit.get().elapsed() < std::time::Duration::from_millis(500) { if last_emit.get().elapsed() < std::time::Duration::from_millis(500) {
return; return;
} }
last_emit.set(std::time::Instant::now()); last_emit.set(std::time::Instant::now());
emit_files(app, true, batch.len()); emit_files(app, true, batch.len());
}); })
}
/// Incremental rescan: walk the roots, diff against the DB (insert
/// new / update changed / delete removed), refresh META, emit the list.
/// Used by the startup scan AND the live watcher — a new photo appears
/// in the grid seconds after it is written.
///
/// Stale-listing (truncation) defense, three layers:
/// 1. per-directory cross-check during the walk (scan::walk_dir) —
/// re-reads any directory whose enumeration is far below the DB;
/// 2. whole-walk re-check below — if the TOTAL is still far below the
/// DB, re-walk once after a pause and keep the larger result;
/// 3. db::apply mass-delete guard — never delete >20% of the DB in a
/// single apply unless a previous walk already saw the low count.
fn rescan(app: &tauri::AppHandle) {
let mut entries = walk_once(app);
// Whole-walk re-check (layer 2).
let db_count = expected_count();
let walked = entries.len() as i64;
if walked < db_count.saturating_sub(100)
&& (db_count - walked) as f64 > (db_count as f64) * 0.2
{
tr(&format!(
"rescan: walk {} << db {} — re-walk in 2s",
entries.len(),
db_count
));
std::thread::sleep(std::time::Duration::from_secs(2));
let again = walk_once(app);
if again.len() > entries.len() {
tr(&format!("rescan: re-walk recovered {} → {}", entries.len(), again.len()));
entries = again;
}
}
let mut conn = match db::open(&db::db_path()) { let mut conn = match db::open(&db::db_path()) {
Ok(c) => c, Ok(c) => c,
Err(e) => { Err(e) => {
@ -768,8 +819,8 @@ fn dir_sig(roots: &[PathBuf]) -> HashMap<PathBuf, i64> {
while let Some(dir) = stack.pop() { while let Some(dir) = stack.pop() {
let Ok(m) = std::fs::metadata(&dir) else { continue }; let Ok(m) = std::fs::metadata(&dir) else { continue };
out.insert(dir.clone(), mtime_ns(&m)); out.insert(dir.clone(), mtime_ns(&m));
if let Ok(rd) = std::fs::read_dir(&dir) { if let Some(entries) = scan::read_dir_full(&dir) {
for ent in rd.flatten() { for ent in entries {
let p = ent.path(); let p = ent.path();
if p.is_symlink() || !p.is_dir() { if p.is_symlink() || !p.is_dir() {
continue; continue;
@ -818,7 +869,14 @@ fn rescan_subtrees(app: &tauri::AppHandle, changed: &[PathBuf]) {
under.iter().map(|e| (e.path.as_str(), *e)).collect(); under.iter().map(|e| (e.path.as_str(), *e)).collect();
if under.len() <= BIG_DIR { if under.len() <= BIG_DIR {
// Small: full walk with stat — adds, deletes AND edits. // Small: full walk with stat — adds, deletes AND edits.
scan::walk_dir_into(d, &mut fresh); // Stale-listing reference: per-directory counts from the DB.
let mut exp: HashMap<std::path::PathBuf, i64> = HashMap::new();
for e in &under {
if let Some(p) = std::path::Path::new(&e.path).parent() {
*exp.entry(p.to_path_buf()).or_insert(0) += 1;
}
}
scan::walk_dir_into(d, &mut fresh, &exp);
} else { } else {
// Huge: readdir-only. New files get stat'd; known files keep // Huge: readdir-only. New files get stat'd; known files keep
// their DB mtime/size (an in-place edit there is caught by // their DB mtime/size (an in-place edit there is caught by
@ -862,48 +920,150 @@ fn rescan_subtrees(app: &tauri::AppHandle, changed: &[PathBuf]) {
tr(&format!("watch: incr changed={} fresh={added}", changed.len())); tr(&format!("watch: incr changed={} fresh={added}", changed.len()));
} }
/// Live file watching: poll the tree signature every second; when any /// Watcher wake events.
/// directory changed (or 30 s elapsed as a safety net in case an mtime #[derive(Debug, Clone, Copy)]
/// fails to update), debounce briefly so a write burst settles, then enum WEvent {
/// run the incremental rescan. New photos appear in the grid within ~2 s /// The OS said the filesystem changed (ContentObserver on Android,
/// of being written; deleted/renamed ones are picked up on the next tick. /// inotify/FSEvents on desktop).
Fs,
/// Periodic full-pass timer.
Full,
}
static WATCH_TX: LazyLock<Mutex<Option<mpsc::Sender<WEvent>>>> =
LazyLock::new(|| Mutex::new(None));
/// Android: called by MainActivity's ContentObserver via JNI.
/// Signature: (JNIEnv*, jobject) for an instance method — both
/// ignored; the symbol name is what binds.
#[cfg(target_os = "android")]
#[no_mangle]
pub extern "system" fn Java_wow_gmp_gallery_MainActivity_nativeFsChange(
_env: *mut std::ffi::c_void,
_obj: *mut std::ffi::c_void,
) {
if let Some(tx) = WATCH_TX.lock().unwrap().as_ref() {
let _ = tx.send(WEvent::Fs);
}
}
/// Desktop: inotify (Linux) / FSEvents (macOS) on the scan roots.
#[cfg(not(target_os = "android"))]
fn spawn_fs_watcher(tx: mpsc::Sender<WEvent>) {
std::thread::spawn(move || {
use notify::{RecursiveMode, Watcher};
let (wt, wm) = mpsc::channel();
let mut w = match notify::recommended_watcher(move |res: Result<notify::Event, _>| {
if res.is_ok() {
let _ = wt.send(()); // consumer coalesces + diffs
}
}) {
Ok(w) => w,
Err(e) => {
log::warn!("fs watcher: {e}");
return;
}
};
for r in scan::roots() {
if r.is_dir() {
let _ = w.watch(&r, RecursiveMode::Recursive);
}
}
let _keep = wm; // drop = stop watching
loop {
if wt.recv().is_err() {
break;
}
let _ = tx.send(WEvent::Fs);
}
});
}
/// Diff two dir-mtime signatures: dirs added, removed, or changed.
fn changed_dirs(
old: &HashMap<PathBuf, i64>,
next: &HashMap<PathBuf, i64>,
) -> Vec<PathBuf> {
let mut v: Vec<PathBuf> = next
.iter()
.filter(|(d, t)| old.get(d.as_path()) != Some(t))
.map(|(d, _)| d.clone())
.collect();
for d in old.keys() {
if !next.contains_key(d) {
v.push(d.clone());
}
}
v
}
/// Live file watching — EVENT-DRIVEN, no tree polling:
/// - Android: MainActivity's MediaStore ContentObserver → JNI → this
/// thread (camera saves, media-scanner indexing, indexed writes).
/// - Desktop: inotify/FSEvents via `notify` on the scan roots.
/// - On wake: coalesce the burst, debounce for the write to settle,
/// diff dir mtimes (a few stats), and rescan ONLY the changed
/// subtrees. A 5-minute full pass is the safety net (in-place edits
/// in the huge dir, anything the OS never reported).
/// - The 1 s recv timeout doubles as a cheap secondary detector
/// (dir-mtime diff, ~0.05 ms): raw out-of-band copies that the media
/// scanner hasn't indexed yet are still caught within ~1 s.
fn watch_files(app: tauri::AppHandle) { fn watch_files(app: tauri::AppHandle) {
let (tx, rx) = mpsc::channel::<WEvent>();
*WATCH_TX.lock().unwrap() = Some(tx.clone());
std::thread::spawn(move || {
loop {
std::thread::sleep(std::time::Duration::from_secs(300));
let _ = tx.send(WEvent::Full);
}
});
#[cfg(not(target_os = "android"))]
spawn_fs_watcher(tx);
let mut sig: HashMap<PathBuf, i64> = HashMap::new(); let mut sig: HashMap<PathBuf, i64> = HashMap::new();
let mut baseline = true;
let mut last_full = std::time::Instant::now(); let mut last_full = std::time::Instant::now();
loop { loop {
std::thread::sleep(std::time::Duration::from_secs(1)); let woke = match rx.recv_timeout(std::time::Duration::from_secs(1)) {
let roots = scan::roots(); Ok(e) => Some(e),
let next = dir_sig(&roots); Err(mpsc::RecvTimeoutError::Timeout) => None,
let changed: Vec<PathBuf> = if sig.is_empty() { Err(mpsc::RecvTimeoutError::Disconnected) => break,
sig = next;
Vec::new() // first pass: just establish the baseline
} else {
let mut v: Vec<PathBuf> = Vec::new();
for (d, t) in next.iter() {
if sig.get(d.as_path()) != Some(t) {
v.push(d.clone());
}
}
for d in sig.keys() {
if !next.contains_key(d) {
v.push(d.clone()); // directory removed
}
}
sig = next;
v
}; };
if changed.is_empty() { let do_full = matches!(woke, Some(WEvent::Full));
// Safety net: in-place edits and anything the mtime poll if let Some(WEvent::Fs) = woke {
// missed. Full walk (~30 s on the phone) every 5 min. // Coalesce the burst, let the write settle, coalesce again.
if last_full.elapsed() >= std::time::Duration::from_secs(300) { while rx.try_recv().is_ok() {}
tr("watch: full rescan"); std::thread::sleep(std::time::Duration::from_millis(600));
rescan(&app); while rx.try_recv().is_ok() {}
last_full = std::time::Instant::now(); }
} let next = dir_sig(&scan::roots());
let changed = if baseline {
baseline = false;
Vec::new()
} else {
changed_dirs(&sig, &next)
};
sig = next;
if do_full {
tr("watch: full rescan (5min)");
rescan(&app);
last_full = std::time::Instant::now();
continue; continue;
} }
// Let the write burst (file + temp + sidecars) settle. if changed.is_empty() {
std::thread::sleep(std::time::Duration::from_millis(600)); if woke.is_some() {
// The OS reported a change but no dir mtime moved
// (in-place edit / event for an untracked file):
// full rescan, throttled to one per minute.
if last_full.elapsed() >= std::time::Duration::from_secs(60) {
tr("watch: full rescan (event, no dir delta)");
rescan(&app);
last_full = std::time::Instant::now();
}
}
// Timeout with no dir delta: nothing to do (~0.05 ms spent).
continue;
}
tr(&format!("watch: {} changed dir(s)", changed.len()));
rescan_subtrees(&app, &changed); rescan_subtrees(&app, &changed);
} }
} }

View File

@ -4,6 +4,7 @@
//! what makes the app testable without MediaStore; MediaStore integration is //! what makes the app testable without MediaStore; MediaStore integration is
//! a later refinement (HANDOFF.md §7.4). //! a later refinement (HANDOFF.md §7.4).
use std::collections::HashMap;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
/// One image on disk. `mtime` is nanoseconds since the Unix epoch (i64). /// One image on disk. `mtime` is nanoseconds since the Unix epoch (i64).
@ -43,30 +44,54 @@ pub fn roots() -> Vec<PathBuf> {
/// Recursively collect all image entries below `roots`. /// Recursively collect all image entries below `roots`.
/// `on_progress(n)` is called every 512 files found. /// `on_progress(n)` is called every 512 files found.
pub fn walk(roots: &[PathBuf], on_progress: &dyn Fn(usize)) -> Vec<Entry> { pub fn walk(roots: &[PathBuf], on_progress: &dyn Fn(usize)) -> Vec<Entry> {
walk_stream(roots, &|batch| on_progress(batch.len())) walk_stream(roots, &HashMap::new(), &|batch| on_progress(batch.len()))
} }
/// Like `walk`, but streams progress: `on_batch(&entries)` receives ALL /// Like `walk`, but streams progress: `on_batch(&entries)` receives ALL
/// entries collected so far every 512 new files, so the UI can show results /// entries collected so far every 512 new files, so the UI can show results
/// while the scan is still running (first launch must not wait for the full /// while the scan is still running (first launch must not wait for the full
/// DB build). /// DB build).
pub fn walk_stream(roots: &[PathBuf], on_batch: &dyn Fn(&[Entry])) -> Vec<Entry> { ///
/// `expected_files` is the per-directory file count from the previous DB
/// state (parent dir → count). When an enumeration returns far fewer
/// files than the DB holds for that directory, the VFS is almost certainly
/// serving a STALE cached directory listing (readdir returns a silent
/// early EOF — no error at all; reproduced on both the phone's FUSE and
/// the emulator's 9p /sdcard). In that case the directory is re-read up
/// to 3 times with delays, keeping the best listing.
pub fn walk_stream(
roots: &[PathBuf],
expected_files: &HashMap<PathBuf, i64>,
on_batch: &dyn Fn(&[Entry]),
) -> Vec<Entry> {
let mut out = Vec::new(); let mut out = Vec::new();
for r in roots { for r in roots {
if r.is_dir() { if r.is_dir() {
walk_dir(r, &mut out, on_batch); walk_dir(r, &mut out, expected_files, on_batch);
} }
} }
out out
} }
/// Full walk (with stat) of ONE directory subtree. /// Full walk (with stat) of ONE directory subtree.
pub fn walk_dir_into(dir: &Path, out: &mut Vec<Entry>) { pub fn walk_dir_into(dir: &Path, out: &mut Vec<Entry>, expected_files: &HashMap<PathBuf, i64>) {
if dir.is_dir() { if dir.is_dir() {
walk_dir(dir, out, &|_| {}); walk_dir(dir, out, expected_files, &|_| {});
} }
} }
/// Count image-looking names in a listing (cheap: names only, no stat).
fn count_image_names(es: &[std::fs::DirEntry]) -> i64 {
es.iter().filter(|e| {
let name = e.file_name().to_string_lossy().into_owned();
!name.starts_with('.')
&& e.path()
.extension()
.and_then(|x| x.to_str())
.is_some_and(|x| IMAGE_EXTS.contains(&x.to_ascii_lowercase().as_str()))
}).count() as i64
}
/// Readdir-only walk (NO per-file stat) of ONE directory subtree — for /// Readdir-only walk (NO per-file stat) of ONE directory subtree — for
/// huge FUSE directories where stat'ing every file costs ~30 s; the /// huge FUSE directories where stat'ing every file costs ~30 s; the
/// caller supplies metadata for new files and keeps DB metadata for /// caller supplies metadata for new files and keeps DB metadata for
@ -77,8 +102,8 @@ pub fn walk_dir_nostats(dir: &Path, cb: &mut dyn FnMut(&Path)) {
} }
let mut stack = vec![dir.to_path_buf()]; let mut stack = vec![dir.to_path_buf()];
while let Some(d) = stack.pop() { while let Some(d) = stack.pop() {
let Ok(rd) = std::fs::read_dir(&d) else { continue }; let Some(entries) = read_dir_full(&d) else { continue };
for ent in rd.flatten() { for ent in entries {
let p = ent.path(); let p = ent.path();
let Ok(ft) = ent.file_type() else { continue }; let Ok(ft) = ent.file_type() else { continue };
if ft.is_dir() { if ft.is_dir() {
@ -96,9 +121,83 @@ pub fn walk_dir_nostats(dir: &Path, cb: &mut dyn FnMut(&Path)) {
} }
} }
fn walk_dir(dir: &Path, out: &mut Vec<Entry>, on_batch: &dyn Fn(&[Entry])) { /// Read a directory to completion, retrying on transient mid-stream
let Ok(rd) = std::fs::read_dir(dir) else { return }; /// errors. CRITICAL on Android FUSE: a std::fs::read_dir over a
for ent in rd.flatten() { /// thousands-of-entries directory latches the iterator into an error
/// state on ANY transient failure (verified on the phone: the 9961-file
/// Camera dir intermittently truncated at ~8400 entries, and the
/// truncated walk made db::apply delete ~1700 good rows). On a
/// mid-stream error the whole directory is re-read; after 3 failed
/// attempts the directory is skipped (the mass-delete guard in
/// db::apply is the backstop).
pub fn read_dir_full(dir: &Path) -> Option<Vec<std::fs::DirEntry>> {
for attempt in 0..3u32 {
let Ok(rd) = std::fs::read_dir(dir) else { return None };
let mut entries = Vec::new();
let mut ok = true;
for ent in rd {
match ent {
Ok(e) => entries.push(e),
Err(_) => {
ok = false;
break;
}
}
}
if ok {
return Some(entries);
}
std::thread::sleep(std::time::Duration::from_millis(200 * (attempt + 1) as u64));
}
log::error!("read_dir {:?}: 3 mid-stream failures, skipping dir", dir);
None
}
fn walk_dir(
dir: &Path,
out: &mut Vec<Entry>,
expected_files: &HashMap<PathBuf, i64>,
on_batch: &dyn Fn(&[Entry]),
) {
let mut entries = match read_dir_full(dir) {
Some(e) => e,
None => return,
};
// STALE-LISTING CROSS-CHECK (see walk_stream docs): if the DB holds
// many more files for this directory than the enumeration produced,
// re-read the directory a few times and keep the best listing.
if let Some(&want) = expected_files.get(dir) {
if want >= 50 {
let mut best_n = count_image_names(&entries);
if best_n * 5 < want * 4 {
for attempt in 1..=3u32 {
std::thread::sleep(std::time::Duration::from_millis(300 * u64::from(attempt)));
if let Some(again) = read_dir_full(dir) {
let n = count_image_names(&again);
if n > best_n {
best_n = n;
entries = again;
}
if best_n * 5 >= want * 4 {
break;
}
}
}
if best_n * 5 < want * 4 {
log::warn!(
"walk {:?}: enumerated {} image files, DB has {} (stale listing?)",
dir,
best_n,
want
);
} else {
log::info!("walk {:?}: stale listing recovered {} → {}",
dir, want, best_n);
}
}
}
}
for ent in entries {
let p = ent.path(); let p = ent.path();
let Ok(ft) = ent.file_type() else { continue }; let Ok(ft) = ent.file_type() else { continue };
if ft.is_dir() { if ft.is_dir() {
@ -108,7 +207,7 @@ fn walk_dir(dir: &Path, out: &mut Vec<Entry>, on_batch: &dyn Fn(&[Entry])) {
} }
// Don't follow symlinked dirs (loop protection). // Don't follow symlinked dirs (loop protection).
if !p.is_symlink() { if !p.is_symlink() {
walk_dir(&p, out, on_batch); walk_dir(&p, out, expected_files, on_batch);
} }
continue; continue;
} }

View File

@ -28,6 +28,9 @@
"bundle": { "bundle": {
"active": true, "active": true,
"targets": "all", "targets": "all",
"android": {
"minSdkVersion": 30
},
"icon": [ "icon": [
"icons/32x32.png", "icons/32x32.png",
"icons/128x128.png", "icons/128x128.png",