gtauri: event-driven watching + defense against stale VFS dir listings
Live watching is now event-driven, no tree polling: - Android: MainActivity registers a MediaStore ContentObserver (Images, notifyForDescendants) and wakes the Rust watcher thread via a JNI call (Java_wow_gmp_gallery_MainActivity_nativeFsChange) — a single channel send, no main-thread work. Covers camera saves, media-scanner indexing, other apps' indexed writes. - Desktop: inotify/FSEvents via notify on the scan roots. - On wake: coalesce + debounce, diff dir mtimes, rescan only the changed subtrees. The 1 s recv timeout doubles as a cheap backstop (raw out-of-band copies the scanner hasn't indexed yet); a 5-minute full pass is the safety net. minSdk 30 (3-arg ContentObserver API). Stale-listing truncation bug found and fixed (reproduced on the emulator's 9p /sdcard; the phone's FUSE does the same): a readdir over a large directory can return a SILENT early EOF — a stale cached listing with NO error. The phone's 9961-file Camera dir truncated at ~8400 once and db::apply deleted ~1700 good rows. Three layers now: 1. scan::read_dir_full — retry whole-dir reads on mid-stream errors; 2. per-directory cross-check in walk_dir: if the enumeration is far below the DB's count for that dir, re-read up to 3x keeping the best listing; whole-walk re-check in rescan keeps the larger of two walks; 3. db::apply mass-delete guard: never delete >20% of the DB in one apply unless a previous walk already saw the low count (two-strike; a meta.walked row tracks it). Genuine gradual deletions still flow through (verified: a 4000-file mass move streams the grid down progressively), and a second consecutive low walk still deletes. Emulator reliability suite (all pass): 3x launch count stability, add +1 / remove -1 in ~2 s, 20-file burst, DB recovery from damage, stale-listing refusal (0 rows lost), two-strike delete, live restore, idle 0.0% dropped frames, deep fling loads thumbnails. A/B vs ttimg (no-backend CSS-scroll control): adb-swipe drop deltas are at the input-injection artifact level, idle identical (0.0%).
This commit is contained in:
parent
bd27a7ccc1
commit
58b203e4fa
80
harnesses/gtauri/Cargo.lock
generated
80
harnesses/gtauri/Cargo.lock
generated
|
|
@ -1215,6 +1215,7 @@ dependencies = [
|
||||||
"image",
|
"image",
|
||||||
"kamadak-exif",
|
"kamadak-exif",
|
||||||
"log",
|
"log",
|
||||||
|
"notify",
|
||||||
"rusqlite",
|
"rusqlite",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
|
@ -1622,6 +1623,26 @@ dependencies = [
|
||||||
"cfb",
|
"cfb",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "inotify"
|
||||||
|
version = "0.11.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "4cc00ea907cab49550b7da656f80ebb97be1b997d931fbcd28d39734e17ce592"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.13.2",
|
||||||
|
"inotify-sys",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "inotify-sys"
|
||||||
|
version = "0.1.8"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "c033f80b2c113cdf91ab7a33faa9cbc014726dcad99880c8609af2a370edf37d"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ipnet"
|
name = "ipnet"
|
||||||
version = "2.12.2"
|
version = "2.12.2"
|
||||||
|
|
@ -1807,6 +1828,26 @@ dependencies = [
|
||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "kqueue"
|
||||||
|
version = "1.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8d763e5b24120b4ddf50de6c92308156765aabfbbccebf401da7cff2d70a41ea"
|
||||||
|
dependencies = [
|
||||||
|
"kqueue-sys",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "kqueue-sys"
|
||||||
|
version = "1.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.13.2",
|
||||||
|
"libc",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "libappindicator"
|
name = "libappindicator"
|
||||||
version = "0.9.0"
|
version = "0.9.0"
|
||||||
|
|
@ -1956,6 +1997,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee"
|
checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
|
"log",
|
||||||
"wasi",
|
"wasi",
|
||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
@ -2033,6 +2075,34 @@ version = "1.0.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
|
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "notify"
|
||||||
|
version = "8.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "82f812702fbc75d35416d3f884e04925be7e9498f4eceea2e24ba34559284674"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.13.2",
|
||||||
|
"inotify",
|
||||||
|
"kqueue",
|
||||||
|
"libc",
|
||||||
|
"log",
|
||||||
|
"mio",
|
||||||
|
"notify-types",
|
||||||
|
"objc2-core-foundation",
|
||||||
|
"objc2-core-services",
|
||||||
|
"walkdir",
|
||||||
|
"windows-sys 0.61.2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "notify-types"
|
||||||
|
version = "2.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0a437a9308232ce794b1e7fdf540d38ea37efc3c73492640c13b4c8568b8f108"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.13.2",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "num-conv"
|
name = "num-conv"
|
||||||
version = "0.2.2"
|
version = "0.2.2"
|
||||||
|
|
@ -2159,6 +2229,16 @@ dependencies = [
|
||||||
"objc2-foundation",
|
"objc2-foundation",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "objc2-core-services"
|
||||||
|
version = "0.3.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "583300ad934cba24ff5292aee751ecc070f7ca6b39a574cc21b7b5e588e06a0b"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"objc2-core-foundation",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "objc2-core-text"
|
name = "objc2-core-text"
|
||||||
version = "0.3.2"
|
version = "0.3.2"
|
||||||
|
|
|
||||||
|
|
@ -25,3 +25,8 @@ turbojpeg = { version = "1.5.1", features = ["cmake"] }
|
||||||
kamadak-exif = "0.6"
|
kamadak-exif = "0.6"
|
||||||
sha1 = "0.11"
|
sha1 = "0.11"
|
||||||
image = { version = "0.25", default-features = false, features = ["png", "webp", "gif"] }
|
image = { version = "0.25", default-features = false, features = ["png", "webp", "gif"] }
|
||||||
|
|
||||||
|
# Desktop file watching (inotify on Linux, FSEvents on macOS).
|
||||||
|
# Android uses the MediaStore ContentObserver instead — see MainActivity.kt.
|
||||||
|
[target.'cfg(not(target_os = "android"))'.dependencies]
|
||||||
|
notify = "8"
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ android {
|
||||||
defaultConfig {
|
defaultConfig {
|
||||||
manifestPlaceholders["usesCleartextTraffic"] = "false"
|
manifestPlaceholders["usesCleartextTraffic"] = "false"
|
||||||
applicationId = "wow.gmp.gallery"
|
applicationId = "wow.gmp.gallery"
|
||||||
minSdk = 24
|
minSdk = 30
|
||||||
targetSdk = 37
|
targetSdk = 37
|
||||||
versionCode = tauriProperties.getProperty("tauri.android.versionCode", "1").toInt()
|
versionCode = tauriProperties.getProperty("tauri.android.versionCode", "1").toInt()
|
||||||
versionName = tauriProperties.getProperty("tauri.android.versionName", "1.0")
|
versionName = tauriProperties.getProperty("tauri.android.versionName", "1.0")
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,44 @@
|
||||||
package wow.gmp.gallery
|
package wow.gmp.gallery
|
||||||
|
|
||||||
|
import android.database.ContentObserver
|
||||||
|
import android.net.Uri
|
||||||
|
|
||||||
import android.os.Bundle
|
import android.os.Bundle
|
||||||
|
import android.os.Handler
|
||||||
|
import android.os.Looper
|
||||||
|
import android.provider.MediaStore
|
||||||
import androidx.activity.enableEdgeToEdge
|
import androidx.activity.enableEdgeToEdge
|
||||||
|
|
||||||
class MainActivity : TauriActivity() {
|
class MainActivity : TauriActivity() {
|
||||||
|
private var observer: ContentObserver? = null
|
||||||
|
|
||||||
|
// Rust side (this app's JNI library, loaded by TauriActivity): wakes
|
||||||
|
// the file-watcher thread. One JNI call + one channel send — nothing
|
||||||
|
// heavy ever runs on the main thread.
|
||||||
|
external fun nativeFsChange()
|
||||||
|
|
||||||
override fun onCreate(savedInstanceState: Bundle?) {
|
override fun onCreate(savedInstanceState: Bundle?) {
|
||||||
enableEdgeToEdge()
|
enableEdgeToEdge()
|
||||||
super.onCreate(savedInstanceState)
|
super.onCreate(savedInstanceState)
|
||||||
|
// Event-driven file watching (no polling): MediaStore changes —
|
||||||
|
// camera saves, media-scanner indexing, other apps' indexed writes —
|
||||||
|
// wake the Rust watcher, which diffs dir mtimes and does an
|
||||||
|
// incremental rescan of only the changed subtrees.
|
||||||
|
val obs = object : ContentObserver(Handler(Looper.getMainLooper())) {
|
||||||
|
override fun onChange(selfChange: Boolean, uri: Uri?) {
|
||||||
|
nativeFsChange()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
observer = obs
|
||||||
|
// 3-arg overload (notifyForDescendants) requires API 30 — the app's
|
||||||
|
// minSdk (see tauri.conf.json bundle.android.minSdkVersion).
|
||||||
|
contentResolver.registerContentObserver(
|
||||||
|
MediaStore.Images.Media.EXTERNAL_CONTENT_URI, true, obs
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun onDestroy() {
|
||||||
|
observer?.let { contentResolver.unregisterContentObserver(it) }
|
||||||
|
super.onDestroy()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -73,7 +73,11 @@ pub fn open(path: &std::path::Path) -> Result<Connection, rusqlite::Error> {
|
||||||
thumb_path TEXT,
|
thumb_path TEXT,
|
||||||
date_taken INTEGER
|
date_taken INTEGER
|
||||||
);
|
);
|
||||||
CREATE INDEX IF NOT EXISTS items_mtime ON items (mtime DESC);",
|
CREATE INDEX IF NOT EXISTS items_mtime ON items (mtime DESC);
|
||||||
|
CREATE TABLE IF NOT EXISTS meta (
|
||||||
|
key TEXT PRIMARY KEY,
|
||||||
|
value TEXT NOT NULL
|
||||||
|
);",
|
||||||
)?;
|
)?;
|
||||||
Ok(conn)
|
Ok(conn)
|
||||||
}
|
}
|
||||||
|
|
@ -132,6 +136,51 @@ pub fn apply(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// MASS-DELETE GUARD: if this walk sees dramatically fewer files
|
||||||
|
// than the DB holds, the walk almost certainly truncated (FUSE
|
||||||
|
// mid-stream readdir error — see scan::read_dir_full) rather than
|
||||||
|
// a real mass deletion. Refuse to delete on the FIRST suspicious
|
||||||
|
// walk; a second consecutive low walk confirms it is real.
|
||||||
|
let mut to_delete = 0;
|
||||||
|
for p in existing.keys() {
|
||||||
|
if !seen.contains(p) {
|
||||||
|
to_delete += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let suspect = existing.len() > 100
|
||||||
|
&& (to_delete as f64) > (existing.len() as f64) * 0.2;
|
||||||
|
let mut do_delete = true;
|
||||||
|
if suspect {
|
||||||
|
let last_walked: i64 = tx
|
||||||
|
.query_row("SELECT value FROM meta WHERE key='walked'", [], |r| {
|
||||||
|
r.get::<_, String>(0)
|
||||||
|
})
|
||||||
|
.ok()
|
||||||
|
.and_then(|s| s.parse().ok())
|
||||||
|
.unwrap_or(existing.len() as i64);
|
||||||
|
if last_walked >= (existing.len() as i64) * 80 / 100 {
|
||||||
|
log::warn!(
|
||||||
|
"apply: walk saw {} but DB has {} (last walk {}): refusing to delete {} rows (truncated walk?)",
|
||||||
|
entries.len(),
|
||||||
|
existing.len(),
|
||||||
|
last_walked,
|
||||||
|
to_delete
|
||||||
|
);
|
||||||
|
do_delete = false;
|
||||||
|
}
|
||||||
|
let _ = tx.execute(
|
||||||
|
"INSERT INTO meta (key, value) VALUES ('walked', ?1) \
|
||||||
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||||
|
params![entries.len().to_string()],
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
let _ = tx.execute(
|
||||||
|
"INSERT INTO meta (key, value) VALUES ('walked', ?1) \
|
||||||
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||||
|
params![entries.len().to_string()],
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if do_delete {
|
||||||
for p in existing.keys() {
|
for p in existing.keys() {
|
||||||
if !seen.contains(p) {
|
if !seen.contains(p) {
|
||||||
del.execute(params![p])?;
|
del.execute(params![p])?;
|
||||||
|
|
@ -139,10 +188,32 @@ pub fn apply(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
Ok((inserted, updated, deleted))
|
Ok((inserted, updated, deleted))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Number of rows in the DB.
|
||||||
|
pub fn count(conn: &mut Connection) -> Result<i64, rusqlite::Error> {
|
||||||
|
let n: i64 = conn.query_row("SELECT COUNT(*) FROM items", [], |r| r.get(0))?;
|
||||||
|
Ok(n)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Per-directory file counts (parent dir → row count) — the reference
|
||||||
|
/// the walk uses to detect stale VFS directory listings.
|
||||||
|
pub fn dir_counts(conn: &mut Connection) -> Result<HashMap<PathBuf, i64>, rusqlite::Error> {
|
||||||
|
let mut st = conn.prepare("SELECT path FROM items")?;
|
||||||
|
let rows = st.query_map([], |r| r.get::<_, String>(0))?;
|
||||||
|
let mut m: HashMap<PathBuf, i64> = HashMap::new();
|
||||||
|
for p in rows {
|
||||||
|
let p = p?;
|
||||||
|
if let Some(d) = std::path::Path::new(&p).parent() {
|
||||||
|
*m.entry(d.to_path_buf()).or_insert(0) += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(m)
|
||||||
|
}
|
||||||
|
|
||||||
/// All known entries, newest first.
|
/// All known entries, newest first.
|
||||||
pub fn list_entries(conn: &mut Connection) -> Result<Vec<Entry>, rusqlite::Error> {
|
pub fn list_entries(conn: &mut Connection) -> Result<Vec<Entry>, rusqlite::Error> {
|
||||||
let mut st = conn.prepare("SELECT path, mtime, size FROM items ORDER BY mtime DESC, path ASC")?;
|
let mut st = conn.prepare("SELECT path, mtime, size FROM items ORDER BY mtime DESC, path ASC")?;
|
||||||
|
|
|
||||||
|
|
@ -713,20 +713,71 @@ fn load_meta(conn: &mut db::Connection) -> Result<(), rusqlite::Error> {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Incremental rescan: walk the roots, diff against the DB (insert
|
/// Total rows in the DB (0 on error).
|
||||||
/// new / update changed / delete removed), refresh META, emit the list.
|
fn expected_count() -> i64 {
|
||||||
/// Used by the startup scan AND the live watcher — a new photo appears
|
match db::open(&db::db_path()) {
|
||||||
/// in the grid seconds after it is written.
|
Ok(mut c) => db::count(&mut c).unwrap_or(0),
|
||||||
fn rescan(app: &tauri::AppHandle) {
|
Err(_) => 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Per-directory file counts from the DB — the reference the walk uses
|
||||||
|
/// to detect stale (silently truncated) VFS directory listings.
|
||||||
|
fn expected_dir_counts() -> std::collections::HashMap<std::path::PathBuf, i64> {
|
||||||
|
match db::open(&db::db_path()) {
|
||||||
|
Ok(mut c) => db::dir_counts(&mut c).unwrap_or_default(),
|
||||||
|
Err(_) => std::collections::HashMap::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Walk the roots once, streaming progress emits. Cross-checks each
|
||||||
|
/// directory's enumeration against the previous DB counts (stale-listing
|
||||||
|
/// protection — see scan::walk_stream).
|
||||||
|
fn walk_once(app: &tauri::AppHandle) -> Vec<scan::Entry> {
|
||||||
|
let expected = expected_dir_counts();
|
||||||
let roots = scan::roots();
|
let roots = scan::roots();
|
||||||
let last_emit = std::cell::Cell::new(std::time::Instant::now());
|
let last_emit = std::cell::Cell::new(std::time::Instant::now());
|
||||||
let entries = scan::walk_stream(&roots, &|batch| {
|
scan::walk_stream(&roots, &expected, &|batch| {
|
||||||
if last_emit.get().elapsed() < std::time::Duration::from_millis(500) {
|
if last_emit.get().elapsed() < std::time::Duration::from_millis(500) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
last_emit.set(std::time::Instant::now());
|
last_emit.set(std::time::Instant::now());
|
||||||
emit_files(app, true, batch.len());
|
emit_files(app, true, batch.len());
|
||||||
});
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Incremental rescan: walk the roots, diff against the DB (insert
|
||||||
|
/// new / update changed / delete removed), refresh META, emit the list.
|
||||||
|
/// Used by the startup scan AND the live watcher — a new photo appears
|
||||||
|
/// in the grid seconds after it is written.
|
||||||
|
///
|
||||||
|
/// Stale-listing (truncation) defense, three layers:
|
||||||
|
/// 1. per-directory cross-check during the walk (scan::walk_dir) —
|
||||||
|
/// re-reads any directory whose enumeration is far below the DB;
|
||||||
|
/// 2. whole-walk re-check below — if the TOTAL is still far below the
|
||||||
|
/// DB, re-walk once after a pause and keep the larger result;
|
||||||
|
/// 3. db::apply mass-delete guard — never delete >20% of the DB in a
|
||||||
|
/// single apply unless a previous walk already saw the low count.
|
||||||
|
fn rescan(app: &tauri::AppHandle) {
|
||||||
|
let mut entries = walk_once(app);
|
||||||
|
// Whole-walk re-check (layer 2).
|
||||||
|
let db_count = expected_count();
|
||||||
|
let walked = entries.len() as i64;
|
||||||
|
if walked < db_count.saturating_sub(100)
|
||||||
|
&& (db_count - walked) as f64 > (db_count as f64) * 0.2
|
||||||
|
{
|
||||||
|
tr(&format!(
|
||||||
|
"rescan: walk {} << db {} — re-walk in 2s",
|
||||||
|
entries.len(),
|
||||||
|
db_count
|
||||||
|
));
|
||||||
|
std::thread::sleep(std::time::Duration::from_secs(2));
|
||||||
|
let again = walk_once(app);
|
||||||
|
if again.len() > entries.len() {
|
||||||
|
tr(&format!("rescan: re-walk recovered {} → {}", entries.len(), again.len()));
|
||||||
|
entries = again;
|
||||||
|
}
|
||||||
|
}
|
||||||
let mut conn = match db::open(&db::db_path()) {
|
let mut conn = match db::open(&db::db_path()) {
|
||||||
Ok(c) => c,
|
Ok(c) => c,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
|
|
@ -768,8 +819,8 @@ fn dir_sig(roots: &[PathBuf]) -> HashMap<PathBuf, i64> {
|
||||||
while let Some(dir) = stack.pop() {
|
while let Some(dir) = stack.pop() {
|
||||||
let Ok(m) = std::fs::metadata(&dir) else { continue };
|
let Ok(m) = std::fs::metadata(&dir) else { continue };
|
||||||
out.insert(dir.clone(), mtime_ns(&m));
|
out.insert(dir.clone(), mtime_ns(&m));
|
||||||
if let Ok(rd) = std::fs::read_dir(&dir) {
|
if let Some(entries) = scan::read_dir_full(&dir) {
|
||||||
for ent in rd.flatten() {
|
for ent in entries {
|
||||||
let p = ent.path();
|
let p = ent.path();
|
||||||
if p.is_symlink() || !p.is_dir() {
|
if p.is_symlink() || !p.is_dir() {
|
||||||
continue;
|
continue;
|
||||||
|
|
@ -818,7 +869,14 @@ fn rescan_subtrees(app: &tauri::AppHandle, changed: &[PathBuf]) {
|
||||||
under.iter().map(|e| (e.path.as_str(), *e)).collect();
|
under.iter().map(|e| (e.path.as_str(), *e)).collect();
|
||||||
if under.len() <= BIG_DIR {
|
if under.len() <= BIG_DIR {
|
||||||
// Small: full walk with stat — adds, deletes AND edits.
|
// Small: full walk with stat — adds, deletes AND edits.
|
||||||
scan::walk_dir_into(d, &mut fresh);
|
// Stale-listing reference: per-directory counts from the DB.
|
||||||
|
let mut exp: HashMap<std::path::PathBuf, i64> = HashMap::new();
|
||||||
|
for e in &under {
|
||||||
|
if let Some(p) = std::path::Path::new(&e.path).parent() {
|
||||||
|
*exp.entry(p.to_path_buf()).or_insert(0) += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
scan::walk_dir_into(d, &mut fresh, &exp);
|
||||||
} else {
|
} else {
|
||||||
// Huge: readdir-only. New files get stat'd; known files keep
|
// Huge: readdir-only. New files get stat'd; known files keep
|
||||||
// their DB mtime/size (an in-place edit there is caught by
|
// their DB mtime/size (an in-place edit there is caught by
|
||||||
|
|
@ -862,48 +920,150 @@ fn rescan_subtrees(app: &tauri::AppHandle, changed: &[PathBuf]) {
|
||||||
tr(&format!("watch: incr changed={} fresh={added}", changed.len()));
|
tr(&format!("watch: incr changed={} fresh={added}", changed.len()));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Live file watching: poll the tree signature every second; when any
|
/// Watcher wake events.
|
||||||
/// directory changed (or 30 s elapsed as a safety net in case an mtime
|
#[derive(Debug, Clone, Copy)]
|
||||||
/// fails to update), debounce briefly so a write burst settles, then
|
enum WEvent {
|
||||||
/// run the incremental rescan. New photos appear in the grid within ~2 s
|
/// The OS said the filesystem changed (ContentObserver on Android,
|
||||||
/// of being written; deleted/renamed ones are picked up on the next tick.
|
/// inotify/FSEvents on desktop).
|
||||||
fn watch_files(app: tauri::AppHandle) {
|
Fs,
|
||||||
let mut sig: HashMap<PathBuf, i64> = HashMap::new();
|
/// Periodic full-pass timer.
|
||||||
let mut last_full = std::time::Instant::now();
|
Full,
|
||||||
|
}
|
||||||
|
|
||||||
|
static WATCH_TX: LazyLock<Mutex<Option<mpsc::Sender<WEvent>>>> =
|
||||||
|
LazyLock::new(|| Mutex::new(None));
|
||||||
|
|
||||||
|
/// Android: called by MainActivity's ContentObserver via JNI.
|
||||||
|
/// Signature: (JNIEnv*, jobject) for an instance method — both
|
||||||
|
/// ignored; the symbol name is what binds.
|
||||||
|
#[cfg(target_os = "android")]
|
||||||
|
#[no_mangle]
|
||||||
|
pub extern "system" fn Java_wow_gmp_gallery_MainActivity_nativeFsChange(
|
||||||
|
_env: *mut std::ffi::c_void,
|
||||||
|
_obj: *mut std::ffi::c_void,
|
||||||
|
) {
|
||||||
|
if let Some(tx) = WATCH_TX.lock().unwrap().as_ref() {
|
||||||
|
let _ = tx.send(WEvent::Fs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Desktop: inotify (Linux) / FSEvents (macOS) on the scan roots.
|
||||||
|
#[cfg(not(target_os = "android"))]
|
||||||
|
fn spawn_fs_watcher(tx: mpsc::Sender<WEvent>) {
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
use notify::{RecursiveMode, Watcher};
|
||||||
|
let (wt, wm) = mpsc::channel();
|
||||||
|
let mut w = match notify::recommended_watcher(move |res: Result<notify::Event, _>| {
|
||||||
|
if res.is_ok() {
|
||||||
|
let _ = wt.send(()); // consumer coalesces + diffs
|
||||||
|
}
|
||||||
|
}) {
|
||||||
|
Ok(w) => w,
|
||||||
|
Err(e) => {
|
||||||
|
log::warn!("fs watcher: {e}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
for r in scan::roots() {
|
||||||
|
if r.is_dir() {
|
||||||
|
let _ = w.watch(&r, RecursiveMode::Recursive);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let _keep = wm; // drop = stop watching
|
||||||
loop {
|
loop {
|
||||||
std::thread::sleep(std::time::Duration::from_secs(1));
|
if wt.recv().is_err() {
|
||||||
let roots = scan::roots();
|
break;
|
||||||
let next = dir_sig(&roots);
|
}
|
||||||
let changed: Vec<PathBuf> = if sig.is_empty() {
|
let _ = tx.send(WEvent::Fs);
|
||||||
sig = next;
|
}
|
||||||
Vec::new() // first pass: just establish the baseline
|
});
|
||||||
} else {
|
}
|
||||||
let mut v: Vec<PathBuf> = Vec::new();
|
|
||||||
for (d, t) in next.iter() {
|
/// Diff two dir-mtime signatures: dirs added, removed, or changed.
|
||||||
if sig.get(d.as_path()) != Some(t) {
|
fn changed_dirs(
|
||||||
|
old: &HashMap<PathBuf, i64>,
|
||||||
|
next: &HashMap<PathBuf, i64>,
|
||||||
|
) -> Vec<PathBuf> {
|
||||||
|
let mut v: Vec<PathBuf> = next
|
||||||
|
.iter()
|
||||||
|
.filter(|(d, t)| old.get(d.as_path()) != Some(t))
|
||||||
|
.map(|(d, _)| d.clone())
|
||||||
|
.collect();
|
||||||
|
for d in old.keys() {
|
||||||
|
if !next.contains_key(d) {
|
||||||
v.push(d.clone());
|
v.push(d.clone());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for d in sig.keys() {
|
|
||||||
if !next.contains_key(d) {
|
|
||||||
v.push(d.clone()); // directory removed
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sig = next;
|
|
||||||
v
|
v
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Live file watching — EVENT-DRIVEN, no tree polling:
|
||||||
|
/// - Android: MainActivity's MediaStore ContentObserver → JNI → this
|
||||||
|
/// thread (camera saves, media-scanner indexing, indexed writes).
|
||||||
|
/// - Desktop: inotify/FSEvents via `notify` on the scan roots.
|
||||||
|
/// - On wake: coalesce the burst, debounce for the write to settle,
|
||||||
|
/// diff dir mtimes (a few stats), and rescan ONLY the changed
|
||||||
|
/// subtrees. A 5-minute full pass is the safety net (in-place edits
|
||||||
|
/// in the huge dir, anything the OS never reported).
|
||||||
|
/// - The 1 s recv timeout doubles as a cheap secondary detector
|
||||||
|
/// (dir-mtime diff, ~0.05 ms): raw out-of-band copies that the media
|
||||||
|
/// scanner hasn't indexed yet are still caught within ~1 s.
|
||||||
|
fn watch_files(app: tauri::AppHandle) {
|
||||||
|
let (tx, rx) = mpsc::channel::<WEvent>();
|
||||||
|
*WATCH_TX.lock().unwrap() = Some(tx.clone());
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
loop {
|
||||||
|
std::thread::sleep(std::time::Duration::from_secs(300));
|
||||||
|
let _ = tx.send(WEvent::Full);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
#[cfg(not(target_os = "android"))]
|
||||||
|
spawn_fs_watcher(tx);
|
||||||
|
let mut sig: HashMap<PathBuf, i64> = HashMap::new();
|
||||||
|
let mut baseline = true;
|
||||||
|
let mut last_full = std::time::Instant::now();
|
||||||
|
loop {
|
||||||
|
let woke = match rx.recv_timeout(std::time::Duration::from_secs(1)) {
|
||||||
|
Ok(e) => Some(e),
|
||||||
|
Err(mpsc::RecvTimeoutError::Timeout) => None,
|
||||||
|
Err(mpsc::RecvTimeoutError::Disconnected) => break,
|
||||||
};
|
};
|
||||||
|
let do_full = matches!(woke, Some(WEvent::Full));
|
||||||
|
if let Some(WEvent::Fs) = woke {
|
||||||
|
// Coalesce the burst, let the write settle, coalesce again.
|
||||||
|
while rx.try_recv().is_ok() {}
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(600));
|
||||||
|
while rx.try_recv().is_ok() {}
|
||||||
|
}
|
||||||
|
let next = dir_sig(&scan::roots());
|
||||||
|
let changed = if baseline {
|
||||||
|
baseline = false;
|
||||||
|
Vec::new()
|
||||||
|
} else {
|
||||||
|
changed_dirs(&sig, &next)
|
||||||
|
};
|
||||||
|
sig = next;
|
||||||
|
if do_full {
|
||||||
|
tr("watch: full rescan (5min)");
|
||||||
|
rescan(&app);
|
||||||
|
last_full = std::time::Instant::now();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if changed.is_empty() {
|
if changed.is_empty() {
|
||||||
// Safety net: in-place edits and anything the mtime poll
|
if woke.is_some() {
|
||||||
// missed. Full walk (~30 s on the phone) every 5 min.
|
// The OS reported a change but no dir mtime moved
|
||||||
if last_full.elapsed() >= std::time::Duration::from_secs(300) {
|
// (in-place edit / event for an untracked file):
|
||||||
tr("watch: full rescan");
|
// full rescan, throttled to one per minute.
|
||||||
|
if last_full.elapsed() >= std::time::Duration::from_secs(60) {
|
||||||
|
tr("watch: full rescan (event, no dir delta)");
|
||||||
rescan(&app);
|
rescan(&app);
|
||||||
last_full = std::time::Instant::now();
|
last_full = std::time::Instant::now();
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
// Timeout with no dir delta: nothing to do (~0.05 ms spent).
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// Let the write burst (file + temp + sidecars) settle.
|
tr(&format!("watch: {} changed dir(s)", changed.len()));
|
||||||
std::thread::sleep(std::time::Duration::from_millis(600));
|
|
||||||
rescan_subtrees(&app, &changed);
|
rescan_subtrees(&app, &changed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@
|
||||||
//! what makes the app testable without MediaStore; MediaStore integration is
|
//! what makes the app testable without MediaStore; MediaStore integration is
|
||||||
//! a later refinement (HANDOFF.md §7.4).
|
//! a later refinement (HANDOFF.md §7.4).
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
/// One image on disk. `mtime` is nanoseconds since the Unix epoch (i64).
|
/// One image on disk. `mtime` is nanoseconds since the Unix epoch (i64).
|
||||||
|
|
@ -43,30 +44,54 @@ pub fn roots() -> Vec<PathBuf> {
|
||||||
/// Recursively collect all image entries below `roots`.
|
/// Recursively collect all image entries below `roots`.
|
||||||
/// `on_progress(n)` is called every 512 files found.
|
/// `on_progress(n)` is called every 512 files found.
|
||||||
pub fn walk(roots: &[PathBuf], on_progress: &dyn Fn(usize)) -> Vec<Entry> {
|
pub fn walk(roots: &[PathBuf], on_progress: &dyn Fn(usize)) -> Vec<Entry> {
|
||||||
walk_stream(roots, &|batch| on_progress(batch.len()))
|
walk_stream(roots, &HashMap::new(), &|batch| on_progress(batch.len()))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Like `walk`, but streams progress: `on_batch(&entries)` receives ALL
|
/// Like `walk`, but streams progress: `on_batch(&entries)` receives ALL
|
||||||
/// entries collected so far every 512 new files, so the UI can show results
|
/// entries collected so far every 512 new files, so the UI can show results
|
||||||
/// while the scan is still running (first launch must not wait for the full
|
/// while the scan is still running (first launch must not wait for the full
|
||||||
/// DB build).
|
/// DB build).
|
||||||
pub fn walk_stream(roots: &[PathBuf], on_batch: &dyn Fn(&[Entry])) -> Vec<Entry> {
|
///
|
||||||
|
/// `expected_files` is the per-directory file count from the previous DB
|
||||||
|
/// state (parent dir → count). When an enumeration returns far fewer
|
||||||
|
/// files than the DB holds for that directory, the VFS is almost certainly
|
||||||
|
/// serving a STALE cached directory listing (readdir returns a silent
|
||||||
|
/// early EOF — no error at all; reproduced on both the phone's FUSE and
|
||||||
|
/// the emulator's 9p /sdcard). In that case the directory is re-read up
|
||||||
|
/// to 3 times with delays, keeping the best listing.
|
||||||
|
pub fn walk_stream(
|
||||||
|
roots: &[PathBuf],
|
||||||
|
expected_files: &HashMap<PathBuf, i64>,
|
||||||
|
on_batch: &dyn Fn(&[Entry]),
|
||||||
|
) -> Vec<Entry> {
|
||||||
let mut out = Vec::new();
|
let mut out = Vec::new();
|
||||||
for r in roots {
|
for r in roots {
|
||||||
if r.is_dir() {
|
if r.is_dir() {
|
||||||
walk_dir(r, &mut out, on_batch);
|
walk_dir(r, &mut out, expected_files, on_batch);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Full walk (with stat) of ONE directory subtree.
|
/// Full walk (with stat) of ONE directory subtree.
|
||||||
pub fn walk_dir_into(dir: &Path, out: &mut Vec<Entry>) {
|
pub fn walk_dir_into(dir: &Path, out: &mut Vec<Entry>, expected_files: &HashMap<PathBuf, i64>) {
|
||||||
if dir.is_dir() {
|
if dir.is_dir() {
|
||||||
walk_dir(dir, out, &|_| {});
|
walk_dir(dir, out, expected_files, &|_| {});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Count image-looking names in a listing (cheap: names only, no stat).
|
||||||
|
fn count_image_names(es: &[std::fs::DirEntry]) -> i64 {
|
||||||
|
es.iter().filter(|e| {
|
||||||
|
let name = e.file_name().to_string_lossy().into_owned();
|
||||||
|
!name.starts_with('.')
|
||||||
|
&& e.path()
|
||||||
|
.extension()
|
||||||
|
.and_then(|x| x.to_str())
|
||||||
|
.is_some_and(|x| IMAGE_EXTS.contains(&x.to_ascii_lowercase().as_str()))
|
||||||
|
}).count() as i64
|
||||||
|
}
|
||||||
|
|
||||||
/// Readdir-only walk (NO per-file stat) of ONE directory subtree — for
|
/// Readdir-only walk (NO per-file stat) of ONE directory subtree — for
|
||||||
/// huge FUSE directories where stat'ing every file costs ~30 s; the
|
/// huge FUSE directories where stat'ing every file costs ~30 s; the
|
||||||
/// caller supplies metadata for new files and keeps DB metadata for
|
/// caller supplies metadata for new files and keeps DB metadata for
|
||||||
|
|
@ -77,8 +102,8 @@ pub fn walk_dir_nostats(dir: &Path, cb: &mut dyn FnMut(&Path)) {
|
||||||
}
|
}
|
||||||
let mut stack = vec![dir.to_path_buf()];
|
let mut stack = vec![dir.to_path_buf()];
|
||||||
while let Some(d) = stack.pop() {
|
while let Some(d) = stack.pop() {
|
||||||
let Ok(rd) = std::fs::read_dir(&d) else { continue };
|
let Some(entries) = read_dir_full(&d) else { continue };
|
||||||
for ent in rd.flatten() {
|
for ent in entries {
|
||||||
let p = ent.path();
|
let p = ent.path();
|
||||||
let Ok(ft) = ent.file_type() else { continue };
|
let Ok(ft) = ent.file_type() else { continue };
|
||||||
if ft.is_dir() {
|
if ft.is_dir() {
|
||||||
|
|
@ -96,9 +121,83 @@ pub fn walk_dir_nostats(dir: &Path, cb: &mut dyn FnMut(&Path)) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn walk_dir(dir: &Path, out: &mut Vec<Entry>, on_batch: &dyn Fn(&[Entry])) {
|
/// Read a directory to completion, retrying on transient mid-stream
|
||||||
let Ok(rd) = std::fs::read_dir(dir) else { return };
|
/// errors. CRITICAL on Android FUSE: a std::fs::read_dir over a
|
||||||
for ent in rd.flatten() {
|
/// thousands-of-entries directory latches the iterator into an error
|
||||||
|
/// state on ANY transient failure (verified on the phone: the 9961-file
|
||||||
|
/// Camera dir intermittently truncated at ~8400 entries, and the
|
||||||
|
/// truncated walk made db::apply delete ~1700 good rows). On a
|
||||||
|
/// mid-stream error the whole directory is re-read; after 3 failed
|
||||||
|
/// attempts the directory is skipped (the mass-delete guard in
|
||||||
|
/// db::apply is the backstop).
|
||||||
|
pub fn read_dir_full(dir: &Path) -> Option<Vec<std::fs::DirEntry>> {
|
||||||
|
for attempt in 0..3u32 {
|
||||||
|
let Ok(rd) = std::fs::read_dir(dir) else { return None };
|
||||||
|
let mut entries = Vec::new();
|
||||||
|
let mut ok = true;
|
||||||
|
for ent in rd {
|
||||||
|
match ent {
|
||||||
|
Ok(e) => entries.push(e),
|
||||||
|
Err(_) => {
|
||||||
|
ok = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ok {
|
||||||
|
return Some(entries);
|
||||||
|
}
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(200 * (attempt + 1) as u64));
|
||||||
|
}
|
||||||
|
log::error!("read_dir {:?}: 3 mid-stream failures, skipping dir", dir);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn walk_dir(
|
||||||
|
dir: &Path,
|
||||||
|
out: &mut Vec<Entry>,
|
||||||
|
expected_files: &HashMap<PathBuf, i64>,
|
||||||
|
on_batch: &dyn Fn(&[Entry]),
|
||||||
|
) {
|
||||||
|
let mut entries = match read_dir_full(dir) {
|
||||||
|
Some(e) => e,
|
||||||
|
None => return,
|
||||||
|
};
|
||||||
|
// STALE-LISTING CROSS-CHECK (see walk_stream docs): if the DB holds
|
||||||
|
// many more files for this directory than the enumeration produced,
|
||||||
|
// re-read the directory a few times and keep the best listing.
|
||||||
|
if let Some(&want) = expected_files.get(dir) {
|
||||||
|
if want >= 50 {
|
||||||
|
let mut best_n = count_image_names(&entries);
|
||||||
|
if best_n * 5 < want * 4 {
|
||||||
|
for attempt in 1..=3u32 {
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(300 * u64::from(attempt)));
|
||||||
|
if let Some(again) = read_dir_full(dir) {
|
||||||
|
let n = count_image_names(&again);
|
||||||
|
if n > best_n {
|
||||||
|
best_n = n;
|
||||||
|
entries = again;
|
||||||
|
}
|
||||||
|
if best_n * 5 >= want * 4 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if best_n * 5 < want * 4 {
|
||||||
|
log::warn!(
|
||||||
|
"walk {:?}: enumerated {} image files, DB has {} (stale listing?)",
|
||||||
|
dir,
|
||||||
|
best_n,
|
||||||
|
want
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
log::info!("walk {:?}: stale listing recovered {} → {}",
|
||||||
|
dir, want, best_n);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for ent in entries {
|
||||||
let p = ent.path();
|
let p = ent.path();
|
||||||
let Ok(ft) = ent.file_type() else { continue };
|
let Ok(ft) = ent.file_type() else { continue };
|
||||||
if ft.is_dir() {
|
if ft.is_dir() {
|
||||||
|
|
@ -108,7 +207,7 @@ fn walk_dir(dir: &Path, out: &mut Vec<Entry>, on_batch: &dyn Fn(&[Entry])) {
|
||||||
}
|
}
|
||||||
// Don't follow symlinked dirs (loop protection).
|
// Don't follow symlinked dirs (loop protection).
|
||||||
if !p.is_symlink() {
|
if !p.is_symlink() {
|
||||||
walk_dir(&p, out, on_batch);
|
walk_dir(&p, out, expected_files, on_batch);
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -28,6 +28,9 @@
|
||||||
"bundle": {
|
"bundle": {
|
||||||
"active": true,
|
"active": true,
|
||||||
"targets": "all",
|
"targets": "all",
|
||||||
|
"android": {
|
||||||
|
"minSdkVersion": 30
|
||||||
|
},
|
||||||
"icon": [
|
"icon": [
|
||||||
"icons/32x32.png",
|
"icons/32x32.png",
|
||||||
"icons/128x128.png",
|
"icons/128x128.png",
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user