//! gtauri — the gallery, on Tauri v2 (Chromium does the heavy lifting). //! //! Division of labor (the experiment's verdict, 2026-10-31): //! Chromium/webview: grid layout, compositor-driven native scroll, lazy //! fetch+decode (async decode threads, downsampled decode, image //! cache + memory-pressure eviction), viewer compositing (CSS transforms). //! Rust: file scan, SQLite, and the on-disk THUMBNAIL CACHE served over //! the `gallery://` uri scheme — `gallery://thumb/` returns a small //! JPEG (cache hit: straight from disk; miss: turbojpeg DCT-scaled //! decode → re-encode → write cache → serve), `gallery://full/` //! returns the original file bytes for the viewer. //! //! The webview's own lazy loader is the demand scheduler: no Rust-side //! prefetch queue, no worker pool — Chromium asks for exactly what the //! viewport is about to show. use std::collections::{HashMap, HashSet, VecDeque}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{mpsc, LazyLock, Mutex, RwLock, Condvar}; use log::{Level, LevelFilter, Log, Metadata, Record}; use serde::{Deserialize, Serialize}; use tauri::Emitter as _; use tauri::Listener as _; use tauri::Manager as _; use sha1::{Digest, Sha1}; mod db; mod images; mod scan; #[cfg(target_os = "android")] const ANDROID_PACKAGE: &str = "wow.gmp.gallery"; /// Grid thumbnail longest side (px). Cells are ~410 device px wide on the /// Pixel 9 Pro (1280 / 3 cols); 512 is sharp at cell size while letting a /// 48 MP photo decode at 1/8 DCT scale instead of 1/4 (4x cheaper). const THUMB_TARGET: u32 = 512; const THUMB_QUALITY: u8 = 82; /// One image known to the app. `id` is the DB rowid (stable per row). #[derive(Debug, Clone, Serialize)] struct Meta { id: i64, path: String, mtime: i64, size: i64, } /// id → metadata, refreshed after every DB apply. static META: LazyLock>> = LazyLock::new(|| RwLock::new(HashMap::new())); /// FR log path (Tauri document_dir = getExternalFilesDir(DOCUMENTS); the /// framework creates it — a raw std mkdir of the app external tree is /// SELinux-denied). Same FR-line format as the other harnesses so /// fr_capture.py remains the referee. static FR_PATH: Mutex> = Mutex::new(None); static FR_TX: Mutex>> = Mutex::new(None); /// Count of images served from the on-disk cache (vs decoded live) — /// reported on the IMG status line. static CACHE_HITS: AtomicUsize = AtomicUsize::new(0); // --------------------------------------------------------------------------- // thumbnail memory cache + scroll prefetch /// In-memory LRU of ENCODED bytes (thumb JPEGs, full-res originals). /// A mem hit is ~0 ms — the whole point: fast/long scrolls hit cache /// because the workers raced ahead of the viewport. struct ByteLru { cap: u64, map: HashMap>, order: VecDeque, bytes: u64, } impl ByteLru { fn new(cap: u64) -> Self { Self { cap, map: HashMap::new(), order: VecDeque::new(), bytes: 0 } } fn get(&self, id: i64) -> Option> { self.map.get(&id).cloned() } fn has(&self, id: i64) -> bool { self.map.contains_key(&id) } fn put(&mut self, id: i64, bytes: Vec) { if let Some(old) = self.map.get(&id) { self.bytes -= old.len() as u64; } self.bytes += bytes.len() as u64; self.order.push_back(id); self.map.insert(id, bytes); while self.bytes > self.cap { match self.order.pop_front() { Some(old) => { if let Some(v) = self.map.remove(&old) { self.bytes -= v.len() as u64; } } None => break, } } } } static MEM: LazyLock> = LazyLock::new(|| Mutex::new(ByteLru::new(128 * 1024 * 1024))); // ~5000 encoded thumbs /// Full-res originals for the viewer: a hit makes openViewer instant even /// after scrolling far (the file read never touches the main thread). static FULL_MEM: LazyLock> = LazyLock::new(|| Mutex::new(ByteLru::new(256 * 1024 * 1024))); fn mem_get(id: i64) -> Option> { MEM.lock().unwrap().get(id) } fn mem_put(id: i64, bytes: Vec) { MEM.lock().unwrap().put(id, bytes); } /// NORMAL queue: the prefetch window (off-screen, ahead of the viewport). /// URGENT queue: ids Chromium actually requested (visible band). The worker /// always drains URGENT first — requested cells jump the off-screen line. /// /// WHY A QUEUE AT ALL: wry calls our protocol handler SYNCHRONOUSLY from /// shouldInterceptRequest on Chromium's MAIN thread (wry RustWebViewClient.kt). /// Decoding there would serialize ~20 visible cells × 30-100 ms of file read /// + decode + encode on the main thread (observed: seconds of blank cells /// after a jump). So the handler returns a 1x1 placeholder instantly on a /// miss; the worker decodes off-main; JS re-requests on `thumb-ready` and /// gets a mem hit. wry also forces Cache-Control: no-store on our responses, /// so the mem cache is effectively the HTTP cache. /// ONE queue, one lock, one condvar (the classic pattern — the earlier /// two-queue design stranded urgent items when the worker slept on the /// other queue's condvar, and a variant holding the urgent lock across the /// sleep was a straight deadlock). `urgent` jobs (real requests for visible /// cells) push_front; prefetch jobs push_back. struct Job { id: i64, /// Display-order index — used to DROP stale work: once the viewport /// has moved STALE_ROWS past a job, that job's cell is far off-screen /// and decoding it is wasted CPU (a user who fled leaves no backlog). idx: u32, urgent: bool, /// true = full-res read for the viewer, false = grid thumbnail. full: bool, } static Q: LazyLock>> = LazyLock::new(|| Mutex::new(VecDeque::new())); static Q_CV: LazyLock = LazyLock::new(|| Condvar::new()); static INFLIGHT: LazyLock>> = LazyLock::new(|| Mutex::new(HashSet::new())); /// First visible grid cell (display order) — the viewport anchor from JS. static VPORT: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); /// Jobs farther behind the viewport than this are dropped, not decoded. const STALE_ROWS: u32 = 60; /// Hard queue ceiling: beyond this, new urgent work is deferred to the /// JS 400 ms safety-net retry instead of growing an unbounded backlog. const Q_CAP: usize = 600; /// id → display-order index (rebuilt on every emit_files). static ID_TO_IDX: LazyLock>> = LazyLock::new(|| RwLock::new(HashMap::new())); /// ids with decode/read failures: id -> consecutive failure count. /// STAGED corruption: with live watching, a photo still being written /// (camera mid-save) fails one or two times and then decodes fine, so a /// single failure must NOT be final. Genuine garbage (mislabeled WEBP, /// 17-byte files, …) keeps failing and gives up after BROKEN_AFTER — /// placeholder forever, no retry loops. A success clears the count. const BROKEN_AFTER: u32 = 3; static FAILS: LazyLock>> = LazyLock::new(|| Mutex::new(HashMap::new())); fn is_broken(id: i64) -> bool { FAILS.lock().unwrap().get(&id).is_some_and(|n| *n >= BROKEN_AFTER) } fn note_fail(id: i64) { *FAILS.lock().unwrap().entry(id).or_insert(0) += 1; } fn note_ok(id: i64) { FAILS.lock().unwrap().remove(&id); } /// 1x1 JPEG in the cell background color (#1c1c22) — served instantly for /// cold misses; JS swaps in the real bytes when the worker finishes. static PLACEHOLDER: LazyLock> = LazyLock::new(|| { let rgb: Vec = vec![28, 28, 34]; images::encode_jpeg(&rgb, 1, 1, 80).expect("1x1 jpeg") }); /// App handle for worker → webview events (`thumb-ready`). static APP_HANDLE: LazyLock>> = LazyLock::new(|| Mutex::new(None)); /// Display order (mtime desc, path asc) — same order the grid renders. /// Prefetch windows are expressed as index ranges into this. static DISPLAY_IDS: LazyLock>> = LazyLock::new(|| RwLock::new(Vec::new())); /// Unique tmp-name suffix so two concurrent decode_and_cache calls (the /// request path and the prefetch worker can hit the same file) never write /// the same tmp file. static TMP_SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); /// Decode → encode → write disk cache. Called from the REQUEST path (which /// must be instant — see set_prefetch_window) and from the prefetch worker. /// No shared lock: a visible-region request never waits on off-screen work. fn decode_and_cache(meta: &Meta) -> Result, String> { let cp = thumb_cache_path(&meta.path, meta.mtime, meta.size); if cp.is_file() { // Another path (worker or protocol) filled the disk cache just now. let b = std::fs::read(&cp).map_err(|e| format!("cache read: {e}"))?; if !b.is_empty() { return Ok(b); } } let thumb = images::make_thumb(Path::new(&meta.path), THUMB_TARGET) .map_err(|e| format!("thumb {}: {e}", meta.id))?; let jpeg = images::encode_jpeg(&thumb.rgb, thumb.width, thumb.height, THUMB_QUALITY) .map_err(|e| format!("thumb encode {}: {e}", meta.id))?; // Atomic-ish write: unique tmp then rename (a torn cache file would // poison the key forever). let seq = TMP_SEQ.fetch_add(1, Ordering::Relaxed); let tmp = cp.with_file_name(format!("{}.tmp{}-{}", cp.file_stem().unwrap().to_string_lossy(), std::process::id(), seq)); std::fs::write(&tmp, &jpeg).map_err(|e| format!("cache write: {e}"))?; std::fs::rename(&tmp, &cp).map_err(|e| format!("cache rename: {e}"))?; Ok(jpeg) } /// Set the prefetch window to [lo, hi) — REPLACING whatever is queued. /// The queue always holds exactly "what is about to be seen": when the /// viewport jumps, stale off-screen work (any job, urgent or not) is /// dropped instantly, so a fast scroll can never leave a decode backlog /// that starves the visible region. fn set_prefetch_window(lo: usize, hi: usize) { let ids = DISPLAY_IDS.read().unwrap(); if ids.is_empty() || lo >= hi { return; } let lo = lo.min(ids.len()); let hi = hi.min(ids.len()); if lo >= hi { return; } VPORT.store(lo as u32, Ordering::Relaxed); let mut q = Q.lock().unwrap(); let mut inq = INFLIGHT.lock().unwrap(); // The anchor just moved to `lo`: everything more than STALE_ROWS // behind it is dead work (urgent included — a fling leaves urgent // jobs for every band it crossed). Drop it instantly so it can never // delay the new viewport or eat queue capacity. let kept = q.len(); q.retain(|j| { if j.idx + STALE_ROWS >= lo as u32 { return true; } inq.remove(&j.id); false }); if q.len() != kept { tr(&format!("window a={lo} drop-stale {}", kept - q.len())); } let mut added = false; for (off, &id) in ids[lo..hi].iter().enumerate() { if inq.contains(&id) { continue; // one decode in flight; it will land in mem soon } if MEM.lock().unwrap().has(id) { continue; } inq.insert(id); q.push_back(Job { id, idx: (lo + off) as u32, urgent: false, full: false }); added = true; } if added { Q_CV.notify_one(); } } /// Pop the next job, dropping stale ones (far behind the viewport) as we /// pass them. Blocks on the condvar when the queue is empty. Hold nothing /// across the block; the wait re-checks under the same lock the pushers /// use, so no notify can be lost. fn pop_next() -> Job { let mut g = Q.lock().unwrap(); loop { if g.is_empty() { g = Q_CV.wait(g).unwrap(); continue; } // Distance-from-viewport policy: any job more than STALE_ROWS // behind the viewport is dead work — the user flung past it, and // decoding it would only delay what is actually on screen. This // applies to URGENT jobs too (a fling queues urgent work for every // band it crosses; all of it except the final band is stale). Safe // because the anchor now always settles at the final scroll // position (trailing send), and on-screen cells have idx >= VPORT // so they can never be dropped. (A pure LIFO would not solve this: // the stale pile sits at the stack bottom, never popped, still // blocking the queue cap.) if let Some(j) = g.front() { if j.idx + STALE_ROWS < VPORT.load(Ordering::Relaxed) { let j = g.pop_front().unwrap(); INFLIGHT.lock().unwrap().remove(&j.id); tr(&format!("drop stale id={} idx={} urgent={}", j.id, j.idx, j.urgent)); continue; } } return g.pop_front().unwrap(); } } /// One decode worker (N spawned in setup). Decodes OFF the main thread; /// emits `thumb-ready` / `full-ready` so JS re-requests (a mem hit, ~1 ms /// on main). Parallel workers keep up with fast long scrolls: one worker /// decodes ~3 phones/sec of full-res photos; four decode ~12/sec. fn decode_worker() { loop { let job = pop_next(); let id = job.id; tr(&format!("decode {} start full={}", id, job.full)); let t0 = std::time::Instant::now(); if let Some(meta) = META.read().unwrap().get(&id).cloned() { if job.full { match std::fs::read(&meta.path) { Ok(bytes) => { note_ok(id); FULL_MEM.lock().unwrap().put(id, bytes); tr(&format!("full {id} done {:.0}ms", t0.elapsed().as_secs_f64())); if let Some(app) = APP_HANDLE.lock().unwrap().as_ref() { let _ = app.emit("full-ready", serde_json::json!({ "id": id })); } } Err(e) => { log::warn!("full {id}: {e}"); note_fail(id); tr(&format!("full {id} fail {}: {e}", FAILS.lock().unwrap().get(&id).copied().unwrap_or(0))); if let Some(app) = APP_HANDLE.lock().unwrap().as_ref() { let _ = app.emit("full-ready", serde_json::json!({ "id": id })); } } } } else { match decode_and_cache(&meta) { Ok(jpeg) => { note_ok(id); mem_put(id, jpeg); tr(&format!("decode {id} done {:.0}ms", t0.elapsed().as_secs_f64())); if let Some(app) = APP_HANDLE.lock().unwrap().as_ref() { let _ = app.emit("thumb-ready", serde_json::json!({ "id": id })); } } Err(e) => { log::warn!("decode {id}: {e}"); note_fail(id); tr(&format!("decode {id} fail {}: {e}", FAILS.lock().unwrap().get(&id).copied().unwrap_or(0))); if let Some(app) = APP_HANDLE.lock().unwrap().as_ref() { let _ = app.emit("thumb-ready", serde_json::json!({ "id": id })); } } } } } INFLIGHT.lock().unwrap().remove(&id); } } /// Writes a heartbeat line every 2 s. Rust-side liveness: if these stop, /// the process died; if they continue but req/js lines stop, the webview /// main thread or renderer is dead. fn heartbeat() { let mut n = 0u64; loop { let qlen = Q.lock().unwrap().len(); let memkb = MEM.lock().unwrap().bytes / 1024; let broken = FAILS.lock().unwrap().values().filter(|n| **n >= BROKEN_AFTER).count(); tr(&format!( "hb n={n} q={qlen} mem={}KB broken={broken}", memkb, )); n += 1; std::thread::sleep(std::time::Duration::from_secs(2)); } } // --------------------------------------------------------------------------- // logging (logcat on Android — the log crate is invisible without a backend) #[cfg(target_os = "android")] fn init_logcat() { extern "C" { fn __android_log_write(prio: i32, tag: *const u8, msg: *const u8) -> i32; } struct Cat; impl Log for Cat { fn enabled(&self, m: &Metadata) -> bool { m.level() <= Level::Info } fn log(&self, record: &Record) { if !self.enabled(record.metadata()) { return; } let prio = match record.level() { Level::Error => 6, Level::Warn => 5, Level::Info => 4, _ => 3, }; let mut s = format!("{}\0", record.args()); unsafe { __android_log_write(prio, b"gtauri\0".as_ptr(), s.as_ptr()); } } fn flush(&self) {} } let _ = log::set_boxed_logger(Box::new(Cat)).map(|()| log::set_max_level(LevelFilter::Info)); } /// File-based trace (logcat drops lines under load — unreliable referee). static TRACE: Mutex> = Mutex::new(None); pub fn init_trace(dir: &Path) { if let Ok(f) = std::fs::OpenOptions::new().create(true).append(true).open(dir.join("trace.log")) { *TRACE.lock().unwrap() = Some(f); } } pub fn tr(msg: &str) { use std::io::Write; if let Some(f) = TRACE.lock().unwrap().as_mut() { let _ = writeln!(f, "{:.2} {}", T0.elapsed().as_secs_f64(), msg); } } static T0: LazyLock = LazyLock::new(std::time::Instant::now); fn init_fr_logger(dir: &Path) { let path = dir.join("fr.log"); match std::fs::OpenOptions::new().create(true).append(true).open(&path) { Ok(f) => { let (tx, rx) = mpsc::channel::(); *FR_TX.lock().unwrap() = Some(tx); std::thread::spawn(move || { let mut w = std::io::BufWriter::new(f); for line in rx { use std::io::Write; let _ = w.write_all(line.as_bytes()); let _ = w.flush(); } }); *FR_PATH.lock().unwrap() = Some(path.clone()); log::info!("FR log → {:?}", path); } Err(e) => log::error!("cannot open FR log {:?}: {e}", path), } } // --------------------------------------------------------------------------- // storage layout /// App-writable external dir (same as db.rs's convention; the egui gallery's /// DB from earlier builds lives here and is reused as-is). fn external_files_dir() -> PathBuf { if let Ok(d) = std::env::var("EXTERNAL_DATA_DIR") { let p = PathBuf::from(d); if std::fs::create_dir_all(&p).is_ok() { return p; } } if let Ok(st) = std::env::var("EXTERNAL_STORAGE") { let p = PathBuf::from(st).join("Android/data").join(ANDROID_PACKAGE).join("files"); if std::fs::create_dir_all(&p).is_ok() { return p; } } PathBuf::from(format!("/data/data/{ANDROID_PACKAGE}/files")) } /// Thumbnail cache dir: /.thumbs. fn thumbs_dir() -> PathBuf { let d = external_files_dir().join(".thumbs"); let _ = std::fs::create_dir_all(&d); d } /// Cache key: sha1(path | mtime | size | target) — invalidates automatically /// when the file changes or the render target moves. fn thumb_cache_path(path: &str, mtime: i64, size: i64) -> PathBuf { let mut h = Sha1::new(); h.update(path.as_bytes()); h.update(mtime.to_le_bytes()); h.update(size.to_le_bytes()); h.update(THUMB_TARGET.to_le_bytes()); let hex: String = h.finalize().iter().map(|b| format!("{b:02x}")).collect(); thumbs_dir().join(format!("{hex}.jpg")) } /// Cache ceiling; oldest entries evicted when exceeded. const THUMB_CACHE_MAX: u64 = 700 * 1024 * 1024; fn prune_cache(dir: &Path, max: u64) { let mut files: Vec<(std::time::SystemTime, PathBuf)> = std::fs::read_dir(dir) .into_iter() .flatten() .flatten() .filter_map(|e| { let m = e.metadata().ok()?; let ft = m.modified().ok()?; Some((ft, e.path())) }) .collect(); let total: u64 = files.iter().map(|(_, p)| std::fs::metadata(p).map(|m| m.len()).unwrap_or(0)).sum(); if total <= max { return; } files.sort(); // oldest first let mut to_drop = total - max; for (_, p) in files { if to_drop == 0 { break; } if let Ok(m) = std::fs::metadata(&p) { let _ = std::fs::remove_file(&p); to_drop = to_drop.saturating_sub(m.len()); } } } // --------------------------------------------------------------------------- // the gallery:// protocol /// Mem → disk → decode → serve a grid thumbnail for `id`. Returns JPEG /// bytes. Each tier below is only reached when the one above is cold, and /// the prefetch worker keeps the top two warm ahead of the viewport. fn serve_thumb(id: i64) -> Result, String> { tr(&format!("req {id}")); if let Some(b) = mem_get(id) { tr(&format!("req {id} hit-mem")); return Ok(b); } let meta = { let m = META.read().unwrap(); m.get(&id).cloned().ok_or_else(|| format!("no item {id}"))? }; let cp = thumb_cache_path(&meta.path, meta.mtime, meta.size); if cp.is_file() { CACHE_HITS.fetch_add(1, Ordering::Relaxed); let b = std::fs::read(&cp).map_err(|e| format!("cache read: {e}"))?; if !b.is_empty() { mem_put(id, b.clone()); tr(&format!("req {id} hit-disk")); return Ok(b); } } // Confirmed corrupt (BROKEN_AFTER consecutive failures): placeholder // forever, no retry loop. A file still being written is NOT broken // yet — it gets queued and retried until it decodes. if is_broken(id) { tr(&format!("req {id} broken")); return Ok(PLACEHOLDER.as_slice().to_vec()); } // Cold miss: NEVER decode here (main thread!). Queue it URGENT (front // of the queue) and return the 1x1 placeholder in ~1 ms. The worker // emits `thumb-ready`; JS re-requests; that hit is instant. let idx = ID_TO_IDX.read().unwrap().get(&id).copied().unwrap_or(u32::MAX); { let mut q = Q.lock().unwrap(); let mut inq = INFLIGHT.lock().unwrap(); if !inq.contains(&id) && q.len() < Q_CAP { inq.insert(id); q.push_front(Job { id, idx, urgent: true, full: false }); } } Q_CV.notify_one(); tr(&format!("req {id} placeholder")); Ok(PLACEHOLDER.as_slice().to_vec()) } /// Serve the original file (viewer). NEVER reads the file on the main /// thread: a 30-50 MB flash read there would freeze the whole WebView. /// Mem hit → real bytes; miss → enqueue an urgent FULL job (front of the /// queue, ahead of any grid work) + 1x1 placeholder. JS already showed /// the grid thumb; when the read lands it swaps in the full-res on /// `full-ready`. fn serve_full(id: i64) -> Result<(Vec, &'static str), String> { let meta = META.read().unwrap().get(&id).cloned().ok_or_else(|| format!("no item {id}"))?; let ct = match meta.path.rsplit('.').next().unwrap_or("").to_ascii_lowercase().as_str() { "png" => "image/png", "webp" => "image/webp", "gif" => "image/gif", _ => "image/jpeg", }; if is_broken(id) { return Ok((PLACEHOLDER.as_slice().to_vec(), "image/jpeg")); } if let Some(b) = FULL_MEM.lock().unwrap().get(id) { tr(&format!("full {id} hit-mem")); return Ok((b, ct)); } tr(&format!("full {id} placeholder")); let idx = ID_TO_IDX.read().unwrap().get(&id).copied().unwrap_or(u32::MAX); { let mut q = Q.lock().unwrap(); let mut inq = INFLIGHT.lock().unwrap(); if !inq.contains(&id) && q.len() < Q_CAP { inq.insert(id); q.push_front(Job { id, idx, urgent: true, full: true }); } } Q_CV.notify_one(); Ok((PLACEHOLDER.as_slice().to_vec(), "image/jpeg")) } /// `gallery://thumb/` and `gallery://full/`. fn serve_sync(ctx: tauri::UriSchemeContext<'_, tauri::Wry>, request: http::Request>) -> http::Response> { let _ = ctx; handle_gallery_inner(request) } fn handle_gallery_inner(request: http::Request>) -> http::Response> { // request.uri() looks like "gallery/thumb/123" (scheme stripped by wry on // Android the path comes through as "thumb/123"); be liberal about it. // wry (Android) reverts the workaround URL to `gallery://localhost/thumb/1` // — strip scheme, then the `localhost` authority, then any '/'. let uri = request.uri().to_string(); let path = uri .trim_start_matches("gallery://") .trim_start_matches("gallery/") .trim_start_matches("localhost/") .trim_start_matches('/'); let mut parts = path.splitn(2, '/'); let kind = parts.next().unwrap_or(""); let id: i64 = match parts.next().and_then(|s| s.split('?').next()).and_then(|s| s.parse().ok()) { Some(i) => i, None => { return http::Response::builder() .status(404) .body(b"bad request".to_vec()) .unwrap(); } }; let (status, ct, bytes) = match kind { "thumb" => match serve_thumb(id) { Ok(b) => (200, "image/jpeg", b), Err(e) => { log::warn!("{e}"); (500, "text/plain", e.into_bytes()) } }, "full" => match serve_full(id) { Ok((b, ct)) => (200, ct, b), Err(e) => { log::warn!("{e}"); (500, "text/plain", e.into_bytes()) } }, _ => (404, "text/plain", format!("unknown {kind}").into_bytes()), }; http::Response::builder() .status(status) .header("content-type", ct) .header("cache-control", "max-age=31536000") .body(bytes) .unwrap() } // --------------------------------------------------------------------------- // file list + scan /// Emit the current id list (display order: mtime desc, path asc — the DB /// query already does that) to the webview. fn emit_files(app: &tauri::AppHandle, scanning: bool, found: usize) { let ids: Vec = { let m = META.read().unwrap(); // rebuild display order from the map (mtime desc, path asc) let mut v: Vec = m.values().cloned().collect(); v.sort_by(|a, b| b.mtime.cmp(&a.mtime).then_with(|| a.path.cmp(&b.path))); v.into_iter().map(|x| x.id).collect() }; *DISPLAY_IDS.write().unwrap() = ids.clone(); *ID_TO_IDX.write().unwrap() = ids.iter().enumerate().map(|(i, &id)| (id, i as u32)).collect(); tr(&format!("emit n={} scanning={scanning}", ids.len())); let _ = app.emit("gallery-files", FileList { ids, scanning, found }); } #[derive(Serialize, Clone)] struct FileList { ids: Vec, scanning: bool, found: usize, } /// Load the DB (reusing the existing file if present) into META and return /// a live connection. fn load_meta(conn: &mut db::Connection) -> Result<(), rusqlite::Error> { let mut m = META.write().unwrap(); m.clear(); let mut st = conn.prepare("SELECT rowid, path, mtime, size FROM items ORDER BY mtime DESC, path ASC")?; let rows = st.query_map([], |r| { Ok(Meta { id: r.get(0)?, path: r.get(1)?, mtime: r.get(2)?, size: r.get(3)?, }) })?; for r in rows { let meta = r?; m.insert(meta.id, meta); } log::info!("loaded {} items from DB", m.len()); Ok(()) } /// Walk the roots once, streaming progress emits. /// A filesystem error fails the walk (see scan.rs) — the caller must /// then NOT apply a partial result to the DB. fn walk_once(app: &tauri::AppHandle) -> std::io::Result> { let roots = scan::roots(); let last_emit = std::cell::Cell::new(std::time::Instant::now()); scan::walk_stream(&roots, &|batch| { if last_emit.get().elapsed() < std::time::Duration::from_millis(500) { return; } last_emit.set(std::time::Instant::now()); emit_files(app, true, batch.len()); }) } /// Set while a full rescan's walk→apply is in flight. The watch loop /// must NOT consume MediaStore payloads meanwhile: a delete consumed /// mid-walk would race the walk's in-memory snapshot (the walk would /// re-add the row the delete already processed). Payloads simply wait /// as files and are drained by the rescan's post-apply drain. static RESCANNING: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); struct RescanGuard; impl RescanGuard { fn new() -> Self { RESCANNING.store(true, std::sync::atomic::Ordering::Relaxed); Self } } impl Drop for RescanGuard { fn drop(&mut self) { RESCANNING.store(false, std::sync::atomic::Ordering::Relaxed); } } /// Full rescan: walk the roots, diff against the DB (insert new / /// update changed / delete removed), refresh META, emit the list. /// Used by the startup scan AND the 5-minute reconciliation pass. /// If the walk fails, nothing is applied — an error is not "those /// files are gone"; the next pass retries. fn rescan(app: &tauri::AppHandle) { let _guard = RescanGuard::new(); let entries = match walk_once(app) { Ok(e) => e, Err(e) => { tr(&format!("rescan: walk FAILED ({e}) — not applying")); log::warn!("rescan walk failed: {e}"); return; } }; let mut conn = match db::open(&db::db_path()) { Ok(c) => c, Err(e) => { log::warn!("rescan db: {e}"); return; } }; if let Err(e) = db::apply(&mut conn, &entries) { log::warn!("rescan apply: {e}"); return; } if let Err(e) = load_meta(&mut conn) { log::warn!("rescan meta: {e}"); return; } emit_files(app, false, entries.len()); log::info!("rescan: {} files", entries.len()); // Draining happens in the watch loop's next tick: the RESCANNING // flag lifted at drop time guarantees any payload consumed from now // on is applied AFTER this walk's apply, so mid-walk deletions // always win over the stale snapshot. (The 5-min pass catches any // straggler in the worst-case millisecond window.) } fn spawn_scan(app: tauri::AppHandle) { std::thread::spawn(move || rescan(&app)); } fn mtime_ns(m: &std::fs::Metadata) -> i64 { m.modified() .ok() .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) .map(|d| d.as_nanos() as i64) .unwrap_or(0) } /// Tree signature: (dir, mtime) for every directory under the roots. /// Verified on the phone: Android's FUSE DOES update a directory's /// mtime when files are added/removed inside it (stat before/after a /// copy showed the change). Cost per tick: one stat per DIRECTORY and /// one readdir per directory — no per-FILE stat. (The old version /// called is_dir() on every entry: 10k stats per second on a 10k-file /// Camera dir, which put a 10 s I/O floor under every event.) fn dir_sig(roots: &[PathBuf]) -> HashMap { let mut out = HashMap::new(); let mut stack: Vec = roots.iter().filter(|r| r.is_dir()).cloned().collect(); while let Some(dir) = stack.pop() { let Ok(m) = std::fs::metadata(&dir) else { continue }; out.insert(dir.clone(), mtime_ns(&m)); match std::fs::read_dir(&dir) { Ok(rd) => { for ent in rd.flatten() { let p = ent.path(); // file_type() uses the d_type from getdents (no // syscall); is_dir() would stat the entry. let Ok(ft) = ent.file_type() else { continue }; if ft.is_symlink() || !ft.is_dir() { continue; } if p.file_name().map_or(false, |n| n.to_string_lossy().starts_with('.')) { continue; } stack.push(p); } } // A directory we cannot read this tick: skip it; the next // tick or the full pass will pick up anything that changed. Err(_) => {} } } out } /// Above 2000 files, stat'ing every file on FUSE costs tens of seconds; /// for such directories a change event is handled readdir-only (adds and /// deletes are still detected exactly; in-place edits wait for the /// periodic full pass). Small directories are walked with stat so edits /// are caught too. const BIG_DIR: usize = 2000; /// Incremental rescan limited to the changed directory subtrees. /// Everything outside the changed trees keeps its DB metadata, so the /// walk cost is proportional to the change, not the library. fn rescan_subtrees(app: &tauri::AppHandle, changed: &[PathBuf]) { let mut conn = match db::open(&db::db_path()) { Ok(c) => c, Err(e) => { log::warn!("incr db: {e}"); return; } }; let existing = match db::list_entries(&mut conn) { Ok(v) => v, Err(e) => { log::warn!("incr list: {e}"); return; } }; let mut fresh: Vec = Vec::new(); for d in changed { let prefix = d.to_string_lossy().into_owned() + "/"; let under: Vec<&scan::Entry> = existing.iter().filter(|e| e.path.starts_with(&prefix)).collect(); let known: HashMap<&str, &scan::Entry> = under.iter().map(|e| (e.path.as_str(), *e)).collect(); if under.len() <= BIG_DIR { // Small: full walk with stat — adds, deletes AND edits. if let Err(e) = scan::walk_dir_into(d, &mut fresh) { tr(&format!("incr: walk {d:?} FAILED ({e}) — not applying")); log::warn!("incr walk failed: {e}"); return; // a failed subtree must not be applied as empty } } else { // Huge: readdir-only. New files get stat'd; known files keep // their DB mtime/size (an in-place edit there is caught by // the 5-minute full pass). if let Err(e) = scan::walk_dir_nostats(d, &mut |p| { let ps = p.to_string_lossy(); let entry = match known.get(ps.as_ref()) { Some(old) => scan::Entry { path: ps.into_owned(), mtime: old.mtime, size: old.size, }, None => match std::fs::metadata(p) { Ok(md) => scan::Entry { path: ps.into_owned(), mtime: mtime_ns(&md), size: md.len() as i64, }, Err(_) => return, // file vanished mid-walk — normal race }, }; fresh.push(entry); }) { tr(&format!("incr: walk {d:?} FAILED ({e}) — not applying")); log::warn!("incr walk failed: {e}"); return; } } } let mut all = existing .into_iter() .filter(|e| !changed.iter().any(|d| e.path.starts_with(&*(d.to_string_lossy().into_owned() + "/")))) .collect::>(); let added = fresh.len(); all.extend(fresh); if let Err(e) = db::apply(&mut conn, &all) { log::warn!("incr apply: {e}"); return; } if let Err(e) = load_meta(&mut conn) { log::warn!("incr meta: {e}"); return; } emit_files(app, false, all.len()); tr(&format!("watch: incr changed={} fresh={added}", changed.len())); } // --------------------------------------------------------------------- // MediaStore fast path (Android). MainActivity's ContentObserver gets // the CHANGED URI(s) directly from the OS; it queries that exact row // (a few ms) and drops a one-line payload file, then wakes us with a // zero-arg JNI call. No filesystem scanning on the event path: the // payload IS the change (added, modified, or deleted). // // Payload line formats (one per file, one file per event — per-event // files so a burst never overwrites an unread event): // A||| row exists → upsert // D| row gone → delete by id // --------------------------------------------------------------------- /// Consume pending event files. Returns true when at least one change /// line was applied (the watch loop uses this to decide whether the /// OS event was fully handled by the payload path). #[cfg(target_os = "android")] fn apply_mediasotre_payload(app: &tauri::AppHandle) -> bool { // Consume every pending .ms_recent.* event file (oldest first). let dir = external_files_dir(); let Ok(rd) = std::fs::read_dir(&dir) else { return false }; let mut files: Vec = rd .flatten() .filter(|e| { e.file_name() .to_string_lossy() .starts_with(".ms_recent.") }) .map(|e| e.path()) .collect(); if files.is_empty() { return false; } files.sort(); let mut applied = false; for f in files { let name = f.file_name().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default(); let Ok(text) = std::fs::read_to_string(&f) else { continue }; let _ = std::fs::remove_file(&f); if text.is_empty() { continue; } tr(&format!("mediastore event {name}: {} lines", text.lines().count())); applied |= mediastore_apply(app, &text); } applied } #[cfg(not(target_os = "android"))] fn apply_mediasotre_payload(_app: &tauri::AppHandle) -> bool { false } #[cfg(target_os = "android")] fn mediastore_apply(app: &tauri::AppHandle, text: &str) -> bool { let mut conn = match db::open(&db::db_path()) { Ok(c) => c, Err(e) => { log::warn!("mediastore db: {e}"); return false; } }; let mut changed = 0usize; let mut removed = 0usize; for line in text.lines() { let mut f = line.splitn(5, '|'); let tag = f.next().unwrap_or(""); match tag { // A|media_id|path|size "A" => { let Ok(id) = f.next().unwrap_or("").parse::() else { continue }; let Some(path) = f.next() else { continue }; if path.is_empty() || !scan::is_image_path(path) { continue; } let _size = f.next().and_then(|s| s.parse::().ok()); // disk stat is the truth // Always record the id→path mapping: a later delete // event must be resolvable even for rows created by a // filesystem walk (no media_id) or never created at all // (the size-0 mid-write row below). if let Err(e) = db::map_media(&mut conn, id, path) { log::warn!("mediastore map: {e}"); } // Trust the DISK, not the provider row: the scanner can // leave _size NULL, and a file still being written is // fine — its finalization event re-upserts with the new // size (content change resets the decode state). let Ok(md) = std::fs::metadata(path) else { continue }; let mtime = mtime_ns(&md); match db::upsert_ms(&mut conn, path, mtime, md.len() as i64, id) { Ok(content_changed) => { // Row id → the rowid is what the grid/queue use; // clear decode-failure state only after a real // content change (a still-being-written file // that failed early gets fresh retries). let rowid: Option = conn .query_row( "SELECT rowid FROM items WHERE path = ?1", rusqlite::params![path], |r| r.get(0), ) .ok(); if content_changed { if let Some(rid) = rowid { FAILS.lock().unwrap().remove(&rid); } } changed += 1; } Err(e) => log::warn!("mediastore upsert: {e}"), } } // D|media_id — the row is gone from MediaStore; resolve the // path (items row first, then the add-event mapping) and // delete by path: the items row may have been created by a // filesystem walk, which doesn't store the media id. "D" => { let Ok(id) = f.next().unwrap_or("").parse::() else { continue }; let path = match db::path_for_media(&mut conn, id) { Ok(p) => p, Err(e) => { log::warn!("mediastore dlookup: {e}"); continue; } }; if let Some(path) = path { // clear decode-failure state before deleting if let Some(rid) = conn .query_row( "SELECT rowid FROM items WHERE path = ?1", rusqlite::params![path], |r| r.get(0), ) .ok() { FAILS.lock().unwrap().remove(&rid); } if let Ok(n) = db::delete_by_path(&mut conn, &path) { removed += n; if n > 0 { tr(&format!("mediastore del id={id} {path}")); } } let _ = db::unmap_media(&mut conn, id); } } "D_PATH" => { let Some(path) = f.next() else { continue }; if let Some(rid) = conn .query_row( "SELECT rowid FROM items WHERE path = ?1", rusqlite::params![path], |r| r.get(0), ) .ok() { FAILS.lock().unwrap().remove(&rid); } if let Ok(n) = db::delete_by_path(&mut conn, path) { removed += n; if n > 0 { tr(&format!("mediastore del path={path}")); } } } _ => {} } } if changed + removed > 0 { tr(&format!("mediastore applied +{changed} -{removed}")); if let Err(e) = load_meta(&mut conn) { log::warn!("mediastore meta: {e}"); } emit_files(app, false, META.read().unwrap().len()); } changed + removed > 0 } /// Watcher wake events. #[derive(Debug, Clone, Copy)] enum WEvent { /// The OS said the filesystem changed (ContentObserver on Android, /// inotify/FSEvents on desktop). Fs, /// Periodic full-pass timer. Full, } static WATCH_TX: LazyLock>>> = LazyLock::new(|| Mutex::new(None)); /// Android: called by MainActivity's ContentObserver via JNI (and after /// each MediaStore fast-path payload drop). Signature: (JNIEnv*, /// jobject) for an instance method. One channel send — nothing heavy, /// never blocks the main thread. #[cfg(target_os = "android")] #[no_mangle] pub extern "system" fn Java_wow_gmp_gallery_MainActivity_nativeFsChange( _env: *mut std::ffi::c_void, _obj: *mut std::ffi::c_void, ) { if let Some(tx) = WATCH_TX.lock().unwrap().as_ref() { let _ = tx.send(WEvent::Fs); } } /// Desktop: inotify (Linux) / FSEvents (macOS) on the scan roots. #[cfg(not(target_os = "android"))] fn spawn_fs_watcher(tx: mpsc::Sender) { std::thread::spawn(move || { use notify::{RecursiveMode, Watcher}; let (wt, wm) = mpsc::channel(); let mut w = match notify::recommended_watcher(move |res: Result| { if res.is_ok() { let _ = wt.send(()); // consumer coalesces + diffs } }) { Ok(w) => w, Err(e) => { log::warn!("fs watcher: {e}"); return; } }; for r in scan::roots() { if r.is_dir() { let _ = w.watch(&r, RecursiveMode::Recursive); } } let _keep = wm; // drop = stop watching loop { if wt.recv().is_err() { break; } let _ = tx.send(WEvent::Fs); } }); } fn collect_watched_dirs(roots: &[PathBuf]) -> HashSet { let mut dirs = HashSet::new(); for r in roots { if r.is_dir() { dirs.insert(r.clone()); if let Ok(rd) = std::fs::read_dir(r) { for ent in rd.flatten() { if let Ok(ft) = ent.file_type() { if ft.is_dir() { let p = ent.path(); if !p.file_name().map_or(false, |n| n.to_string_lossy().starts_with('.')) { dirs.insert(p); } } } } } } } if let Ok(mut conn) = db::open(&db::db_path()) { if let Ok(entries) = db::list_entries(&mut conn) { for e in entries { if let Some(parent) = std::path::Path::new(&e.path).parent() { dirs.insert(parent.to_path_buf()); } } } } dirs } fn fast_dir_sig(dirs: &HashSet) -> HashMap { let mut out = HashMap::new(); for d in dirs { if let Ok(m) = std::fs::metadata(d) { out.insert(d.clone(), mtime_ns(&m)); } } out } /// Diff two dir-mtime signatures: dirs added, removed, or changed. fn changed_dirs( old: &HashMap, next: &HashMap, ) -> Vec { let mut v: Vec = next .iter() .filter(|(d, t)| old.get(d.as_path()) != Some(t)) .map(|(d, _)| d.clone()) .collect(); for d in old.keys() { if !next.contains_key(d) { v.push(d.clone()); } } v } /// Live file watching — EVENT-DRIVEN, no tree polling: /// - Android: MainActivity's MediaStore ContentObserver → JNI → this /// thread (camera saves, media-scanner indexing, indexed writes). /// - Desktop: inotify/FSEvents via `notify` on the scan roots. /// - On wake: coalesce the burst, debounce for the write to settle, /// diff dir mtimes (a few stats), and rescan ONLY the changed /// subtrees. A 5-minute full pass is the safety net (in-place edits /// in the huge dir, anything the OS never reported). /// - The 1 s recv timeout doubles as a cheap secondary detector /// (dir-mtime diff, ~0.05 ms): raw out-of-band copies that the media /// scanner hasn't indexed yet are still caught within ~1 s. fn watch_files(app: tauri::AppHandle) { let (tx, rx) = mpsc::channel::(); *WATCH_TX.lock().unwrap() = Some(tx.clone()); std::thread::spawn(move || { loop { std::thread::sleep(std::time::Duration::from_secs(300)); let _ = tx.send(WEvent::Full); } }); #[cfg(not(target_os = "android"))] spawn_fs_watcher(tx); let mut watched_dirs = collect_watched_dirs(&scan::roots()); let mut sig = fast_dir_sig(&watched_dirs); let mut baseline = true; let mut last_dir_refresh = std::time::Instant::now(); let mut last_full = std::time::Instant::now(); loop { let woke = match rx.recv_timeout(std::time::Duration::from_millis(100)) { Ok(e) => Some(e), Err(mpsc::RecvTimeoutError::Timeout) => None, Err(mpsc::RecvTimeoutError::Disconnected) => break, }; let do_full = matches!(woke, Some(WEvent::Full)); if let Some(WEvent::Fs) = woke { #[cfg(target_os = "android")] { apply_mediasotre_payload(&app); continue; } #[cfg(not(target_os = "android"))] { while rx.try_recv().is_ok() {} std::thread::sleep(std::time::Duration::from_millis(600)); while rx.try_recv().is_ok() {} } } if do_full { tr("watch: full rescan (5min)"); rescan(&app); watched_dirs = collect_watched_dirs(&scan::roots()); sig = fast_dir_sig(&watched_dirs); last_full = std::time::Instant::now(); continue; } // Refresh watched dirs periodically (every 30s) to pick up new subdirs if last_dir_refresh.elapsed() >= std::time::Duration::from_secs(30) { watched_dirs = collect_watched_dirs(&scan::roots()); last_dir_refresh = std::time::Instant::now(); } // MediaStore event path (Android): MainActivity drops a one-line // payload per changed URI and wakes us via nativeFsChange. let _applied_ms = if RESCANNING.load(std::sync::atomic::Ordering::Relaxed) { false } else { apply_mediasotre_payload(&app) }; // Fast directory mtime-diff watch (Android and desktop): catches // out-of-band filesystem adds/removes within ~1 second without // scanning 10k files. let next = fast_dir_sig(&watched_dirs); let changed = if baseline { baseline = false; Vec::new() } else { changed_dirs(&sig, &next) }; sig = next; if !changed.is_empty() { tr(&format!("watch: {} changed dir(s)", changed.len())); rescan_subtrees(&app, &changed); watched_dirs = collect_watched_dirs(&scan::roots()); sig = fast_dir_sig(&watched_dirs); } } } // --------------------------------------------------------------------------- #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { #[cfg(target_os = "android")] init_logcat(); tauri::Builder::default() .register_uri_scheme_protocol("gallery", |ctx, request| { // Sync protocol: the handler runs on tauri's io thread pool, // off the webview main thread; no response channel involved. serve_sync(ctx, request) }) .setup(|app| { let handle = app.handle().clone(); #[cfg(target_os = "android")] { // Tauri's document_dir = getExternalFilesDir(DOCUMENTS); // calling the resolver is what creates the dir. match app.path().document_dir() { Ok(dir) => { init_fr_logger(&dir); init_trace(&dir); } Err(e) => log::error!("document_dir: {e}"), } } log::info!("setup (mobile={})", cfg!(target_os = "android")); // FR batch → file (same as the harnesses). let _ = app.listen("gtauri-fr", |event| { if let Some(tx) = FR_TX.lock().unwrap().as_ref() { // payload arrives JSON-encoded (quoted, \n-escaped) — decode it let s = serde_json::from_str::(event.payload()) .unwrap_or_else(|_| event.payload().to_string()); let _ = tx.send(s); } }); let _ = app.listen("gtauri-log", |event| { // JS-side trace lines (JSON-encoded string payload) let s = serde_json::from_str::(event.payload()) .unwrap_or_else(|_| event.payload().to_string()); tr(&s); }); // JS ready handshake: the WebView's page loads ~2-3 s after // process start, while the first gallery-files emit can land // 10 ms in (small/fast DB) and be lost. On handshake, re-emit // the current state so the grid always gets its initial list. let _ = app.listen("gtauri-js-ready", move |event| { let _ = event; let h = APP_HANDLE.lock().unwrap().clone(); if let Some(h) = h { tr("js ready — re-emitting state"); emit_files(&h, false, META.read().unwrap().len()); } }); // Scroll anchor from JS (throttled to ~7 Hz): `a` = index of the // first visible grid cell. REPLACES the prefetch window with // [a, a+60) — the visible band plus ~13 rows ahead. Stale // off-screen work from a previous position is dropped, never // allowed to run ahead of what is about to be seen. let _ = app.listen("gtauri-scroll", |event| { #[derive(Deserialize)] struct Anchor { a: usize, } if let Ok(x) = serde_json::from_str::(event.payload()) { tr(&format!("anchor a={}", x.a)); // Window starts AT the first visible row, not behind it: // Chromium's lazy loader can lag a fast jump by a beat, // and the workers warming the visible band make those // requests pure mem hits. (Warm cells dedupe out.) // set_prefetch_window also VPORTs and drops all work // left more than STALE_ROWS behind the new position. set_prefetch_window(x.a, x.a + 60); } }); *APP_HANDLE.lock().unwrap() = Some(handle.clone()); // Parallel decode pool: one worker could not keep up with a // fast long scroll (observed 2400-deep backlog draining at // ~3 cells/s). Four workers decode ~12 full-res photos/s. for _ in 0..4 { std::thread::spawn(decode_worker); } std::thread::spawn(heartbeat); // Cache prune on startup (cheap: one readdir). std::thread::spawn(|| prune_cache(&thumbs_dir(), THUMB_CACHE_MAX)); // Initial meta from the existing DB (instant grid — "show what // is available immediately"), then the scan refreshes it. let path = db::db_path(); match db::open(&path) { Ok(mut conn) => { if let Err(e) = load_meta(&mut conn) { log::error!("load meta: {e}"); } } Err(e) => log::warn!("no DB yet ({e}); scan will create it"), } emit_files(&handle, true, META.read().unwrap().len()); // Warm the first two screens before Chromium's lazy loader gets // to them (it requests ~30 at once; we'd rather be ahead). set_prefetch_window(0, 36); // (The startup MediaStore fast path is triggered by // MainActivity.onCreate — it pushes via // nativeMediaStoreResults ~1.5 s in.) spawn_scan(handle.clone()); // New photos appear in the grid ~2 s after being written. std::thread::spawn(move || watch_files(handle)); Ok(()) }) .run(tauri::generate_context!()) .expect("tauri run"); }