gtauri: URI-driven MediaStore events; remove truncation machinery
Watching is now exactly what the OS tells us, with no filesystem scanning on the event path: - MainActivity's ContentObserver receives the changed URI from Android. It queries that ONE row (~ms, no debounce) and drops a one-line payload: A|media_id|path|size (row exists) or D|media_id (row gone). Per-event files (.ms_recent.N, never overwritten) so a burst cannot clobber an unread event. A zero-arg JNI call wakes Rust; a 1 s backstop also drains the queue, so a lost wake self-heals in a second. - Rust applies the payload directly: upsert with media_id stored, or delete resolved by id (items row first, then a media_map table that records id→path for EVERY add event — including rows created by a filesystem walk, which doesn't know the id). Content changes reset decode-failure state so a still-being-written photo retries. - The disk is the truth: the payload size is ignored in favor of a stat (the scanner can leave _size NULL). - The full FS walk remains as startup baseline + 5-min reconciliation, and the mtime-diff incremental rescan remains as a second independent path. Verified on the emulator: add → grid in ~2 s (incl. OS scanner), delete → grid in ~2 s via media_id, missed wake drained in 1 s. Truncation machinery REMOVED (per user: FUSE is reliable; the 'truncations' were counting errors and a stale-DB cross-check firing on legitimate changes): the two-strike mass-delete guard, the walked_max watermark, the layer-2 re-walk, the per-directory stale cross-check, and the read_dir_full retries are all gone. The one invariant that replaces them: a walk that hits ANY filesystem error fails and is NOT applied (an error is not 'those files are gone'). The phone DB now converges to the exact disk count (8,439 = 8,399 Camera + 27 Screenshots + 11 Messaging + 2 Bluesky); the earlier 10,166/8,438 swings were real disk-state changes, not truncation. Emulator display fixed: the AVD image ships with debug.hwui.drawing_enabled=0 (every window renders black, HWUI drawing disabled system-wide). 'adb shell setprop debug.hwui.drawing_enabled 1' after each boot restores rendering — it is a runtime prop, so it must be re-set on every cold boot.
This commit is contained in:
parent
1bc1d6beb3
commit
bb349dac82
|
|
@ -1,6 +1,7 @@
|
|||
package wow.gmp.gallery
|
||||
|
||||
import android.Manifest
|
||||
import android.content.ContentUris
|
||||
import android.database.ContentObserver
|
||||
import android.net.Uri
|
||||
|
||||
|
|
@ -17,11 +18,22 @@ import java.util.concurrent.Executors
|
|||
class MainActivity : TauriActivity() {
|
||||
private var observer: ContentObserver? = null
|
||||
|
||||
// Runtime request for the media-read permission. The MediaStore fast
|
||||
// path (instant camera-photo appearance) needs it; WITHOUT it the
|
||||
// query returns nothing and new photos only appear after the slow
|
||||
// full FUSE walk (~10-30 s on a big library). Declaring it in the
|
||||
// manifest is not enough on Android 13+.
|
||||
// MediaStore event path: the OS tells us WHICH URI changed; we query
|
||||
// that one row (~ms) and hand the result to Rust via a payload file +
|
||||
// a zero-arg JNI wake. No filesystem scanning, no debounce: a
|
||||
// just-taken camera photo is pushed to the grid immediately.
|
||||
private val fsExecutor = Executors.newSingleThreadExecutor()
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
private var seq = 0L
|
||||
|
||||
// Rust side (this app's JNI library, loaded by TauriActivity): wakes
|
||||
// the file-watcher thread. One JNI call + one channel send — nothing
|
||||
// heavy ever runs on the main thread.
|
||||
external fun nativeFsChange()
|
||||
|
||||
// Runtime request for the media-read permission. The MediaStore path
|
||||
// needs it; without it the queries return nothing. Declaring it in
|
||||
// the manifest is not enough on Android 13+.
|
||||
private val permLauncher = registerForActivityResult(
|
||||
ActivityResultContracts.RequestMultiplePermissions()
|
||||
) { grantOneMoreChance() }
|
||||
|
|
@ -40,9 +52,8 @@ class MainActivity : TauriActivity() {
|
|||
}
|
||||
}
|
||||
|
||||
// If the user denied, the app still works — photos arrive via the
|
||||
// slow walk. Retry the request once per process start only (no
|
||||
// nagging loop).
|
||||
// If the user denied, the app still works (FS walk fallback) — retry
|
||||
// once per process start only (no nagging loop).
|
||||
private var permRetried = false
|
||||
private fun grantOneMoreChance() {
|
||||
if (!permRetried && checkSelfPermission(mediaPermission()) != PackageManager.PERMISSION_GRANTED) {
|
||||
|
|
@ -51,31 +62,67 @@ class MainActivity : TauriActivity() {
|
|||
}
|
||||
}
|
||||
|
||||
// MediaStore fast path: when the OS reports a media change (or right
|
||||
// after startup), query MediaStore on a background thread for images
|
||||
// indexed in the last ~2 min and push them to Rust via JNI. This is
|
||||
// what makes a just-taken camera photo appear in ~1 s instead of
|
||||
// waiting for the ~10 s FUSE re-read of a huge directory.
|
||||
private val fsExecutor = Executors.newSingleThreadExecutor()
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
private var fsScheduled = false
|
||||
/** Write one payload line to a fresh per-event file and wake Rust.
|
||||
* Per-event files (never overwriting) so a burst of changes cannot
|
||||
* clobber an event Rust has not consumed yet. */
|
||||
private fun deliver(line: String) {
|
||||
try {
|
||||
val dir = getExternalFilesDir(null) ?: return
|
||||
seq += 1
|
||||
val f = java.io.File(dir, ".ms_recent." + seq)
|
||||
f.writeText(line + "\n")
|
||||
nativeFsChange()
|
||||
} catch (e: Exception) {
|
||||
android.util.Log.w("gallery-ms", "deliver: " + e)
|
||||
}
|
||||
}
|
||||
|
||||
// Rust side (this app's JNI library, loaded by TauriActivity): wakes
|
||||
// the file-watcher thread. One JNI call + one channel send — nothing
|
||||
// heavy ever runs on the main thread.
|
||||
external fun nativeFsChange()
|
||||
/** Handle ONE changed URI: query that exact MediaStore row.
|
||||
* Row exists → A (added/modified). Row gone → D (deleted). */
|
||||
private fun handleUri(uri: Uri?) {
|
||||
val base = MediaStore.Images.Media.EXTERNAL_CONTENT_URI
|
||||
if (uri == null || uri == base || uri.lastPathSegment == null) {
|
||||
// Generic change (media scan finished, bulk op with no per-row
|
||||
// URI): fall back to the recent-window query.
|
||||
windowQuery()
|
||||
return
|
||||
}
|
||||
val id = try { ContentUris.parseId(uri) } catch (e: Exception) { -1L }
|
||||
try {
|
||||
contentResolver.query(uri, null, null, null, null)?.use { c ->
|
||||
if (c.moveToFirst()) {
|
||||
val iId = c.getColumnIndexOrThrow(MediaStore.Images.Media._ID)
|
||||
val iData = c.getColumnIndex(MediaStore.Images.Media.DATA)
|
||||
val iRel = c.getColumnIndex(MediaStore.Images.Media.RELATIVE_PATH)
|
||||
val iSize = c.getColumnIndexOrThrow(MediaStore.Images.Media.SIZE)
|
||||
var path = if (iData >= 0) c.getString(iData) else null
|
||||
if (path.isNullOrEmpty() && iRel >= 0) {
|
||||
val rel = c.getString(iRel)
|
||||
if (!rel.isNullOrEmpty()) path = "/storage/emulated/0/" + rel
|
||||
}
|
||||
if (path.isNullOrEmpty()) return
|
||||
val size = c.getLong(iSize)
|
||||
deliver("A|" + c.getLong(iId) + "|" + path + "|" + size)
|
||||
} else {
|
||||
// The row is gone → the media file was deleted.
|
||||
if (id >= 0) deliver("D|" + id)
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
android.util.Log.w("gallery-ms", "handleUri: " + e)
|
||||
if (id >= 0) deliver("D|" + id)
|
||||
}
|
||||
}
|
||||
|
||||
// Rust side reads the payload file (getExternalFilesDir/.ms_recent)
|
||||
// when nativeFsChange wakes it. No string crosses JNI.
|
||||
|
||||
/** Query MediaStore for recently-indexed images and push to Rust. */
|
||||
private fun pushRecent() {
|
||||
/** Fallback for generic (no-URI) change notifications: query rows
|
||||
* indexed in the last ~2 min. */
|
||||
private fun windowQuery() {
|
||||
try {
|
||||
val since = System.currentTimeMillis() / 1000 - 120
|
||||
val sb = StringBuilder()
|
||||
val rows = contentResolver.query(
|
||||
MediaStore.Images.Media.EXTERNAL_CONTENT_URI,
|
||||
arrayOf(
|
||||
MediaStore.Images.Media._ID,
|
||||
MediaStore.Images.Media.DATA,
|
||||
MediaStore.Images.Media.RELATIVE_PATH,
|
||||
MediaStore.Images.Media.SIZE,
|
||||
|
|
@ -84,81 +131,52 @@ class MainActivity : TauriActivity() {
|
|||
"${MediaStore.Images.Media.DATE_ADDED} >= ?",
|
||||
arrayOf(since.toString()),
|
||||
"${MediaStore.Images.Media.DATE_ADDED} DESC"
|
||||
)
|
||||
if (rows == null) {
|
||||
android.util.Log.w("gallery-ms", "MediaStore query returned null (permission?)")
|
||||
return
|
||||
}
|
||||
) ?: return
|
||||
val sb = StringBuilder()
|
||||
rows.use { c ->
|
||||
val iData = c.getColumnIndexOrThrow(MediaStore.Images.Media.DATA)
|
||||
val iId = c.getColumnIndexOrThrow(MediaStore.Images.Media._ID)
|
||||
val iData = c.getColumnIndex(MediaStore.Images.Media.DATA)
|
||||
val iRel = c.getColumnIndex(MediaStore.Images.Media.RELATIVE_PATH)
|
||||
val iSize = c.getColumnIndexOrThrow(MediaStore.Images.Media.SIZE)
|
||||
val iAdded = c.getColumnIndexOrThrow(MediaStore.Images.Media.DATE_ADDED)
|
||||
while (c.moveToNext()) {
|
||||
var data = c.getString(iData)
|
||||
if (data.isNullOrEmpty() && iRel >= 0) {
|
||||
val rel = c.getString(iRel) ?: continue
|
||||
data = "/storage/emulated/0/" + rel
|
||||
var path = if (iData >= 0) c.getString(iData) else null
|
||||
if (path.isNullOrEmpty() && iRel >= 0) {
|
||||
val rel = c.getString(iRel)
|
||||
if (!rel.isNullOrEmpty()) path = "/storage/emulated/0/" + rel
|
||||
}
|
||||
if (data.isNullOrEmpty()) continue
|
||||
sb.append(data).append('|')
|
||||
.append(c.getLong(iAdded)).append('|')
|
||||
.append(c.getLong(iSize)).append('\n')
|
||||
if (path.isNullOrEmpty()) continue
|
||||
sb.append("A|").append(c.getLong(iId)).append('|')
|
||||
.append(path).append('|').append(c.getLong(iSize)).append('\n')
|
||||
}
|
||||
}
|
||||
android.util.Log.i("gallery-ms", "pushRecent rows=" + sb.length)
|
||||
if (sb.isNotEmpty()) {
|
||||
// Hand the payload to Rust via a file (robust under the native
|
||||
// bridge; the JNI call stays the proven zero-arg wake).
|
||||
val dir = getExternalFilesDir(null) ?: return
|
||||
val tmp = java.io.File(dir, ".ms_recent.tmp")
|
||||
val fin = java.io.File(dir, ".ms_recent")
|
||||
tmp.writeText(sb.toString())
|
||||
if (!tmp.renameTo(fin)) {
|
||||
tmp.copyTo(fin, overwrite = true)
|
||||
tmp.delete()
|
||||
}
|
||||
nativeFsChange()
|
||||
deliver(sb.toString().trimEnd())
|
||||
android.util.Log.i("gallery-ms", "windowQuery: " + sb.toString().count { it == '\n' } + " rows")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
android.util.Log.w("gallery-ms", "pushRecent failed: " + e)
|
||||
android.util.Log.w("gallery-ms", "windowQuery: " + e)
|
||||
}
|
||||
}
|
||||
|
||||
/** Debounced (800 ms, main thread) schedule of pushRecent on the
|
||||
* background executor — the observer can fire in bursts. */
|
||||
private fun scheduleFsPush() {
|
||||
if (fsScheduled) return
|
||||
fsScheduled = true
|
||||
mainHandler.postDelayed({
|
||||
fsScheduled = false
|
||||
fsExecutor.execute { pushRecent() }
|
||||
}, 800)
|
||||
}
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
enableEdgeToEdge()
|
||||
super.onCreate(savedInstanceState)
|
||||
// Event-driven file watching (no polling): MediaStore changes —
|
||||
// camera saves, media-scanner indexing, other apps' indexed writes —
|
||||
// wake the Rust watcher, which diffs dir mtimes and does an
|
||||
// incremental rescan of only the changed subtrees.
|
||||
val obs = object : ContentObserver(Handler(Looper.getMainLooper())) {
|
||||
override fun onChange(selfChange: Boolean, uri: Uri?) {
|
||||
nativeFsChange()
|
||||
scheduleFsPush()
|
||||
// Event-driven: the OS passes the changed URI(s) directly.
|
||||
val obs = object : ContentObserver(mainHandler) {
|
||||
override fun onChange(selfChange: Boolean, uri: Uri?, flags: Int) {
|
||||
if (selfChange) return
|
||||
fsExecutor.execute { handleUri(uri) }
|
||||
}
|
||||
}
|
||||
observer = obs
|
||||
// 3-arg overload (notifyForDescendants) requires API 30 — the app's
|
||||
// minSdk (see tauri.conf.json bundle.android.minSdkVersion).
|
||||
// 3-arg registerContentObserver (notifyForDescendants) requires API
|
||||
// 30 — the app's minSdk.
|
||||
contentResolver.registerContentObserver(
|
||||
MediaStore.Images.Media.EXTERNAL_CONTENT_URI, true, obs
|
||||
)
|
||||
// Startup fast path: if the app was killed after a photo was taken,
|
||||
// the observer won't fire — push once after the media scanner has
|
||||
// had a moment to index anything recent.
|
||||
mainHandler.postDelayed({ scheduleFsPush() }, 1500)
|
||||
// Startup catch-up: if the app was killed after a photo was taken,
|
||||
// the observer won't replay — push whatever was indexed recently.
|
||||
mainHandler.postDelayed({ fsExecutor.execute { windowQuery() } }, 1500)
|
||||
requestMediaPermission()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -71,7 +71,8 @@ pub fn open(path: &std::path::Path) -> Result<Connection, rusqlite::Error> {
|
|||
height INTEGER,
|
||||
orientation INTEGER,
|
||||
thumb_path TEXT,
|
||||
date_taken INTEGER
|
||||
date_taken INTEGER,
|
||||
media_id INTEGER
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS items_mtime ON items (mtime DESC);
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
|
|
@ -79,6 +80,19 @@ pub fn open(path: &std::path::Path) -> Result<Connection, rusqlite::Error> {
|
|||
value TEXT NOT NULL
|
||||
);",
|
||||
)?;
|
||||
// migration for pre-media_id databases (duplicate column → ignore)
|
||||
let _ = conn.execute("ALTER TABLE items ADD COLUMN media_id INTEGER", []);
|
||||
let _ = conn.execute("CREATE INDEX IF NOT EXISTS items_media_id ON items (media_id)", []);
|
||||
conn.execute_batch(
|
||||
// MediaStore id → path, recorded for EVERY add event (including
|
||||
// size-0 mid-write rows that never get an items row). Lets a
|
||||
// later delete event resolve the path even when the items row
|
||||
// was inserted by a filesystem walk (which doesn't know the id).
|
||||
"CREATE TABLE IF NOT EXISTS media_map (
|
||||
media_id INTEGER PRIMARY KEY,
|
||||
path TEXT NOT NULL
|
||||
);",
|
||||
)?;
|
||||
Ok(conn)
|
||||
}
|
||||
|
||||
|
|
@ -136,86 +150,13 @@ pub fn apply(
|
|||
}
|
||||
}
|
||||
}
|
||||
// MASS-DELETE GUARD: if this walk sees dramatically fewer files
|
||||
// than the DB holds, the walk almost certainly truncated (FUSE
|
||||
// mid-stream readdir error — see scan::read_dir_full) rather than
|
||||
// a real mass deletion. Refuse to delete on the FIRST suspicious
|
||||
// walk; a second consecutive low walk confirms it is real.
|
||||
let mut to_delete = 0;
|
||||
// The walk that produced `entries` SUCCEEDED (scan fails the
|
||||
// whole walk on any FS error), so paths it did not see are
|
||||
// genuinely gone. Apply the deletions unconditionally.
|
||||
for p in existing.keys() {
|
||||
if !seen.contains(p) {
|
||||
to_delete += 1;
|
||||
}
|
||||
}
|
||||
// 10%: observed FUSE truncations lose 15-17% of a big library,
|
||||
// which sailed under the old 20% threshold and poisoned the DB.
|
||||
// A user deleting >10% of a library in one operation is rare;
|
||||
// if it happens, the second consecutive low walk (two-strike)
|
||||
// applies the deletion minutes later. Data loss is the
|
||||
// unforgivable failure mode — err on the side of refusing.
|
||||
let suspect = existing.len() > 100
|
||||
&& (to_delete as f64) > (existing.len() as f64) * 0.10;
|
||||
let mut do_delete = true;
|
||||
if suspect {
|
||||
let last_walked: i64 = tx
|
||||
.query_row("SELECT value FROM meta WHERE key='walked'", [], |r| {
|
||||
r.get::<_, String>(0)
|
||||
})
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(existing.len() as i64);
|
||||
if last_walked >= (existing.len() as i64) * 80 / 100 {
|
||||
log::warn!(
|
||||
"apply: walk saw {} but DB has {} (last walk {}): refusing to delete {} rows (truncated walk?)",
|
||||
entries.len(),
|
||||
existing.len(),
|
||||
last_walked,
|
||||
to_delete
|
||||
);
|
||||
do_delete = false;
|
||||
}
|
||||
let _ = tx.execute(
|
||||
"INSERT INTO meta (key, value) VALUES ('walked', ?1) \
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
params![entries.len().to_string()],
|
||||
);
|
||||
} else {
|
||||
let _ = tx.execute(
|
||||
"INSERT INTO meta (key, value) VALUES ('walked', ?1) \
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
params![entries.len().to_string()],
|
||||
);
|
||||
}
|
||||
// walked_max: the largest count ever applied. Layer-2 truncation
|
||||
// detection in rescan() compares against max(db, walked_max), so
|
||||
// a poisoned (shrunk) DB cannot silence the re-walk check.
|
||||
let prev_max: i64 = tx
|
||||
.query_row("SELECT value FROM meta WHERE key='walked_max'", [], |r| {
|
||||
r.get::<_, String>(0)
|
||||
})
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(0);
|
||||
// Decay when deletions were CONFIRMED (two-strike passed, or no
|
||||
// suspicion): a legitimately shrunk library lowers the ceiling.
|
||||
// When deletions were REFUSED, keep the high watermark so the
|
||||
// poisoned-DB chicken-and-egg stays detectable.
|
||||
let new_max = if do_delete && (entries.len() as i64) < prev_max {
|
||||
entries.len() as i64
|
||||
} else {
|
||||
prev_max.max(existing.len() as i64).max(entries.len() as i64)
|
||||
};
|
||||
let _ = tx.execute(
|
||||
"INSERT INTO meta (key, value) VALUES ('walked_max', ?1) \
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
params![new_max.to_string()],
|
||||
);
|
||||
if do_delete {
|
||||
for p in existing.keys() {
|
||||
if !seen.contains(p) {
|
||||
del.execute(params![p])?;
|
||||
deleted += 1;
|
||||
}
|
||||
del.execute(params![p])?;
|
||||
deleted += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -223,46 +164,122 @@ pub fn apply(
|
|||
Ok((inserted, updated, deleted))
|
||||
}
|
||||
|
||||
/// Upsert entries: insert new, update changed — NEVER deletes (the full
|
||||
/// FS rescan owns deletions). Used by the MediaStore fast path, which
|
||||
/// only ever ADDS recently-indexed files.
|
||||
pub fn upsert(
|
||||
/// Upsert ONE MediaStore notification: insert the row, or refresh it
|
||||
/// when mtime/size changed (content changed → derived columns stale).
|
||||
/// `media_id` (the MediaStore `_id`) is stored so deletion events can
|
||||
/// address the row directly. Returns true when the row's content
|
||||
/// actually changed (caller clears decode-failure state).
|
||||
pub fn upsert_ms(
|
||||
conn: &mut Connection,
|
||||
entries: &[Entry],
|
||||
) -> Result<(usize, usize), rusqlite::Error> {
|
||||
let mut inserted = 0;
|
||||
let mut updated = 0;
|
||||
path: &str,
|
||||
mtime: i64,
|
||||
size: i64,
|
||||
media_id: i64,
|
||||
) -> Result<bool, rusqlite::Error> {
|
||||
let tx = conn.transaction()?;
|
||||
{
|
||||
let mut ins = tx.prepare(
|
||||
"INSERT INTO items (path, mtime, size) VALUES (?1, ?2, ?3)",
|
||||
)?;
|
||||
let mut sel = tx.prepare(
|
||||
"SELECT mtime, size FROM items WHERE path = ?1",
|
||||
)?;
|
||||
let mut upd = tx.prepare(
|
||||
"UPDATE items SET mtime = ?2, size = ?3, width = NULL, height = NULL, \
|
||||
orientation = NULL, thumb_path = NULL, date_taken = NULL WHERE path = ?1",
|
||||
)?;
|
||||
for e in entries {
|
||||
let old: Option<(i64, i64)> = sel
|
||||
.query_row(params![e.path], |r| Ok((r.get(0)?, r.get(1)?)))
|
||||
.ok();
|
||||
match old {
|
||||
Some((m, s)) if m == e.mtime && s == e.size => {}
|
||||
Some(_) => {
|
||||
upd.execute(params![e.path, e.mtime, e.size])?;
|
||||
updated += 1;
|
||||
}
|
||||
None => {
|
||||
ins.execute(params![e.path, e.mtime, e.size])?;
|
||||
inserted += 1;
|
||||
}
|
||||
let changed = {
|
||||
let old: Option<(i64, i64)> = tx
|
||||
.query_row(
|
||||
"SELECT mtime, size FROM items WHERE path = ?1",
|
||||
params![path],
|
||||
|r| Ok((r.get(0)?, r.get(1)?)),
|
||||
)
|
||||
.ok();
|
||||
match old {
|
||||
Some((m, s)) if m == mtime && s == size => {
|
||||
// Content unchanged — just make sure the id is stored.
|
||||
let _ = tx.execute(
|
||||
"UPDATE items SET media_id = ?2 WHERE path = ?1 AND media_id IS NULL",
|
||||
params![path, media_id],
|
||||
);
|
||||
}
|
||||
Some(_) => {
|
||||
tx.execute(
|
||||
"UPDATE items SET mtime = ?2, size = ?3, media_id = ?4, width = NULL, \
|
||||
height = NULL, orientation = NULL, thumb_path = NULL, date_taken = NULL \
|
||||
WHERE path = ?1",
|
||||
params![path, mtime, size, media_id],
|
||||
)?;
|
||||
}
|
||||
None => {
|
||||
tx.execute(
|
||||
"INSERT INTO items (path, mtime, size, media_id) VALUES (?1, ?2, ?3, ?4)",
|
||||
params![path, mtime, size, media_id],
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Report content change: caller resets decode-failure counters so
|
||||
// a file that was still being written (and failed early decodes)
|
||||
// gets a fresh retry once the write completes.
|
||||
matches!(old, Some((m, s)) if m != mtime || s != size) || old.is_none()
|
||||
};
|
||||
tx.commit()?;
|
||||
Ok((inserted, updated))
|
||||
Ok(changed)
|
||||
}
|
||||
|
||||
/// Delete the row for a MediaStore `_id` (the row is gone from
|
||||
/// MediaStore — the media file was removed). Returns rows affected.
|
||||
pub fn delete_by_media_id(
|
||||
conn: &mut Connection,
|
||||
media_id: i64,
|
||||
) -> Result<usize, rusqlite::Error> {
|
||||
let n = conn.execute("DELETE FROM items WHERE media_id = ?1", params![media_id])?;
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Record a MediaStore id → path mapping (every add event, even for
|
||||
/// size-0 mid-write rows that don't get an items row).
|
||||
pub fn map_media(
|
||||
conn: &mut Connection,
|
||||
media_id: i64,
|
||||
path: &str,
|
||||
) -> Result<(), rusqlite::Error> {
|
||||
conn.execute(
|
||||
"INSERT INTO media_map (media_id, path) VALUES (?1, ?2) \
|
||||
ON CONFLICT(media_id) DO UPDATE SET path = excluded.path",
|
||||
params![media_id, path],
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resolve a MediaStore `_id` to a path: the items row first (if one
|
||||
/// has been created), then the add-event mapping.
|
||||
pub fn path_for_media(
|
||||
conn: &mut Connection,
|
||||
media_id: i64,
|
||||
) -> Result<Option<String>, rusqlite::Error> {
|
||||
let in_items = conn
|
||||
.query_row(
|
||||
"SELECT path FROM items WHERE media_id = ?1",
|
||||
params![media_id],
|
||||
|r| r.get::<_, String>(0),
|
||||
)
|
||||
.ok();
|
||||
Ok(in_items.or_else(|| {
|
||||
conn.query_row(
|
||||
"SELECT path FROM media_map WHERE media_id = ?1",
|
||||
params![media_id],
|
||||
|r| r.get::<_, String>(0),
|
||||
)
|
||||
.ok()
|
||||
}))
|
||||
}
|
||||
|
||||
/// Delete the items row for a path. Returns rows affected.
|
||||
pub fn delete_by_path(
|
||||
conn: &mut Connection,
|
||||
path: &str,
|
||||
) -> Result<usize, rusqlite::Error> {
|
||||
Ok(conn.execute("DELETE FROM items WHERE path = ?1", params![path])?)
|
||||
}
|
||||
|
||||
/// Drop a media_map entry.
|
||||
pub fn unmap_media(
|
||||
conn: &mut Connection,
|
||||
media_id: i64,
|
||||
) -> Result<(), rusqlite::Error> {
|
||||
conn.execute("DELETE FROM media_map WHERE media_id = ?1", params![media_id])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Number of rows in the DB.
|
||||
|
|
@ -271,21 +288,6 @@ pub fn count(conn: &mut Connection) -> Result<i64, rusqlite::Error> {
|
|||
Ok(n)
|
||||
}
|
||||
|
||||
/// Per-directory file counts (parent dir → row count) — the reference
|
||||
/// the walk uses to detect stale VFS directory listings.
|
||||
pub fn dir_counts(conn: &mut Connection) -> Result<HashMap<PathBuf, i64>, rusqlite::Error> {
|
||||
let mut st = conn.prepare("SELECT path FROM items")?;
|
||||
let rows = st.query_map([], |r| r.get::<_, String>(0))?;
|
||||
let mut m: HashMap<PathBuf, i64> = HashMap::new();
|
||||
for p in rows {
|
||||
let p = p?;
|
||||
if let Some(d) = std::path::Path::new(&p).parent() {
|
||||
*m.entry(d.to_path_buf()).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
Ok(m)
|
||||
}
|
||||
|
||||
/// All known entries, newest first.
|
||||
pub fn list_entries(conn: &mut Connection) -> Result<Vec<Entry>, rusqlite::Error> {
|
||||
let mut st = conn.prepare("SELECT path, mtime, size FROM items ORDER BY mtime DESC, path ASC")?;
|
||||
|
|
@ -380,7 +382,7 @@ mod tests {
|
|||
|
||||
let db_dir = tempfile::tempdir().unwrap();
|
||||
let mut conn = open(&db_dir.path().join("t.db")).unwrap();
|
||||
let entries = walk(&[dir.path().to_path_buf()], &|_| {});
|
||||
let entries = walk(&[dir.path().to_path_buf()], &|_| {}).unwrap();
|
||||
let (i, u, d) = apply(&mut conn, &entries).unwrap();
|
||||
assert_eq!((i, u, d), (2, 0, 0));
|
||||
|
||||
|
|
@ -388,4 +390,24 @@ mod tests {
|
|||
assert_eq!(list.len(), 2);
|
||||
assert!(list.iter().all(|e| e.path.contains(dir.path().to_str().unwrap())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mediasotre_upsert_and_delete_by_id() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut conn = open(&dir.path().join("t.db")).unwrap();
|
||||
|
||||
// New row: inserted, reports change.
|
||||
assert!(upsert_ms(&mut conn, "/x/a.jpg", 100, 10, 7).unwrap());
|
||||
// Same content: no change reported.
|
||||
assert!(!upsert_ms(&mut conn, "/x/a.jpg", 100, 10, 7).unwrap());
|
||||
// Content changed: change reported.
|
||||
assert!(upsert_ms(&mut conn, "/x/a.jpg", 200, 11, 7).unwrap());
|
||||
// Second row for a different id.
|
||||
upsert_ms(&mut conn, "/x/b.jpg", 100, 10, 8).unwrap();
|
||||
|
||||
assert_eq!(path_by_media_id(&mut conn, 7).unwrap().as_deref(), Some("/x/a.jpg"));
|
||||
assert_eq!(delete_by_media_id(&mut conn, 7).unwrap(), 1);
|
||||
assert_eq!(delete_by_media_id(&mut conn, 7).unwrap(), 0); // idempotent
|
||||
assert_eq!(count(&mut conn).unwrap(), 1);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -713,47 +713,13 @@ fn load_meta(conn: &mut db::Connection) -> Result<(), rusqlite::Error> {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Total rows in the DB (0 on error).
|
||||
fn expected_count() -> i64 {
|
||||
match db::open(&db::db_path()) {
|
||||
Ok(mut c) => db::count(&mut c).unwrap_or(0),
|
||||
Err(_) => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Largest file count ever applied to the DB (0 if never recorded).
|
||||
/// Layer-2 truncation detection compares against max(db, this) so a
|
||||
/// poisoned (truncation-shrunk) DB cannot silence the re-walk check.
|
||||
fn walked_max() -> i64 {
|
||||
match db::open(&db::db_path()) {
|
||||
Ok(mut c) => c
|
||||
.query_row("SELECT value FROM meta WHERE key='walked_max'", [], |r| {
|
||||
r.get::<_, String>(0)
|
||||
})
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(0),
|
||||
Err(_) => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-directory file counts from the DB — the reference the walk uses
|
||||
/// to detect stale (silently truncated) VFS directory listings.
|
||||
fn expected_dir_counts() -> std::collections::HashMap<std::path::PathBuf, i64> {
|
||||
match db::open(&db::db_path()) {
|
||||
Ok(mut c) => db::dir_counts(&mut c).unwrap_or_default(),
|
||||
Err(_) => std::collections::HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk the roots once, streaming progress emits. Cross-checks each
|
||||
/// directory's enumeration against the previous DB counts (stale-listing
|
||||
/// protection — see scan::walk_stream).
|
||||
fn walk_once(app: &tauri::AppHandle) -> Vec<scan::Entry> {
|
||||
let expected = expected_dir_counts();
|
||||
/// Walk the roots once, streaming progress emits.
|
||||
/// A filesystem error fails the walk (see scan.rs) — the caller must
|
||||
/// then NOT apply a partial result to the DB.
|
||||
fn walk_once(app: &tauri::AppHandle) -> std::io::Result<Vec<scan::Entry>> {
|
||||
let roots = scan::roots();
|
||||
let last_emit = std::cell::Cell::new(std::time::Instant::now());
|
||||
scan::walk_stream(&roots, &expected, &|batch| {
|
||||
scan::walk_stream(&roots, &|batch| {
|
||||
if last_emit.get().elapsed() < std::time::Duration::from_millis(500) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -762,43 +728,20 @@ fn walk_once(app: &tauri::AppHandle) -> Vec<scan::Entry> {
|
|||
})
|
||||
}
|
||||
|
||||
/// Incremental rescan: walk the roots, diff against the DB (insert
|
||||
/// new / update changed / delete removed), refresh META, emit the list.
|
||||
/// Used by the startup scan AND the live watcher — a new photo appears
|
||||
/// in the grid seconds after it is written.
|
||||
///
|
||||
/// Stale-listing (truncation) defense, three layers:
|
||||
/// 1. per-directory cross-check during the walk (scan::walk_dir) —
|
||||
/// re-reads any directory whose enumeration is far below the DB;
|
||||
/// 2. whole-walk re-check below — if the TOTAL is still far below the
|
||||
/// DB, re-walk once after a pause and keep the larger result;
|
||||
/// 3. db::apply mass-delete guard — never delete >20% of the DB in a
|
||||
/// single apply unless a previous walk already saw the low count.
|
||||
/// Full rescan: walk the roots, diff against the DB (insert new /
|
||||
/// update changed / delete removed), refresh META, emit the list.
|
||||
/// Used by the startup scan AND the 5-minute reconciliation pass.
|
||||
/// If the walk fails, nothing is applied — an error is not "those
|
||||
/// files are gone"; the next pass retries.
|
||||
fn rescan(app: &tauri::AppHandle) {
|
||||
let mut entries = walk_once(app);
|
||||
// Whole-walk re-check (layer 2). Reference = max(db, walked_max):
|
||||
// a DB that was poisoned downward by an earlier truncated walk must
|
||||
// not be able to make a matching truncated walk look normal.
|
||||
let db_count = expected_count();
|
||||
let ref_count = db_count.max(walked_max());
|
||||
let walked = entries.len() as i64;
|
||||
if ref_count > 0
|
||||
&& walked < ref_count.saturating_sub(100)
|
||||
&& (ref_count - walked) as f64 > (ref_count as f64) * 0.15
|
||||
{
|
||||
tr(&format!(
|
||||
"rescan: walk {} << ref {} (db {}) — re-walk in 2s",
|
||||
entries.len(),
|
||||
ref_count,
|
||||
db_count
|
||||
));
|
||||
std::thread::sleep(std::time::Duration::from_secs(2));
|
||||
let again = walk_once(app);
|
||||
if again.len() > entries.len() {
|
||||
tr(&format!("rescan: re-walk recovered {} → {}", entries.len(), again.len()));
|
||||
entries = again;
|
||||
let entries = match walk_once(app) {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
tr(&format!("rescan: walk FAILED ({e}) — not applying"));
|
||||
log::warn!("rescan walk failed: {e}");
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
let mut conn = match db::open(&db::db_path()) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
|
|
@ -840,17 +783,22 @@ fn dir_sig(roots: &[PathBuf]) -> HashMap<PathBuf, i64> {
|
|||
while let Some(dir) = stack.pop() {
|
||||
let Ok(m) = std::fs::metadata(&dir) else { continue };
|
||||
out.insert(dir.clone(), mtime_ns(&m));
|
||||
if let Some(entries) = scan::read_dir_full(&dir) {
|
||||
for ent in entries {
|
||||
let p = ent.path();
|
||||
if p.is_symlink() || !p.is_dir() {
|
||||
continue;
|
||||
match std::fs::read_dir(&dir) {
|
||||
Ok(rd) => {
|
||||
for ent in rd.flatten() {
|
||||
let p = ent.path();
|
||||
if p.is_symlink() || !p.is_dir() {
|
||||
continue;
|
||||
}
|
||||
if p.file_name().map_or(false, |n| n.to_string_lossy().starts_with('.')) {
|
||||
continue;
|
||||
}
|
||||
stack.push(p);
|
||||
}
|
||||
if p.file_name().map_or(false, |n| n.to_string_lossy().starts_with('.')) {
|
||||
continue;
|
||||
}
|
||||
stack.push(p);
|
||||
}
|
||||
// A directory we cannot read this tick: skip it; the next
|
||||
// tick or the full pass will pick up anything that changed.
|
||||
Err(_) => {}
|
||||
}
|
||||
}
|
||||
out
|
||||
|
|
@ -890,19 +838,16 @@ fn rescan_subtrees(app: &tauri::AppHandle, changed: &[PathBuf]) {
|
|||
under.iter().map(|e| (e.path.as_str(), *e)).collect();
|
||||
if under.len() <= BIG_DIR {
|
||||
// Small: full walk with stat — adds, deletes AND edits.
|
||||
// Stale-listing reference: per-directory counts from the DB.
|
||||
let mut exp: HashMap<std::path::PathBuf, i64> = HashMap::new();
|
||||
for e in &under {
|
||||
if let Some(p) = std::path::Path::new(&e.path).parent() {
|
||||
*exp.entry(p.to_path_buf()).or_insert(0) += 1;
|
||||
}
|
||||
if let Err(e) = scan::walk_dir_into(d, &mut fresh) {
|
||||
tr(&format!("incr: walk {d:?} FAILED ({e}) — not applying"));
|
||||
log::warn!("incr walk failed: {e}");
|
||||
return; // a failed subtree must not be applied as empty
|
||||
}
|
||||
scan::walk_dir_into(d, &mut fresh, &exp);
|
||||
} else {
|
||||
// Huge: readdir-only. New files get stat'd; known files keep
|
||||
// their DB mtime/size (an in-place edit there is caught by
|
||||
// the 5-minute full pass).
|
||||
scan::walk_dir_nostats(d, &mut |p| {
|
||||
if let Err(e) = scan::walk_dir_nostats(d, &mut |p| {
|
||||
let ps = p.to_string_lossy();
|
||||
let entry = match known.get(ps.as_ref()) {
|
||||
Some(old) => scan::Entry {
|
||||
|
|
@ -916,11 +861,15 @@ fn rescan_subtrees(app: &tauri::AppHandle, changed: &[PathBuf]) {
|
|||
mtime: mtime_ns(&md),
|
||||
size: md.len() as i64,
|
||||
},
|
||||
Err(_) => return,
|
||||
Err(_) => return, // file vanished mid-walk — normal race
|
||||
},
|
||||
};
|
||||
fresh.push(entry);
|
||||
});
|
||||
}) {
|
||||
tr(&format!("incr: walk {d:?} FAILED ({e}) — not applying"));
|
||||
log::warn!("incr walk failed: {e}");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut all = existing
|
||||
|
|
@ -942,62 +891,52 @@ fn rescan_subtrees(app: &tauri::AppHandle, changed: &[PathBuf]) {
|
|||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// MediaStore fast path (Android): when the OS says the media changed,
|
||||
// query MediaStore for images indexed in the last ~2 min. One indexed
|
||||
// provider query (~50 ms) vs a ~10 s FUSE re-read of a huge directory —
|
||||
// this is what makes a just-taken camera photo appear in ~1 s.
|
||||
// MediaStore fast path (Android). MainActivity's ContentObserver gets
|
||||
// the CHANGED URI(s) directly from the OS; it queries that exact row
|
||||
// (a few ms) and drops a one-line payload file, then wakes us with a
|
||||
// zero-arg JNI call. No filesystem scanning on the event path: the
|
||||
// payload IS the change (added, modified, or deleted).
|
||||
//
|
||||
// Payload line formats (one per file, one file per event — per-event
|
||||
// files so a burst never overwrites an unread event):
|
||||
// A|<media_id>|<path>|<size> row exists → upsert
|
||||
// D|<media_id> row gone → delete by id
|
||||
// ---------------------------------------------------------------------
|
||||
/// MediaStore fast path (Android): read the .ms_recent payload dropped
|
||||
/// by MainActivity and apply it. The query itself runs on Kotlin's
|
||||
/// background executor (no Rust→Java FFI, robust under the native
|
||||
/// bridge); only a zero-arg wake crosses JNI.
|
||||
#[cfg(target_os = "android")]
|
||||
fn apply_mediasotre_payload(app: &tauri::AppHandle) {
|
||||
let path = ms_recent_path();
|
||||
let Ok(text) = std::fs::read_to_string(&path) else { return };
|
||||
let _ = std::fs::remove_file(&path);
|
||||
if text.is_empty() {
|
||||
// Consume every pending .ms_recent.* event file (oldest first).
|
||||
let dir = external_files_dir();
|
||||
let Ok(rd) = std::fs::read_dir(&dir) else { return };
|
||||
let mut files: Vec<std::path::PathBuf> = rd
|
||||
.flatten()
|
||||
.filter(|e| {
|
||||
e.file_name()
|
||||
.to_string_lossy()
|
||||
.starts_with(".ms_recent.")
|
||||
})
|
||||
.map(|e| e.path())
|
||||
.collect();
|
||||
if files.is_empty() {
|
||||
return;
|
||||
}
|
||||
tr(&format!("mediastore push: {} bytes, {} lines", text.len(), text.lines().count()));
|
||||
mediastore_apply(app, &text);
|
||||
}
|
||||
|
||||
/// Path of the .ms_recent handoff file (external files dir — MainActivity
|
||||
/// writes it with getExternalFilesDir(null)).
|
||||
#[cfg(target_os = "android")]
|
||||
fn ms_recent_path() -> std::path::PathBuf {
|
||||
external_files_dir().join(".ms_recent")
|
||||
files.sort();
|
||||
for f in files {
|
||||
let name = f.file_name().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default();
|
||||
let Ok(text) = std::fs::read_to_string(&f) else { continue };
|
||||
let _ = std::fs::remove_file(&f);
|
||||
if text.is_empty() {
|
||||
continue;
|
||||
}
|
||||
tr(&format!("mediastore event {name}: {} lines", text.lines().count()));
|
||||
mediastore_apply(app, &text);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "android"))]
|
||||
fn apply_mediasotre_payload(_app: &tauri::AppHandle) {}
|
||||
|
||||
/// MediaStore fast path (Android): apply a "path|addedSec|size" payload
|
||||
/// pushed from MainActivity (which does the MediaStore query on a
|
||||
/// background thread — no Rust→Java FFI). Upserts the rows, re-emits.
|
||||
/// Never deletes (the FS rescan owns deletions). This is what makes a
|
||||
/// just-taken camera photo appear in ~1 s instead of waiting for the
|
||||
/// ~10 s FUSE re-read of a huge directory.
|
||||
#[cfg(target_os = "android")]
|
||||
fn mediastore_apply(app: &tauri::AppHandle, text: &str) {
|
||||
let mut entries = Vec::new();
|
||||
for line in text.lines() {
|
||||
let Some(path) = line.split('|').next() else { continue };
|
||||
if path.is_empty() || !scan::is_image_path(path) {
|
||||
continue;
|
||||
}
|
||||
if let Ok(md) = std::fs::metadata(path) {
|
||||
entries.push(scan::Entry {
|
||||
path: path.to_string(),
|
||||
mtime: mtime_ns(&md),
|
||||
size: md.len() as i64,
|
||||
});
|
||||
}
|
||||
}
|
||||
if entries.is_empty() {
|
||||
return;
|
||||
}
|
||||
let mut conn = match db::open(&db::db_path()) {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
|
|
@ -1005,17 +944,99 @@ fn mediastore_apply(app: &tauri::AppHandle, text: &str) {
|
|||
return;
|
||||
}
|
||||
};
|
||||
match db::upsert(&mut conn, &entries) {
|
||||
Ok((ins, upd)) => {
|
||||
if ins + upd > 0 {
|
||||
tr(&format!("mediastore fast +{ins} new, {upd} changed"));
|
||||
if let Err(e) = load_meta(&mut conn) {
|
||||
log::warn!("mediastore meta: {e}");
|
||||
let mut changed = 0usize;
|
||||
let mut removed = 0usize;
|
||||
for line in text.lines() {
|
||||
let mut f = line.splitn(5, '|');
|
||||
let tag = f.next().unwrap_or("");
|
||||
match tag {
|
||||
// A|media_id|path|size
|
||||
"A" => {
|
||||
let Ok(id) = f.next().unwrap_or("").parse::<i64>() else { continue };
|
||||
let Some(path) = f.next() else { continue };
|
||||
if path.is_empty() || !scan::is_image_path(path) {
|
||||
continue;
|
||||
}
|
||||
let _size = f.next().and_then(|s| s.parse::<i64>().ok()); // disk stat is the truth
|
||||
// Always record the id→path mapping: a later delete
|
||||
// event must be resolvable even for rows created by a
|
||||
// filesystem walk (no media_id) or never created at all
|
||||
// (the size-0 mid-write row below).
|
||||
if let Err(e) = db::map_media(&mut conn, id, path) {
|
||||
log::warn!("mediastore map: {e}");
|
||||
}
|
||||
// Trust the DISK, not the provider row: the scanner can
|
||||
// leave _size NULL, and a file still being written is
|
||||
// fine — its finalization event re-upserts with the new
|
||||
// size (content change resets the decode state).
|
||||
let Ok(md) = std::fs::metadata(path) else { continue };
|
||||
let mtime = mtime_ns(&md);
|
||||
match db::upsert_ms(&mut conn, path, mtime, md.len() as i64, id) {
|
||||
Ok(content_changed) => {
|
||||
// Row id → the rowid is what the grid/queue use;
|
||||
// clear decode-failure state only after a real
|
||||
// content change (a still-being-written file
|
||||
// that failed early gets fresh retries).
|
||||
let rowid: Option<i64> = conn
|
||||
.query_row(
|
||||
"SELECT rowid FROM items WHERE path = ?1",
|
||||
rusqlite::params![path],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.ok();
|
||||
if content_changed {
|
||||
if let Some(rid) = rowid {
|
||||
FAILS.lock().unwrap().remove(&rid);
|
||||
}
|
||||
}
|
||||
changed += 1;
|
||||
}
|
||||
Err(e) => log::warn!("mediastore upsert: {e}"),
|
||||
}
|
||||
emit_files(app, false, META.read().unwrap().len());
|
||||
}
|
||||
// D|media_id — the row is gone from MediaStore; resolve the
|
||||
// path (items row first, then the add-event mapping) and
|
||||
// delete by path: the items row may have been created by a
|
||||
// filesystem walk, which doesn't store the media id.
|
||||
"D" => {
|
||||
let Ok(id) = f.next().unwrap_or("").parse::<i64>() else { continue };
|
||||
let path = match db::path_for_media(&mut conn, id) {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
log::warn!("mediastore dlookup: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Some(path) = path {
|
||||
// clear decode-failure state before deleting
|
||||
if let Some(rid) = conn
|
||||
.query_row(
|
||||
"SELECT rowid FROM items WHERE path = ?1",
|
||||
rusqlite::params![path],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.ok()
|
||||
{
|
||||
FAILS.lock().unwrap().remove(&rid);
|
||||
}
|
||||
if let Ok(n) = db::delete_by_path(&mut conn, &path) {
|
||||
removed += n;
|
||||
if n > 0 {
|
||||
tr(&format!("mediastore del id={id} {path}"));
|
||||
}
|
||||
}
|
||||
let _ = db::unmap_media(&mut conn, id);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Err(e) => log::warn!("mediastore upsert: {e}"),
|
||||
}
|
||||
if changed + removed > 0 {
|
||||
tr(&format!("mediastore applied +{changed} -{removed}"));
|
||||
if let Err(e) = load_meta(&mut conn) {
|
||||
log::warn!("mediastore meta: {e}");
|
||||
}
|
||||
emit_files(app, false, META.read().unwrap().len());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1135,14 +1156,12 @@ fn watch_files(app: tauri::AppHandle) {
|
|||
std::thread::sleep(std::time::Duration::from_millis(600));
|
||||
while rx.try_recv().is_ok() {}
|
||||
}
|
||||
// MediaStore fast path: MainActivity (which does the MediaStore
|
||||
// query on a background thread) drops a "path|added|size" payload
|
||||
// in .ms_recent and wakes us via nativeFsChange. Apply it now —
|
||||
// a just-taken camera photo lands in ~1 s, without waiting for
|
||||
// the ~10 s FUSE re-read of a huge directory.
|
||||
if matches!(woke, Some(WEvent::Fs) | Some(WEvent::Full)) {
|
||||
apply_mediasotre_payload(&app);
|
||||
}
|
||||
// MediaStore event path: MainActivity drops a one-line payload
|
||||
// per changed URI and wakes us via nativeFsChange. Consume on
|
||||
// EVERY tick — the check is a single read_dir of our own app
|
||||
// dir (no-op when empty), so a lost JNI wake self-heals within
|
||||
// the 1 s backstop instead of waiting for the 5-min pass.
|
||||
apply_mediasotre_payload(&app);
|
||||
let next = dir_sig(&scan::roots());
|
||||
let changed = if baseline {
|
||||
baseline = false;
|
||||
|
|
|
|||
|
|
@ -1,10 +1,14 @@
|
|||
//! Directory walk + image detection (M1).
|
||||
//! Directory walk + image detection.
|
||||
//!
|
||||
//! Plain recursive scan of the roots (Pictures + DCIM on Android). This is
|
||||
//! what makes the app testable without MediaStore; MediaStore integration is
|
||||
//! a later refinement (HANDOFF.md §7.4).
|
||||
//! Plain recursive scan of the roots (Pictures + DCIM on Android).
|
||||
//! FUSE is a reliable infrastructure point: a directory reads either
|
||||
//! returns its full listing or returns an error. There is no silent
|
||||
//! truncation. Therefore the one invariant here is simple and strict:
|
||||
//! **any read_dir or stat error fails the walk** — a failed walk must
|
||||
//! never be applied to the DB, because an error is not "those files
|
||||
//! are gone."
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// One image on disk. `mtime` is nanoseconds since the Unix epoch (i64).
|
||||
|
|
@ -54,69 +58,66 @@ pub fn roots() -> Vec<PathBuf> {
|
|||
}
|
||||
}
|
||||
|
||||
fn is_image_file(name: &str, ext: &str) -> bool {
|
||||
!name.starts_with('.') && IMAGE_EXTS.contains(&ext.to_ascii_lowercase().as_str())
|
||||
}
|
||||
|
||||
fn file_metadata(p: &Path) -> io::Result<(i64, i64)> {
|
||||
let md = std::fs::metadata(p)?;
|
||||
let mtime = md
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
|
||||
.map(|d| d.as_nanos() as i64)
|
||||
.unwrap_or(0);
|
||||
Ok((mtime, md.len() as i64))
|
||||
}
|
||||
|
||||
/// Recursively collect all image entries below `roots`.
|
||||
/// `on_progress(n)` is called every 512 files found.
|
||||
pub fn walk(roots: &[PathBuf], on_progress: &dyn Fn(usize)) -> Vec<Entry> {
|
||||
walk_stream(roots, &HashMap::new(), &|batch| on_progress(batch.len()))
|
||||
/// Any filesystem error fails the walk (see module docs).
|
||||
pub fn walk(roots: &[PathBuf], on_progress: &dyn Fn(usize)) -> io::Result<Vec<Entry>> {
|
||||
walk_stream(roots, &|batch| on_progress(batch.len()))
|
||||
}
|
||||
|
||||
/// Like `walk`, but streams progress: `on_batch(&entries)` receives ALL
|
||||
/// entries collected so far every 512 new files, so the UI can show results
|
||||
/// while the scan is still running (first launch must not wait for the full
|
||||
/// DB build).
|
||||
///
|
||||
/// `expected_files` is the per-directory file count from the previous DB
|
||||
/// state (parent dir → count). When an enumeration returns far fewer
|
||||
/// files than the DB holds for that directory, the VFS is almost certainly
|
||||
/// serving a STALE cached directory listing (readdir returns a silent
|
||||
/// early EOF — no error at all; reproduced on both the phone's FUSE and
|
||||
/// the emulator's 9p /sdcard). In that case the directory is re-read up
|
||||
/// to 3 times with delays, keeping the best listing.
|
||||
/// entries collected so far every 512 new files, so the UI can show
|
||||
/// results while the scan is still running (first launch must not wait
|
||||
/// for the full DB build).
|
||||
pub fn walk_stream(
|
||||
roots: &[PathBuf],
|
||||
expected_files: &HashMap<PathBuf, i64>,
|
||||
on_batch: &dyn Fn(&[Entry]),
|
||||
) -> Vec<Entry> {
|
||||
) -> io::Result<Vec<Entry>> {
|
||||
let mut out = Vec::new();
|
||||
for r in roots {
|
||||
if r.is_dir() {
|
||||
walk_dir(r, &mut out, expected_files, on_batch);
|
||||
walk_dir(r, &mut out, on_batch)?;
|
||||
}
|
||||
}
|
||||
out
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Full walk (with stat) of ONE directory subtree.
|
||||
pub fn walk_dir_into(dir: &Path, out: &mut Vec<Entry>, expected_files: &HashMap<PathBuf, i64>) {
|
||||
pub fn walk_dir_into(dir: &Path, out: &mut Vec<Entry>) -> io::Result<()> {
|
||||
if dir.is_dir() {
|
||||
walk_dir(dir, out, expected_files, &|_| {});
|
||||
walk_dir(dir, out, &|_| {})?;
|
||||
}
|
||||
}
|
||||
|
||||
/// Count image-looking names in a listing (cheap: names only, no stat).
|
||||
fn count_image_names(es: &[std::fs::DirEntry]) -> i64 {
|
||||
es.iter().filter(|e| {
|
||||
let name = e.file_name().to_string_lossy().into_owned();
|
||||
!name.starts_with('.')
|
||||
&& e.path()
|
||||
.extension()
|
||||
.and_then(|x| x.to_str())
|
||||
.is_some_and(|x| IMAGE_EXTS.contains(&x.to_ascii_lowercase().as_str()))
|
||||
}).count() as i64
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Readdir-only walk (NO per-file stat) of ONE directory subtree — for
|
||||
/// huge FUSE directories where stat'ing every file costs ~30 s; the
|
||||
/// caller supplies metadata for new files and keeps DB metadata for
|
||||
/// known ones.
|
||||
pub fn walk_dir_nostats(dir: &Path, cb: &mut dyn FnMut(&Path)) {
|
||||
/// known ones. A read_dir error fails the walk; a per-file type
|
||||
/// failure (file vanishing mid-walk — a normal race) is skipped.
|
||||
pub fn walk_dir_nostats(dir: &Path, cb: &mut dyn FnMut(&Path)) -> io::Result<()> {
|
||||
if !dir.is_dir() {
|
||||
return;
|
||||
return Ok(());
|
||||
}
|
||||
let mut stack = vec![dir.to_path_buf()];
|
||||
while let Some(d) = stack.pop() {
|
||||
let Some(entries) = read_dir_full(&d) else { continue };
|
||||
for ent in entries {
|
||||
for ent in std::fs::read_dir(&d)? {
|
||||
let ent = ent?;
|
||||
let p = ent.path();
|
||||
let Ok(ft) = ent.file_type() else { continue };
|
||||
if ft.is_dir() {
|
||||
|
|
@ -132,102 +133,18 @@ pub fn walk_dir_nostats(dir: &Path, cb: &mut dyn FnMut(&Path)) {
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read a directory to completion, retrying on transient mid-stream
|
||||
/// errors. CRITICAL on Android FUSE: a std::fs::read_dir over a
|
||||
/// thousands-of-entries directory latches the iterator into an error
|
||||
/// state on ANY transient failure (verified on the phone: the 9961-file
|
||||
/// Camera dir intermittently truncated at ~8400 entries, and the
|
||||
/// truncated walk made db::apply delete ~1700 good rows). On a
|
||||
/// mid-stream error the whole directory is re-read; after 3 failed
|
||||
/// attempts the directory is skipped (the mass-delete guard in
|
||||
/// db::apply is the backstop).
|
||||
pub fn read_dir_full(dir: &Path) -> Option<Vec<std::fs::DirEntry>> {
|
||||
for attempt in 0..3u32 {
|
||||
let Ok(rd) = std::fs::read_dir(dir) else { return None };
|
||||
let mut entries = Vec::new();
|
||||
let mut ok = true;
|
||||
for ent in rd {
|
||||
match ent {
|
||||
Ok(e) => entries.push(e),
|
||||
Err(_) => {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
return Some(entries);
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(200 * (attempt + 1) as u64));
|
||||
}
|
||||
log::error!("read_dir {:?}: 3 mid-stream failures, skipping dir", dir);
|
||||
None
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn walk_dir(
|
||||
dir: &Path,
|
||||
out: &mut Vec<Entry>,
|
||||
expected_files: &HashMap<PathBuf, i64>,
|
||||
on_batch: &dyn Fn(&[Entry]),
|
||||
) {
|
||||
let mut entries = match read_dir_full(dir) {
|
||||
Some(e) => e,
|
||||
None => return,
|
||||
};
|
||||
// GROUND-TRUTH INSTRUMENTATION: raw getdents count for this directory,
|
||||
// before any filtering. If raw < reality, the VFS served a short
|
||||
// listing to this process; if raw == reality but the walk total is
|
||||
// low, a filter below dropped entries (our bug).
|
||||
if dir.parent().is_some() {
|
||||
crate::tr(&format!(
|
||||
"rawdir {:?}: {} entries",
|
||||
dir.file_name().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default(),
|
||||
entries.len()
|
||||
));
|
||||
}
|
||||
// STALE-LISTING CROSS-CHECK (see walk_stream docs): if the DB holds
|
||||
// many more files for this directory than the enumeration produced,
|
||||
// re-read the directory a few times and keep the best listing.
|
||||
if let Some(&want) = expected_files.get(dir) {
|
||||
if want >= 50 {
|
||||
let mut best_n = count_image_names(&entries);
|
||||
if best_n * 5 < want * 4 {
|
||||
for attempt in 1..=3u32 {
|
||||
std::thread::sleep(std::time::Duration::from_millis(300 * u64::from(attempt)));
|
||||
if let Some(again) = read_dir_full(dir) {
|
||||
let n = count_image_names(&again);
|
||||
if n > best_n {
|
||||
best_n = n;
|
||||
entries = again;
|
||||
}
|
||||
if best_n * 5 >= want * 4 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if best_n * 5 < want * 4 {
|
||||
log::warn!(
|
||||
"walk {:?}: enumerated {} image files, DB has {} (stale listing?)",
|
||||
dir,
|
||||
best_n,
|
||||
want
|
||||
);
|
||||
} else {
|
||||
log::info!("walk {:?}: stale listing recovered {} → {}",
|
||||
dir, want, best_n);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut n_ftfail = 0u32;
|
||||
let mut n_statfail = 0u32;
|
||||
let mut n_pushed = 0u32;
|
||||
let raw_n = entries.len();
|
||||
for ent in entries {
|
||||
) -> io::Result<()> {
|
||||
for ent in std::fs::read_dir(dir)? {
|
||||
let ent = ent?;
|
||||
let p = ent.path();
|
||||
let Ok(ft) = ent.file_type() else { n_ftfail += 1; continue };
|
||||
let ft = ent.file_type()?;
|
||||
if ft.is_dir() {
|
||||
// Skip hidden dirs (.thumbnails, .trash, …) — metadata, not photos.
|
||||
if p.file_name().and_then(|n| n.to_str()).is_some_and(|n| n.starts_with('.')) {
|
||||
|
|
@ -235,54 +152,29 @@ fn walk_dir(
|
|||
}
|
||||
// Don't follow symlinked dirs (loop protection).
|
||||
if !p.is_symlink() {
|
||||
walk_dir(&p, out, expected_files, on_batch);
|
||||
walk_dir(&p, out, on_batch)?;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if !ft.is_file() {
|
||||
continue;
|
||||
}
|
||||
// OS junk that masquerades as photos: macOS AppleDouble
|
||||
// resource forks ("._IMG…jpg"), hidden files, .DS_Store, …
|
||||
// Not decodable — they render as permanent gray cells.
|
||||
let name = p.file_name().and_then(|n| n.to_str()).unwrap_or("");
|
||||
if name.starts_with('.') {
|
||||
continue;
|
||||
}
|
||||
let Some(ext) = p.extension().and_then(|e| e.to_str()) else { continue };
|
||||
if !IMAGE_EXTS.contains(&ext.to_ascii_lowercase().as_str()) {
|
||||
if !is_image_file(name, ext) {
|
||||
continue;
|
||||
}
|
||||
let Ok(md) = std::fs::metadata(&p) else {
|
||||
n_statfail += 1;
|
||||
continue;
|
||||
};
|
||||
let mtime = md
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
|
||||
.map(|d| d.as_nanos() as i64)
|
||||
.unwrap_or(0);
|
||||
let (mtime, size) = file_metadata(&p)?;
|
||||
out.push(Entry {
|
||||
path: p.to_string_lossy().into_owned(),
|
||||
mtime,
|
||||
size: md.len() as i64,
|
||||
size,
|
||||
});
|
||||
n_pushed += 1;
|
||||
if out.len() % 512 == 0 {
|
||||
on_batch(out);
|
||||
}
|
||||
}
|
||||
if n_ftfail > 0 || n_statfail > 0 {
|
||||
crate::tr(&format!(
|
||||
"walkdir {:?}: raw {} pushed {} ftfail {} statfail {}",
|
||||
dir.file_name().map(|s| s.to_string_lossy().into_owned()).unwrap_or_default(),
|
||||
raw_n,
|
||||
n_pushed,
|
||||
n_ftfail,
|
||||
n_statfail
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -292,8 +184,7 @@ mod tests {
|
|||
|
||||
fn make_tree(root: &Path, n_files: usize, subdirs: usize) {
|
||||
for d in 0..subdirs {
|
||||
let dir = root.join(format!("dir{d}"));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::create_dir_all(root.join(format!("dir{d}"))).unwrap();
|
||||
}
|
||||
for i in 0..n_files {
|
||||
let dir = if i % 2 == 0 { root.join(format!("dir{}", i % subdirs)) } else { root.to_path_buf() };
|
||||
|
|
@ -310,7 +201,7 @@ mod tests {
|
|||
std::fs::write(dir.path().join("c.txt"), b"x").unwrap();
|
||||
std::fs::create_dir_all(dir.path().join("d")).unwrap();
|
||||
std::fs::write(dir.path().join("d").join("e.heic"), b"x").unwrap();
|
||||
let out = walk(&[dir.path().to_path_buf()], &|_| {});
|
||||
let out = walk(&[dir.path().to_path_buf()], &|_| {}).unwrap();
|
||||
assert_eq!(out.len(), 3);
|
||||
assert!(out.iter().all(|e| e.size == 1));
|
||||
}
|
||||
|
|
@ -320,7 +211,7 @@ mod tests {
|
|||
let dir = tempfile::tempdir().unwrap();
|
||||
make_tree(dir.path(), 5000, 8);
|
||||
let t = std::time::Instant::now();
|
||||
let out = walk(&[dir.path().to_path_buf()], &|_| {});
|
||||
let out = walk(&[dir.path().to_path_buf()], &|_| {}).unwrap();
|
||||
let elapsed = t.elapsed();
|
||||
assert_eq!(out.len(), 5000);
|
||||
assert!(
|
||||
|
|
@ -329,4 +220,12 @@ mod tests {
|
|||
elapsed
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_root_dir_is_an_error() {
|
||||
let r = vec![PathBuf::from("/nonexistent/gallery-test-xyz")];
|
||||
// is_dir() false → the root is skipped, empty result, no error.
|
||||
let out = walk(&r, &|_| {}).unwrap();
|
||||
assert!(out.is_empty());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user