diff --git a/fundlab/RESEARCH.md b/fundlab/RESEARCH.md index b00a4ea..ad06ac7 100644 --- a/fundlab/RESEARCH.md +++ b/fundlab/RESEARCH.md @@ -100,6 +100,21 @@ not a gate. Infra lesson: /tmp gets cleaned mid-run - keep logs + caches in the project (fundlab/overnight.log, fundlab/universe_cache/), and use `setsid nohup ... < /dev/null &` so a closed shell can't kill the job. + +### Watchdog incident (2026-08-26 ~22:00) +- First watchdog used `pgrep -f "fundlab.overnight"` for liveness. + **Failed test**: a monitoring one-liner whose command line merely + CONTAINED the string made pgrep match the monitor's own bash -c + wrapper -> watchdog reported "alive" while the real run was dead. + Lesson: `pgrep -f` on a distinctive-but-quotable string is fragile; + any shell quoting the string is a false positive. Use a PID file the + process writes itself (`fundlab/overnight.pid`), pattern only as + fallback for pre-pidfile launches. +- Fixed watchdog restarted; it detected the dead run on its first + check, relaunched, and the run resumed at "1823 missing" (137 + symbols already cached - exactly the expected resume). +- Also hardened stage_download: a hung goget batch (TimeoutExpired) + no longer kills the whole run - caught per batch. - 2026-08-26 16:01 === stage verify === - 2026-08-26 16:01 verify: 10372 tickers, 9149 to hit Yahoo - 2026-08-26 16:01 verify: 500/9149 @@ -127,3 +142,12 @@ project (fundlab/overnight.log, fundlab/universe_cache/), and use - 2026-08-26 21:31 === overnight run finished in 0.0h === - 2026-08-26 21:31 === stage download === - 2026-08-26 21:31 download: 1960 missing symbols +- 2026-08-26 21:54:22 watchdog: watchdog started (interval 300s) +- 2026-08-26 21:54:22 watchdog: overnight alive, still_missing=2384 +- 2026-08-26 21:59:22 watchdog: overnight alive, still_missing=2384 +- 2026-08-26 22:04:22 watchdog: overnight alive, still_missing=2384 +- 2026-08-26 22:08:01 watchdog: watchdog started (interval 300s) +- 2026-08-26 22:08:01 watchdog: process DEAD with 2384 funds unscreened - relaunching +- 2026-08-26 22:08 === stage download === +- 2026-08-26 22:08 download: 1823 missing symbols +- 2026-08-26 22:09:01 watchdog: relaunch confirmed alive diff --git a/fundlab/overnight.log b/fundlab/overnight.log index d1ee95b..cb3a24b 100644 --- a/fundlab/overnight.log +++ b/fundlab/overnight.log @@ -1,2 +1,4 @@ 21:31:57 === stage download === 21:31:57 download: 1960 missing symbols +22:08:01 === stage download === +22:08:01 download: 1823 missing symbols diff --git a/fundlab/overnight.pid b/fundlab/overnight.pid new file mode 100644 index 0000000..684e9f4 --- /dev/null +++ b/fundlab/overnight.pid @@ -0,0 +1 @@ +11169 \ No newline at end of file diff --git a/fundlab/overnight.py b/fundlab/overnight.py index aed777b..3aba048 100644 --- a/fundlab/overnight.py +++ b/fundlab/overnight.py @@ -175,10 +175,13 @@ def stage_download(batch: int = 200) -> None: log(f"download: {len(missing)} missing symbols") for i in range(0, len(missing), batch): chunk = missing[i:i + batch] - subprocess.run([str(GOGET), *chunk], cwd=DATA, capture_output=True, - timeout=7200) + try: + subprocess.run([str(GOGET), *chunk], cwd=DATA, + capture_output=True, timeout=7200) + except Exception as e: # never kill the whole run for one batch + log(f"download: batch {i // batch + 1} FAILED ({e}) - continuing") got = sum(1 for s in chunk if (DATA / f"{s}-history.csv").exists()) - log(f"download: batch {i//batch + 1} -> {got}/{len(chunk)}") + log(f"download: batch {i // batch + 1} -> {got}/{len(chunk)}") still = sum(1 for s in missing if not (DATA / f"{s}-history.csv").exists()) log(f"download done: {still} still missing (no Yahoo data?)") @@ -222,8 +225,13 @@ def stage_finalize() -> None: f"{[v['sym'] for v in missed]}") +PIDFILE = HERE / "overnight.pid" + + def run(stages: list[str] | None = None) -> None: + import os stages = stages or ["verify", "select", "download", "screen", "finalize"] + PIDFILE.write_text(str(os.getpid())) # liveness marker for watchdog t0 = time.time() for s in stages: log(f"=== stage {s} ===") diff --git a/fundlab/watchdog.log b/fundlab/watchdog.log new file mode 100644 index 0000000..bae4c73 --- /dev/null +++ b/fundlab/watchdog.log @@ -0,0 +1,7 @@ +[2026-08-26 21:54:22] watchdog started (interval 300s) +[2026-08-26 21:54:22] overnight alive, still_missing=2384 +[2026-08-26 21:59:22] overnight alive, still_missing=2384 +[2026-08-26 22:04:22] overnight alive, still_missing=2384 +[2026-08-26 22:08:01] watchdog started (interval 300s) +[2026-08-26 22:08:01] process DEAD with 2384 funds unscreened - relaunching +[2026-08-26 22:09:01] relaunch confirmed alive diff --git a/fundlab/watchdog.sh b/fundlab/watchdog.sh new file mode 100755 index 0000000..e02eefb --- /dev/null +++ b/fundlab/watchdog.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Watchdog for the overnight comprehensive screen. +# +# The pipeline (fundlab.overnight) is fully resumable: every stage skips +# work already in its cache. So if the python process dies before it has +# screened the whole selected worklist, we simply relaunch the same +# command and it picks up where it left off. +# +# "Complete" = search_all.json holds at least as many symbols as +# selected.json (the screen stage wrote every fund). Then we stop. +# +# Usage: setsid nohup fundlab/watchdog.sh >fundlab/watchdog.log 2>&1 & +set -u +cd "$(dirname "$0")/.." +PY=".venv/bin/python" +INTERVAL=300 # check every 5 min + +stamp() { date "+%Y-%m-%d %H:%M:%S"; } +note() { + echo "[$(stamp)] $*" + # mirror into the research log so the knowledge survives + echo "- $(stamp) watchdog: $*" >> fundlab/RESEARCH.md +} + +need_syms() { # how many funds still need screening + "$PY" - <<'EOF' +import json, os +sel = "fundlab/universe_cache/selected.json" +res = "fundlab/search_all.json" +if not os.path.exists(sel): + print(0); raise SystemExit +n = len(json.load(open(sel))) +done = len(json.load(open(res))) if os.path.exists(res) else 0 +print(max(0, n - done)) +EOF +} + +run_alive() { + # prefer the pidfile the run process writes (immune to pattern + # false-positives, e.g. a monitor shell containing the string); + # fall back to a pattern match for pre-pidfile launches. + local pidfile="fundlab/overnight.pid" + if [ -f "$pidfile" ]; then + local pid; pid=$(cat "$pidfile" 2>/dev/null) + if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then + return 0 + fi + return 1 # stale pidfile: the process is gone + fi + pgrep -f "python -m fundlab\.overnight" >/dev/null 2>&1 +} + +note "watchdog started (interval ${INTERVAL}s)" +while true; do + if run_alive; then + note "overnight alive, still_missing=$(need_syms)" + else + missing=$(need_syms) + if [ "${missing:-0}" -eq 0 ]; then + note "process gone but screen COMPLETE (0 missing) - watchdog exiting" + rm -f fundlab/overnight.pid + break + fi + note "process DEAD with ${missing} funds unscreened - relaunching" + setsid nohup "$PY" -m fundlab.overnight download screen finalize \ + > fundlab/overnight.log 2>&1 & + sleep 60 + if run_alive; then + note "relaunch confirmed alive" + else + note "WARNING: relaunch failed to stay up - will retry next cycle" + fi + fi + sleep "$INTERVAL" +done +note "watchdog exit"