77 lines
2.4 KiB
Bash
Executable File
77 lines
2.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Watchdog for the overnight comprehensive screen.
|
|
#
|
|
# The pipeline (fundlab.overnight) is fully resumable: every stage skips
|
|
# work already in its cache. So if the python process dies before it has
|
|
# screened the whole selected worklist, we simply relaunch the same
|
|
# command and it picks up where it left off.
|
|
#
|
|
# "Complete" = search_all.json holds at least as many symbols as
|
|
# selected.json (the screen stage wrote every fund). Then we stop.
|
|
#
|
|
# Usage: setsid nohup fundlab/watchdog.sh </dev/null >>fundlab/watchdog.log 2>&1 &
|
|
set -u
|
|
cd "$(dirname "$0")/.."
|
|
PY=".venv/bin/python"
|
|
INTERVAL=300 # check every 5 min
|
|
|
|
stamp() { date "+%Y-%m-%d %H:%M:%S"; }
|
|
note() {
|
|
echo "[$(stamp)] $*"
|
|
# mirror into the research log so the knowledge survives
|
|
echo "- $(stamp) watchdog: $*" >> fundlab/RESEARCH.md
|
|
}
|
|
|
|
need_syms() { # how many funds still need screening
|
|
"$PY" - <<'EOF'
|
|
import json, os
|
|
sel = "fundlab/universe_cache/selected.json"
|
|
res = "fundlab/search_all.json"
|
|
if not os.path.exists(sel):
|
|
print(0); raise SystemExit
|
|
n = len(json.load(open(sel)))
|
|
done = len(json.load(open(res))) if os.path.exists(res) else 0
|
|
print(max(0, n - done))
|
|
EOF
|
|
}
|
|
|
|
run_alive() {
|
|
# prefer the pidfile the run process writes (immune to pattern
|
|
# false-positives, e.g. a monitor shell containing the string);
|
|
# fall back to a pattern match for pre-pidfile launches.
|
|
local pidfile="fundlab/overnight.pid"
|
|
if [ -f "$pidfile" ]; then
|
|
local pid; pid=$(cat "$pidfile" 2>/dev/null)
|
|
if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
return 1 # stale pidfile: the process is gone
|
|
fi
|
|
pgrep -f "python -m fundlab\.overnight" >/dev/null 2>&1
|
|
}
|
|
|
|
note "watchdog started (interval ${INTERVAL}s)"
|
|
while true; do
|
|
if run_alive; then
|
|
note "overnight alive, still_missing=$(need_syms)"
|
|
else
|
|
missing=$(need_syms)
|
|
if [ "${missing:-0}" -eq 0 ]; then
|
|
note "process gone but screen COMPLETE (0 missing) - watchdog exiting"
|
|
rm -f fundlab/overnight.pid
|
|
break
|
|
fi
|
|
note "process DEAD with ${missing} funds unscreened - relaunching"
|
|
setsid nohup "$PY" -m fundlab.overnight download screen finalize \
|
|
</dev/null >> fundlab/overnight.log 2>&1 &
|
|
sleep 60
|
|
if run_alive; then
|
|
note "relaunch confirmed alive"
|
|
else
|
|
note "WARNING: relaunch failed to stay up - will retry next cycle"
|
|
fi
|
|
fi
|
|
sleep "$INTERVAL"
|
|
done
|
|
note "watchdog exit"
|