#!/usr/bin/env bash
# Watchdog for the overnight comprehensive screen.
#
# The pipeline (fundlab.overnight) is fully resumable: every stage skips
# work already in its cache. So if the python process dies before it has
# screened the whole selected worklist, we simply relaunch the same
# command and it picks up where it left off.
#
# "Complete" = search_all.json holds at least as many symbols as
# selected.json (the screen stage wrote every fund). Then we stop.
#
# Usage: setsid nohup fundlab/watchdog.sh </dev/null >>fundlab/watchdog.log 2>&1 &
set -u
cd "$(dirname "$0")/.."
PY=".venv/bin/python"
INTERVAL=300            # check every 5 min

stamp() { date "+%Y-%m-%d %H:%M:%S"; }
note() {
  echo "[$(stamp)] $*"
  # mirror into the research log so the knowledge survives
  echo "- $(stamp) watchdog: $*" >> fundlab/RESEARCH.md
}

need_syms() {  # how many funds still need screening
  "$PY" - <<'EOF'
import json, os
sel = "fundlab/universe_cache/selected.json"
res = "fundlab/search_all.json"
if not os.path.exists(sel):
    print(0); raise SystemExit
n = len(json.load(open(sel)))
done = len(json.load(open(res))) if os.path.exists(res) else 0
print(max(0, n - done))
EOF
}

run_alive() {
  # prefer the pidfile the run process writes (immune to pattern
  # false-positives, e.g. a monitor shell containing the string);
  # fall back to a pattern match for pre-pidfile launches.
  local pidfile="fundlab/overnight.pid"
  if [ -f "$pidfile" ]; then
    local pid; pid=$(cat "$pidfile" 2>/dev/null)
    if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then
      return 0
    fi
    return 1   # stale pidfile: the process is gone
  fi
  pgrep -f "python -m fundlab\.overnight" >/dev/null 2>&1
}

note "watchdog started (interval ${INTERVAL}s)"
while true; do
  if run_alive; then
    note "overnight alive, still_missing=$(need_syms)"
  else
    missing=$(need_syms)
    if [ "${missing:-0}" -eq 0 ]; then
      note "process gone but screen COMPLETE (0 missing) - watchdog exiting"
      rm -f fundlab/overnight.pid
      break
    fi
    note "process DEAD with ${missing} funds unscreened - relaunching"
    setsid nohup "$PY" -m fundlab.overnight download screen finalize \
      </dev/null >> fundlab/overnight.log 2>&1 &
    sleep 60
    if run_alive; then
      note "relaunch confirmed alive"
    else
      note "WARNING: relaunch failed to stay up - will retry next cycle"
    fi
  fi
  sleep "$INTERVAL"
done
note "watchdog exit"