#!/usr/bin/env python3
"""Detect stale tickers (Yahoo now returns no price data) and freeze them.

A ticker is "stale" when its {SYM}.json in the data root is a Yahoo chart
response WITHOUT a `timestamp` array. goget still overwrites that .json on
every download pass, and ohlc.Conv then refuses to (re)write the CSVs — so
the existing CSVs are the last complete series we will ever get from Yahoo
(usually because the fund was terminated, merged or liquidated).

This script snapshots those CSVs into overrides/frozen/ so the data survives
any future goget re-download, and writes reports/stale-funds.md for the
Tier-3 (official source) verification workflow.

The data root is never modified. Frozen copies are created/refreshed only
when the root CSVs are NEWER than the frozen copy (i.e. a download extended
the series before Yahoo went empty), so a re-run is idempotent.

Usage: python3 scripts/audit_stale.py [data_root]
"""

import json
import shutil
import sys
from datetime import date
from pathlib import Path

ROOT = Path(sys.argv[1]).expanduser() if len(sys.argv) > 1 else Path("~/prog/fin/stocks").expanduser()
FROZEN = Path(__file__).parent.parent / "overrides" / "frozen"
REPORT = Path(__file__).parent.parent / "reports" / "stale-funds.md"
SUFFIXES = ("history", "dividend", "capitalGain")


def stale_symbols(root: Path):
    """{(sym, longName)} for JSONs without a timestamp array."""
    out = []
    for jf in sorted(root.glob("*.json")):
        try:
            d = json.loads(jf.read_text())
            r = d.get("chart", {}).get("result")
            if not isinstance(r, list) or not r:
                continue  # not a chart payload (e.g. stray files)
            r0 = r[0]
            if r0.get("timestamp"):
                continue
            out.append((jf.stem, r0.get("meta", {}).get("longName", "?")))
        except Exception:
            continue
    return out


def last_bar(f: Path) -> str:
    if not f.exists():
        return ""
    lines = f.read_text(errors="replace").strip().splitlines()
    if len(lines) < 2:
        return ""
    return lines[-1].split(",")[0]


def main():
    FROZEN.mkdir(parents=True, exist_ok=True)
    stale = stale_symbols(ROOT)
    rows = []
    n_new = n_refresh = n_keep = 0
    for sym, name in stale:
        meta_path = FROZEN / f"{sym}.json"
        meta = {}
        if meta_path.exists():
            meta = json.loads(meta_path.read_text())

        # refresh frozen CSVs if the root CSVs moved forward
        for sfx in SUFFIXES:
            src = ROOT / f"{sym}-{sfx}.csv"
            dst = FROZEN / f"{sym}-{sfx}.csv"
            if not src.exists():
                continue
            if dst.exists() and dst.stat().st_mtime_ns >= src.stat().st_mtime_ns:
                continue
            if dst.exists():
                # root CSV changed: verify it is a superset (newer last bar),
                # never shrink the frozen series
                if last_bar(src) < last_bar(dst):
                    print(f"WARN {sym}: root {sfx} last bar {last_bar(src)!r} is OLDER than "
                          f"frozen {last_bar(dst)!r}; keeping frozen copy")
                    continue
                n_refresh += 1
            else:
                n_new += 1
            shutil.copy2(src, dst)

        meta["symbol"] = sym
        meta["longName"] = name
        meta["freeze_date"] = last_bar(FROZEN / f"{sym}-history.csv")
        meta.setdefault("created", date.today().isoformat())
        meta["reason"] = "yahoo-empty"
        meta_path.write_text(json.dumps(meta, indent=1))
        n_keep += 1
        rows.append((sym, name, meta["freeze_date"],
                     meta.get("verified", False), meta.get("note", "")))

    rows.sort(key=lambda r: (r[2] == "", r[2], r[0]))
    REPORT.parent.mkdir(parents=True, exist_ok=True)
    with REPORT.open("w") as f:
        f.write("# Stale (Yahoo-empty) tickers — frozen in `overrides/frozen/`\n\n")
        f.write("Data root copies are snapshots; `data.py` reads the frozen copies for these "
                "symbols. Tier-3 verification: cross-check distributions/termination against "
                "official sources, then set `verified: true` + `note` in `overrides/frozen/{SYM}.json`.\n\n")
        f.write("| Symbol | Name | Last bar | Verified | Note |\n")
        f.write("|---|---|---|---|---|\n")
        for sym, name, fb, ver, note in rows:
            f.write(f"| {sym} | {name} | {fb} | {'yes' if ver else ''} | {note} |\n")
    print(f"{len(stale)} stale tickers; {n_new} new frozen, {n_refresh} refreshed, "
          f"{n_keep} metadata entries; report -> {REPORT}")


if __name__ == "__main__":
    main()