f/scripts/check_adj_consistency.py

#!/usr/bin/env python3
"""Check Yahoo internal consistency for a set of symbols.

For each symbol we compare three "total return" views over a window:

  1. TR(Adj)      : CAGR of Yahoo's Adj Close series (their dividend/split
                    adjustment, whatever vintage they computed it from)
  2. TR(sim)      : CAGR of a pre-tax reinvestment simulation built from the
                    RAW Close series + the declared dividend/capitalGain event
                    files (the same inputs the after-tax engine consumes)
  3. declared yield: (sum div + sum capg) / avg price over the window

Views 1 and 2 must agree if Yahoo's adj factor was built from the same event
history as the event files. A large gap means the two data sources are
inconsistent (stale adj vintage, restated/stacked events, mislabeled events,
or double-counting of the same distribution in both files).

Also checks the "post > pre is impossible" invariant directly, and analyzes
same-date div/capG event overlap (double-listing patterns the dedup invariant
may not catch, e.g. dividend file carrying the TOTAL distribution =
dividend + capgain, with a different amount than the capGain file).

Usage:  python scripts/check_adj_consistency.py [SYM ...]
        (default: the pool in POOL below)
Output: stdout table + reports/adj_consistency/check.md
"""
from __future__ import annotations

import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
import data as D  # noqa: E402

POOL = [
    "bntx", "pmaix", "slg", "atesx", "cosix", "egrax", "pfe", "qspnx",
    "svarx", "cvsix", "flcsx", "menkx", "flpsx", "spy",
]
START = "2016-01-01"


def cagr(a: float, b: float, yrs: float) -> float:
    return (b / a) ** (1 / yrs) - 1


def sim_tr(close: pd.Series, div: pd.Series | None, capg: pd.Series | None,
           start: str) -> float | None:
    """Pre-tax total-return CAGR from raw close + declared events."""
    import numpy as np
    c = close.loc[start:].dropna()
    if len(c) < 250:
        return None
    parts = []
    for s_ in (div, capg):
        parts.append(s_.loc[start:].reindex(c.index).fillna(0.0)
                     if s_ is not None else pd.Series(0.0, index=c.index))
    dist = parts[0] + parts[1]
    prev = c.shift(1)
    # standard dividend adjustment: on ex-date units *= P_prev / (P_prev - d)
    adj_factor = pd.Series(1.0, index=c.index)
    m = (dist > 0) & prev.notna() & (prev > dist)
    adj_factor[m] = prev[m] / (prev[m] - dist[m])
    units = adj_factor.cumprod() / c.iloc[0]
    value = units * c
    yrs = (c.index[-1] - c.index[0]).days / 365.25
    return float((value.iloc[-1] / value.iloc[0]) ** (1 / yrs) - 1)


def main(syms: list[str]) -> None:
    import numpy as np  # noqa: F401
    b = D.load_bundle()
    rows = []
    for s in syms:
        if s not in b.close.columns:
            rows.append((s, "MISSING", 0, 0, 0, 0, 0, "", ""))
            continue
        c = b.close[s].loc[START:].dropna()
        a = b.adj[s].loc[START:].dropna()
        if len(c) < 250 or len(a) < 250:
            rows.append((s, "TOO_SHORT", 0, 0, 0, 0, 0, "", ""))
            continue
        yrs = (c.index[-1] - c.index[0]).days / 365.25
        po = cagr(float(c.iloc[0]), float(c.iloc[-1]), yrs)
        tr_adj = cagr(float(a.iloc[0]), float(a.iloc[-1]), yrs)
        d = b.div[s].loc[START:].fillna(0.0) if s in b.div.columns else None
        g = b.capg[s].loc[START:].fillna(0.0) if s in b.capg.columns else None
        sdiv = float(d.sum()) if d is not None else 0.0
        scap = float(g.sum()) if g is not None else 0.0
        decl = (sdiv + scap) / float(c.mean())
        sim = sim_tr(b.close[s], d, g, START)
        gap = (sim - tr_adj) if sim is not None else float("nan")
        # same-date overlap analysis
        overlap = ""
        if d is not None and g is not None:
            both = (d > 0) & (g > 0)
            n = int(both.sum())
            if n:
                dv, gv = d[both], g[both]
                exact = int((abs(dv - gv) < 1e-6 * gv).sum())
                divgt = int((dv > gv * 1.05).sum())
                overlap = f"same-date {n} (exact {exact}, div>cap {divgt})"
            else:
                overlap = "same-date 0"
        flag = ""
        if sim is not None and abs(gap) > 0.003:
            flag = "INCONSISTENT"
        if sim is not None and sim > tr_adj + 0.001:
            flag += " POST>PRE"
        rows.append((s, flag, po, tr_adj, sim if sim is not None else float("nan"),
                     decl, gap if sim is not None else float("nan"),
                     f"div {sdiv:.2f} capg {scap:.2f}", overlap))
    w = max(len(r[0]) for r in rows)
    print(f"{'sym':<{w}} {'flag':18s} {'price-only':>10s} {'TR(adj)':>8s} "
          f"{'TR(sim)':>8s} {'decl.yld':>9s} {'gap':>8s}  {'dist':22s} {''}")
    for r in rows:
        s, flag, po, tr, sim, decl, gap, dist, ov = r
        print(f"{s:<{w}} {flag:18s} {po:10.2%} {tr:8.2%} {sim:8.2%} {decl:9.2%} "
              f"{gap:8.2%}  {dist:22s} {ov}")
    # report
    rep = ROOT / "reports" / "adj_consistency"
    rep.mkdir(parents=True, exist_ok=True)
    with open(rep / "check.md", "w") as fh:
        fh.write(f"# Adj/event consistency check (window from {START})\n\n")
        fh.write("TR(sim) = pre-tax reinvestment of raw Close + declared "
                 "dividend/capitalGain events. Must match TR(adj) if Yahoo's\n"
                 "adj factor was built from the same history. POST>PRE = "
                 "post-tax engine\nwould beat pre-tax: impossible, data broken.\n\n")
        fh.write("| sym | flag | price-only | TR(adj) | TR(sim) | decl yield | gap |\n")
        fh.write("|---|---|---|---|---|---|---|\n")
        for r in rows:
            s, flag, po, tr, sim, decl, gap, dist, ov = r
            fh.write(f"| {s} | {flag} | {po:.2%} | {tr:.2%} | {sim:.2%} | "
                     f"{decl:.2%} | {gap:+.2%} |\n")
        fh.write("\n## details\n\n")
        for r in rows:
            s, flag, po, tr, sim, decl, gap, dist, ov = r
            fh.write(f"- **{s}**: {dist}; {ov}\n")
    print(f"\nreport: {rep / 'check.md'}")


if __name__ == "__main__":
    import pandas as pd  # noqa: E402  (type hint in sim_tr)
    main(sys.argv[1:] or POOL)