f/fundlab/narrative.py

"""Per-fund narrative: English prose explaining the performance and its
drivers, built from the analysis we have on file.

The prose is data-driven (every number comes from the computed results),
and it is required to be honest about limits: when the return drivers
are unknown or only weakly identified, the text says so explicitly.

Paragraphs:
  1. what the fund is (name, stated strategy if on file, what its
     N-PORT actually holds, which return-driver cluster it sits in)
  2. the performance in plain terms (full history, 5y, recent year,
     behaviour in the defined market episodes, calendar-year pattern)
  3. what is driving it (fit quality -> whose exposures vs idiosyncratic
     alpha; the economic meaning of the top loadings; weight stability)
  4. what we do NOT know (unexplained share, alpha steadiness, sample
     size, holdings opacity) - stated plainly
  5. bottom line (account placement, best cluster peers, role)

Run: .venv/bin/python -m fundlab.narrative   (prints all narratives)
"""
from __future__ import annotations

import json
from pathlib import Path

import numpy as np

from fundlab import report as _r
from fundlab.searchlist import BROAD_SLEEVES
from fundlab import decompose as _dc

HERE = Path(__file__).parent

_XC = json.loads((HERE / "xcheck_report.json").read_text()) \
    if (HERE / "xcheck_report.json").exists() else {}


def _pct(x, nd=1) -> str:
    if x is None or (isinstance(x, float) and np.isnan(x)):
        return "no data"
    return f"{100 * x:+.{nd}f}%"


def _num(x, nd=2) -> str:
    if x is None or (isinstance(x, float) and np.isnan(x)):
        return "n/a"
    return f"{x:.{nd}f}"


# short prose names for sleeves whose SLEEVE_DESC lead-in references a
# sibling ticker ("same HY exposure as VWEAX via a different fund")
_PROSE_NAME = {
    "vea": "international developed markets",
    "efa": "international developed markets",
    "hyg": "high-yield corporate bonds",
    "vweax": "high-yield corporate bonds",
    "dbb": "broad commodities",
    "gsg": "broad commodities",
    "vblix": "crash vol (VIX futures)",
    "bil": "cash (T-bills)",
    "shv": "cash (T-bills)",
}


def _sleeve_prose(sym: str, beta: float) -> str:
    nm, desc = _r.sleeve_desc(sym)
    short = _PROSE_NAME.get(sym)
    head = short.split(";")[0].strip() if short else \
        desc.split(";")[0].strip()
    head = head[0].upper() + head[1:] if head else nm
    if abs(beta) < 0.15:
        return f"a small tilt to {head.lower()}"
    if beta > 0:
        return f"exposure to {head.lower()}"
    return f"short-like exposure to {head.lower()} (it rises when that " \
           f"asset falls)"


def _top_sleeves(fr: dict, window: str, n: int = 3) -> list[tuple[str, float]]:
    f = fr.get(window) or {}
    betas = f.get("betas") or {}
    top = sorted(betas.items(), key=lambda kv: -abs(kv[1]))[:n]
    return [(s, b) for s, b in top if abs(b) >= 0.08]


def _fit(fr: dict, srow: dict) -> tuple[float | None, float | None,
                                        float | None]:
    """(r2, alpha_ann_5y, t5) preferring the reference-model numbers."""
    r2 = srow.get("r2_5y")
    a5 = srow.get("alpha_ann_5y")
    t5 = srow.get("alpha_t_5y")
    if not isinstance(r2, (int, float)):
        r2 = (fr.get("rec5") or fr.get("full") or {}).get("r2")
    if not isinstance(a5, (int, float)):
        a5 = (fr.get("rec5") or fr.get("full") or {}).get("alpha_ann")
    if not isinstance(t5, (int, float)):
        t5 = (fr.get("rec5") or fr.get("full") or {}).get("alpha_t")
    return (r2 if isinstance(r2, (int, float)) else None,
            a5 if isinstance(a5, (int, float)) else None,
            t5 if isinstance(t5, (int, float)) else None)


def narrate(sym: str) -> list[str]:
    sym = sym.lower()
    srow = _r.search_row(sym)
    fr = _r.factor_row(sym)
    dr = _r.decomp_row(sym)
    tax = _r.tax_row(sym)
    ddr = _r.dd_row(sym)
    xcr = _XC.get(sym, {})
    p = _r.price(sym)
    name = (srow.get("name") or _r.FONDS.get(sym, {}).get("name")
            or (fr or {}).get("name") or sym.upper())
    out: list[str] = []

    # ------------------------------------------------------------ 1. what
    who = []
    who.append(f"{sym.upper()} is the {name}.")
    obj = (_r.FONDS.get(sym) or {}).get("objective")
    if obj:
        first = obj.split(". ")[0].strip().rstrip(".")
        who.append(f"Stated strategy: {first}.")
    if xcr and not xcr.get("error"):
        b = [x["name"] for x in xcr.get("buckets", [])[:2]]
        if b:
            who.append(f"Per its latest N-PORT its actual holdings are "
                       f"mostly {b[0]}"
                       + (f" and {b[1]}" if len(b) > 1 else "") + ".")
    out.append(" ".join(who))

    if p is None:
        # no return history: do NOT claim a cluster - the loading vector
        # is all zeros and would park the fund in the cash cluster
        out.append("There is no local return history for this share "
                   "class yet (new class), so performance and drivers "
                   "cannot be measured from returns. The N-PORT shows "
                   "what it holds: " + ((dr or {}).get("note")
                                        or "see holdings"))
        return out
    if sym in _r.MEMBER:
        _cid, clabel, cn = _r.MEMBER[sym]
        who = out[0]
        out[0] = (who + f" In the return-driver analysis it sits in the "
                  f"'{clabel}' cluster with {cn - 1} other funds.")
    else:
        try:
            load = _r.fund_loading_row(fr)
            _cid, clabel = _r.cluster_of_row(load)
            out[0] = (out[0] + f" Its return-driver signature is closest "
                       f"to the '{clabel}' cluster.")
        except Exception:
            pass

    st = _r.perf_stats(p)
    r2, a5, t5 = _fit(fr, srow)
    tf = (fr.get("full") or {}).get("alpha_t")
    a_f = (fr.get("full") or {}).get("alpha_ann")
    refs = _r.ref_components(sym, fr)
    r2f_ = (fr.get("rec5") or fr.get("full") or {}).get("r2")
    fit = r2 if r2 is not None else r2f_
    r25 = srow.get("r2_5y") if isinstance(srow.get("r2_5y"), (int, float)) \
        else r2f_
    refs_curve = refs if (fit is not None and fit >= 0.5) else []
    mix = _r.mix_series(refs_curve) if refs_curve else _r.price("bil")
    ivv = _r.price("ivv")
    t5y = _r.window_ret(p, "2021-01-01", st["end"])
    t1y = _r.window_ret(p, "2025-09-01", st["end"])
    ref5y = _r.window_ret(mix, "2021-01-01", st["end"]) if mix is not None \
        else None
    iv5y = _r.window_ret(ivv, "2021-01-01", st["end"]) if ivv is not None \
        else None

    # --------------------------------------------------------- 2. performance
    perf = []
    tot = p.iloc[-1] / p.iloc[0] - 1
    perf.append(
        f"Since {st['start'][:4]} it has compounded at "
        f"{_pct(st['cagr'])} per year (a {_pct(tot, 0)} total return) "
        f"with {_pct(st['vol'], 0)} annualized volatility and a maximum "
        f"drawdown of {100 * abs(st['mdd']):.0f}%.")
    perf.append(
        f"Over the last five years it returned {_pct(t5y)} versus "
        f"{_pct(ref5y)} for its reference"
        + (f" and {_pct(iv5y)} for the S&P 500" if iv5y is not None else "")
        + f". The last twelve months have done {_pct(t1y)}.")
    yrs = _r.annual_table(p, 7)
    pos = sum(1 for _y, v in yrs if v > 0)
    perf.append(f"It was positive in {pos} of the last {len(yrs)} "
                f"calendar years (including the partial current year).")
    # episodes
    rets = ddr.get("rets", {})
    if rets:
        best = max(rets.items(), key=lambda kv: kv[1])
        worst = min(rets.items(), key=lambda kv: kv[1])
        perf.append(
            f"Across the defined market episodes it did best in "
            f"'{best[0]}' ({_pct(best[1])}) and worst in '{worst[0]}' "
            f"({_pct(worst[1])}) - the peak-to-trough windows when "
            f"equities fell hardest.")
        _dv = ddr.get("verdict") or ""
        if _dv and not _dv.startswith("CANDIDATE"):
            perf.append(f"Its drawdown screen verdict: {_dv}.")
    out.append(" ".join(perf))

    # --------------------------------------------------------- 3. drivers
    drv = []
    tops = _top_sleeves(fr, "rec5", 3)
    if fit is not None and fit >= 0.7:
        tops_txt = ", ".join(_sleeve_prose(s, b) for s, b in tops)
        drv.append(
            f"The fit is strong: {100 * fit:.0f}% of its excess returns "
            f"over the last five years are explained by its measured "
            f"exposures"
            + (f" - {tops_txt}" if tops else "") + ".")
        drv.append(
            f"In other words, most of what this fund does is charge you "
            f"for those exposures in the form of a fund; what is left - "
            f"an alpha of {_pct(a5)} per year (t = {_num(t5, 1)}) - is "
            + ("statistically indistinguishable from zero, i.e. the "
               "strategy is not clearly adding anything on top of the "
               "mix it owns."
               if (t5 or 0) < 1.75 else
               "small relative to the beta, but it is what separates "
               "this fund from the equivalent index mix.")
        )
    elif fit is not None and fit >= 0.5:
        tops_txt = ", ".join(_sleeve_prose(s, b) for s, b in tops)
        drv.append(
            f"The measured exposures explain a good share but not all "
            f"({100 * fit:.0f}% R²) of its excess returns."
            + (f" The main ones: {tops_txt}." if tops else ""))
        drv.append(
            f"The residual is {_pct(a5)} per year (t = {_num(t5, 1)}): "
            + ("a real edge on top of the mix, but a material part of "
               "the performance IS the mix - judge the exposures and "
               "the skill separately."
               if (t5 or 0) >= 1.75 else
               "not statistically distinct from noise at the five-year "
               "horizon, so treat the outperformance as part beta, part "
               "luck until it accumulates more history.")
            )
    else:
        tops_txt = ", ".join(_sleeve_prose(s, b) for s, b in tops)
        drv.append(
            f"The broad sleeve framework explains little of its excess "
            f"returns (R² = {_num(fit) if fit is not None else 'n/a'}). "
            + (f"The loadings that do show up - {tops_txt} - are minor "
               "tilts, not the story."
               if tops else
               "No benchmark loading is even large."))
        long_ctx = ""
        if isinstance(a_f, (int, float)):
            long_ctx = (f" Over the full history the same estimate is "
                        f"{_pct(a_f)} per year (t = {_num(tf, 1)}).")
        if (t5 or 0) >= 2:
            drv.append(
                f"The bulk of the performance is therefore "
                f"idiosyncratic: it comes from the fund's own holdings "
                f"and decisions, which nothing in our 34-sleeve space "
                f"replicates. The estimate of that idiosyncratic return "
                f"is {_pct(a5)} per year over the last five years "
                f"(t = {_num(t5, 1)}) - statistically significant, so "
                f"the outperformance is real; what the returns alone "
                f"cannot tell us is its source.{long_ctx}")
        elif (a5 or 0) >= 0:
            drv.append(
                f"The bulk of the performance is therefore "
                f"idiosyncratic: it comes from the fund's own holdings "
                f"and decisions, which nothing in our 34-sleeve space "
                f"replicates. The estimate of that idiosyncratic return "
                f"is {_pct(a5)} per year over the last five years "
                f"(t = {_num(t5, 1)}) - at that t-stat it is not yet "
                f"clearly different from a lucky streak.{long_ctx}")
        else:
            drv.append(
                f"Most of the performance is therefore idiosyncratic "
                f"rather than benchmark-like. The honest five-year read "
                f"of its excess return is {_pct(a5)} per year "
                f"(t = {_num(t5, 1)}): over the recent window this fund "
                f"is NOT measurably adding to its fitted reference.{long_ctx}")
        if (a5 or 0) < 0 and isinstance(tf, (int, float)) and tf >= 1.25:
            drv.append(
                f"The longer record is better: over the full history the "
                f"alpha is {_pct(a_f)} per year (t = {_num(tf, 1)}), "
                f"which IS significant - so the five-year estimate "
                f"looks like a flat patch in a longer positive trend, "
                f"not evidence the edge has gone.")
        if xcr and not xcr.get("error"):
            b = [x["name"] for x in xcr.get("buckets", [])[:2]]
            if b:
                drv.append(
                    f"The N-PORT is consistent with that: the alpha is "
                    f"plausibly coming from {b[0]}"
                    + (f" and {b[1]}" if len(b) > 1 else "")
                    + " - asset classes our sleeve set either does not "
                      "model directly or models too coarsely. That is an "
                      "interpretation from the holdings, not something "
                      "the returns prove.")
        if obj and (a5 or 0) > 0:
            _o = obj.split(". ")[0].rstrip(".")
            if len(_o) > 110:
                _o = _o[:110].rsplit(" ", 1)[0] + "..."
            drv.append(
                f"The stated strategy ('{_o}') is a plausible mechanism "
                f"as well, but that is a hypothesis - the returns only "
                f"tell us the outperformance exists, not why.")
    # stability (curated funds)
    roll = (dr or {}).get("rolling") or {}
    if "verdict" in (dr or {}) and roll.get("max_drift") is not None:
        md = roll["max_drift"]
        drv.append(
            f"The curated decomposition's read: {dr['verdict']}. "
            f"Weight stability: the one-year rolling betas move by at "
            f"most {md:.2f} (relative to the full-sample weights) - "
            + ("so the weights are roughly stable over time."
               if md < 0.5 else
               "so treat this as a tactically active fund, not a "
               "static mix."))
    out.append(" ".join(drv))

    # ------------------------------------------------- 4. what we do NOT know
    unk = []
    if fit is not None:
        if (t5 or 0) >= 2:
            unk.append(
                f"To be explicit about the limits: "
                f"{100 * (1 - fit):.0f}% of this fund's excess return "
                f"is NOT attributed to any measured exposure by our "
                f"analysis - we know the outperformance is real, we do "
                f"not know from returns what produces it; the source "
                f"has to be read from the filings before sizing up.")
        else:
            unk.append(
                f"To be explicit about the limits: the excess return is "
                f"not measurable against noise at the current sample "
                f"size, so there is not even a reliable alpha to "
                f"attribute; {100 * (1 - fit):.0f}% of the return is "
                f"outside our sleeve model either way.")
    pf = srow.get("alpha_pos_frac")
    if isinstance(pf, (int, float)) and pf < 0.60:
        unk.append(
            f"The alpha is also not steady: in {100 * (1 - pf):.0f}% of "
            f"rolling six-month windows the fund trailed its reference, "
            f"so the five-year number is carrying periods of "
            f"underperformance.")
    if (t5 or 0) >= 2 and isinstance(tf, (int, float)) and tf < 1.25:
        unk.append(
            "The significance is recent - over the full history the "
            "alpha is not significant, so this is a provisional read on "
            "a ~5-year sample.")
    if st["years"] < 3:
        unk.append(
            f"The whole sample is short ({st['years']:.1f} years), so "
            f"every number above has a wide error bar.")
    if xcr.get("note"):
        unk.append(xcr["note"].rstrip(".") + ".")
    note = (dr or {}).get("note")
    if note and "N-PORT" in note:
        # holdings cross-check sentence (shortlist)
        for sentence in note.split(". "):
            if "N-PORT" in sentence or "holdings" in sentence.lower():
                unk.append(sentence.strip().rstrip(".") +
                           " - the holdings check is the ground truth "
                           "here, and the return model is only a "
                           "description of it.")
                break
    if not unk:
        unk.append("No major caveats beyond the usual: the past fit may "
                   "not persist, and the sleeves are models of the "
                   "fund, not the fund itself.")
    out.append(" ".join(unk))

    # --------------------------------------------------------- 5. bottom line
    bot = []
    loc = tax.get("location")
    if loc:
        basis = "N-PORT holdings" if tax.get("basis") == "N-PORT" \
            else "the sleeve model"
        if "MIXED" in loc:
            bot.append(
                f"Account placement: not settled by the model - the "
                f"sleeve model calls it MIXED, so the last 1099-DIV is "
                f"the arbiter before choosing taxable vs IRA.")
        else:
            bot.append(
                f"Account placement: "
                f"{'taxable' if 'TAXABLE' in loc else 'IRA'} "
                f"({basis} says {loc}).")
    # peers
    if sym in _r.MEMBER:
        cid = _r.MEMBER[sym][0]
        members = _r.KMEANS["clusters"][cid]["syms"]
        peers = []
        for s in members:
            if s == sym:
                continue
            v = _r.search_row(s)
            if isinstance(v.get("alpha_t_5y"), (int, float)) and \
                    v["alpha_t_5y"] > 0:
                pp = _r.price(s)
                if pp is None:
                    continue
                peers.append((v["alpha_t_5y"], v.get("alpha_ann_5y"), s))
        peers.sort(reverse=True)
        if peers:
            b_t, b_a, b_s = peers[0]
            bp = _r.price(b_s)
            bt5 = _r.window_ret(bp, "2021-01-01",
                                _r.perf_stats(bp)["end"])
            bot.append(
                f"Within its cluster the strongest alternative is "
                f"{b_s.upper()} (5y {_pct(bt5)}, alpha t = {b_t:+.1f}); "
                f"the choice between them should turn on the conviction "
                f"in the strategy, the tax fit and the price paid, not "
                f"on the statistics - they are the same kind of "
                f"position.")
    cp = srow.get("corr_portfolio")
    if isinstance(cp, (int, float)):
        bot.append(
            f"Its correlation with your current portfolio is "
            f"{cp:+.2f} - "
            + ("a genuine diversifier" if cp < 0.3
               else "not very diversified vs what you already own".rstrip("."))
            + ".")
    out.append(" ".join(bot))
    return out


def main() -> None:
    from fundlab.nport import FUND_TOKENS
    from fundlab.decompose import ALIAS
    cand = [v["sym"] for v in _r.SEARCH.values()
            if isinstance(v, dict)
            and str(v.get("verdict", "")).startswith("CANDIDATE")]
    short = [s for s in FUND_TOKENS if s not in ALIAS]
    for s in cand + short:
        print(f"=== {s.upper()} ===")
        for para in narrate(s):
            print(para)
            print()


if __name__ == "__main__":
    main()