diff --git a/app.py b/app.py index 70a9c48..9e0c752 100644 --- a/app.py +++ b/app.py @@ -595,25 +595,124 @@ with tab_fundlab: _DR = json.loads(_dc.RESULTS.read_text()) except Exception: _DR = {} - with st.expander(f"Summary — all {len(_FL_LABELS)} funds"): - _sum_rows = [] - for _s in _ORDER: - _d = _DR.get(_s, {}) - _rec = _d.get("recent", {}) - _comp = ", ".join(f"{c['sym']} {c['beta']:+.2f}" - for c in _rec.get("components", [])[:4]) or \ - ", ".join(f"{c['sym']} {c['beta']:+.2f}" - for c in _d.get("components", [])[:4]) - _sum_rows.append({ - "fund": _FL_LABELS.get(_s, _s), - "verdict": _d.get("verdict", "n/a"), - "R² 5y": round(_rec["r2"], 3) if "r2" in _rec else None, - "alpha 5y": (f"{_rec['alpha_ann']*100:+.1f}% " - f"(t={_rec['alpha_t']:+.1f})") - if "alpha_ann" in _rec else None, - "components": _comp or "—", - }) - st.dataframe(pd.DataFrame(_sum_rows), width="stretch") + # --- per-fund report: narrative + performance + drivers + peers ---- + # static build (fundlab.reportdata) so the page stays fast + _RD = Path(__file__).parent / "reports" / "report_data.json" + if not _RD.exists(): + with st.expander("Summary - per-fund reports (not built yet)"): + st.info("Run `.venv/bin/python -m fundlab.reportdata` in the " + "project root, then refresh.") + else: + try: + _rd = json.loads(_RD.read_text()) + _rfunds = _rd["funds"] + _rorder = sorted(_rfunds, + key=lambda s: _rfunds[s]["meta"]["order"]) + except Exception as _e: # noqa: BLE001 + st.warning(f"report data unreadable: {_e}") + _rfunds = {} + if _rfunds: + st.subheader(f"Summary - per-fund reports " + f"({len(_rfunds)} funds)") + st.caption( + f"Generated {_rd['generated']}. Each fund: a narrative " + "discussion first (the performance, what drives it, and " + "what we do NOT know), then the equity curve, the period " + "table (fund vs fitted reference vs S&P 500, with the " + "fund-minus-reference gap), the return drivers, the " + "reference mix explained sleeve by sleeve, tax placement, " + "and the best peers from the same return-driver cluster. " + "All alphas are in excess of the 3-mo T-bill rate.") + _ag = [] + for _s in _rorder: + _f = _rfunds[_s] + _m, _st = _f["meta"], _f.get("stats", {}) + _ag.append({ + "fund": f"{_m['sym'].upper()} — {_m['name'][:44]}", + "group": _m["group"], + "5y": _st.get("t5y", "—"), + "CAGR": _st.get("cagr", "—"), + "maxDD": _st.get("mdd", "—"), + "R² 5y": _st.get("r2_5y", "—"), + "alpha 5y": _st.get("alpha_5y", "—"), + "cluster": (_f.get("peers") or {}).get("cluster", ""), + }) + st.dataframe(pd.DataFrame(_ag), width="stretch", + hide_index=True) + _rall = st.checkbox("Expand all fund reports", value=False, + key="fl_rpt_expand") + for _s in _rorder: + _f = _rfunds[_s] + _m, _st = _f["meta"], _f.get("stats", {}) + _grp = "C" if _m["group"] == "candidate" else "S" + _title = (f"{_grp}{_m['order']:02d} · {_m['sym'].upper()} " + f"— {_m['name']}") + if _m.get("verdict"): + _title += f" [{_m['verdict'][:44]}]" + with st.expander(_title, expanded=_rall): + if _f.get("narrative"): + st.markdown("\n\n".join(_f["narrative"])) + _c = _f.get("chart", {}) + if _c.get("dates"): + import plotly.graph_objects as _go + _fig = _go.Figure() + _fig.add_trace(_go.Scatter( + x=pd.to_datetime(_c["dates"]), + y=_c["fund"], name=_m["sym"].upper(), + line=dict(width=2))) + if _c.get("ref"): + _fig.add_trace(_go.Scatter( + x=pd.to_datetime(_c["ref_dates"]), + y=_c["ref"], name="fitted reference", + line=dict(width=1.2, dash="dash"))) + if _c.get("ivv"): + _fig.add_trace(_go.Scatter( + x=pd.to_datetime(_c["ivv_dates"]), + y=_c["ivv"], name="S&P 500 (IVV)", + line=dict(width=1, dash="dot"), + opacity=0.7)) + _fig.update_layout( + height=420, margin=dict(l=10, r=10, t=25, b=10), + legend=dict(orientation="h", y=1.1), + hovermode="x unified", + title=f"total return since " + f"{_c['dates'][0][:4]} (rebased 100)") + st.plotly_chart(_fig, width="stretch") + if _f.get("perf"): + st.dataframe(pd.DataFrame( + _f["perf"], + columns=["period", "fund", "reference", + "S&P 500", "fund − reference"]), + width="stretch", hide_index=True) + st.caption( + "fund − reference = period alpha/timing (the " + "part of that period the fitted mix does not " + "explain). For cash-anchored funds the " + "reference is the T-bill rate itself.") + if _f.get("drivers"): + st.markdown("\n\n".join(_f["drivers"])) + _ref = _f.get("reference") + if _ref and _ref.get("rows"): + st.markdown(f"**Reference mix** ({_ref['netcash']})") + st.dataframe(pd.DataFrame( + _ref["rows"], + columns=["loading", "what it is", + "what it exposes you to"]), + width="stretch", hide_index=True) + st.caption(_ref["text"]) + if _f.get("tax"): + st.markdown(f"**Tax** — {_f['tax']}") + _pe = _f.get("peers") + if _pe and _pe.get("rows"): + st.markdown( + f"**Peers — cluster: {_pe['cluster']}** " + f"(n={_pe['n']})") + st.dataframe(pd.DataFrame( + _pe["rows"], + columns=["fund", "5y", "CAGR", "maxDD", + "R² 5y", "alpha 5y", "tax"]), + width="stretch", hide_index=True) + st.caption(_pe["proscons"]) # --- alpha search: all screened funds (shortlist + longlist + harvest) with st.expander("Alpha search — all screened funds, ranked"): diff --git a/fundlab/RESEARCH.md b/fundlab/RESEARCH.md index e4e821c..9ba7a49 100644 --- a/fundlab/RESEARCH.md +++ b/fundlab/RESEARCH.md @@ -1073,3 +1073,66 @@ search_all.json; the app's alpha-search table now has a "reference **Backups of the raw-alpha era (NOT deleted):** factor_results_rawalpha.json, decompose_results_rawalpha.json, search_all_rawalpha.json, cluster_kmeans_rawalpha.json. + +--- + +## Per-fund report: HTML + app + narrative engine (2026-08-30) + +**Deliverable.** `fundlab/report.py` -> `reports/fund_report.html` +(self-contained, plotly inlined, 20 MB) and `fundlab/reportdata.py` -> +`reports/report_data.json` (3 MB, the app's static build). 24 funds: the +11 excess-return candidates + the 13 unique shortlist funds (pmfkx/lcrix/ +egrsx are share classes of pmaix/lcorx/eagmx and are merged). + +Per fund: max-history equity curve (fund vs fitted reference vs IVV); +period table (full/5y/1y, the 5 market episodes, 6 calendar years) with +the **fund-minus-reference** column (period alpha/timing); drivers +(reference-model R²/alpha/t + 34-sleeve signature + curated decomposition +verdict + N-PORT cross-check notes); the reference mix explained +sleeve-by-sleeve (SLEEVE_DESC: what each exposure actually is); tax +character + placement; and a peer table of the 4 best-alpha funds in the +same k=30 return-driver cluster with computed adv/dis. + +**BUG FOUND AND FIXED — `mix_series()` never applied the betas.** The +"fitted reference" curve summed the RAW sleeve daily returns instead of +β·return. For high-fit funds the reference was garbage (JLPSX "reference" ++407% 5y vs the fund's +123%; CVSIX +465%; PMAIX +1160%). After the fix +the mix tracks the fund (JLPSX ref +127% vs fund +123%). Every reference +curve and table in the HTML report was wrong before this and has been +regenerated. Lesson: a regression on returns does not hand you a price +path — the β weighting is the whole point. + +**Weak-fit funds anchor to CASH, not their fitted mix.** For idiosyncratic +funds (R² < 0.5) the BIC forward selection can pick a statistically-thin +OFFSETTING combination (SCFZX: +0.35 VIX futures, −0.23 20+y Treasuries, ++0.10 HY) whose cumulated path is meaningless (a "reference" that loses +79% in 2022 for a fund that lost 2.4%). Below the 0.5 fit gate the +performance table and curve use BIL as the reference; the loadings are +still shown with an explicit "weak fit — read with care" caveat. + +**Narrative engine (`fundlab/narrative.py`).** Data-driven English prose, +5 paragraphs per fund: (1) what the fund is (objective, N-PORT buckets, +cluster); (2) the performance in plain terms (CAGR/vol/DD, 5y vs reference, +episode behaviour, calendar-year pattern); (3) what is driving it — tiered +by fit (≥0.7 "mostly the exposures", 0.5–0.7 "mix + residual", <0.5 +"idiosyncratic: here is the estimate and its t-stat"); (4) **what we do +NOT know** — stated explicitly: the unexplained share, alpha steadiness +(share of 6m windows trailing), recent-vs-full significance, short sample, +holdings opacity (egrix: 100% in one managed fund, underlying undisclosed); +(5) bottom line (account placement, best cluster peer, portfolio +correlation). Rules learned while reviewing the prose: negative/weak 5y +alpha must not be called "outperformance" (the template assumed positive +alpha and lied about atesx, −1.6% t=−0.5); MIXED tax placement must not be +rendered as "IRA"; a fund with no price history must not be assigned a +cluster (zero loading vector parks everything in the cash cluster); +drift < 0.5 = "roughly stable" (0.44 is not "tactical"); sleeve prose +needs short names (dbb/gsg → "broad commodities") because SLEEVE_DESC +lead-ins reference sibling tickers. + +**App integration.** The Fund Lab "Summary" page now renders +`reports/report_data.json`: at-a-glance table (24 funds: 5y, CAGR, maxDD, +R²5y, alpha5y, cluster) + one expander per fund (narrative first, then +chart, period table, drivers, reference mix, tax, peers) + an "expand +all" checkbox. Static build so the page stays fast; regenerate with +`python -m fundlab.reportdata` (~8 s) and `python -m fundlab.report` +(~11 s for the HTML). diff --git a/fundlab/narrative.py b/fundlab/narrative.py new file mode 100644 index 0000000..ed6dc52 --- /dev/null +++ b/fundlab/narrative.py @@ -0,0 +1,444 @@ +"""Per-fund narrative: English prose explaining the performance and its +drivers, built from the analysis we have on file. + +The prose is data-driven (every number comes from the computed results), +and it is required to be honest about limits: when the return drivers +are unknown or only weakly identified, the text says so explicitly. + +Paragraphs: + 1. what the fund is (name, stated strategy if on file, what its + N-PORT actually holds, which return-driver cluster it sits in) + 2. the performance in plain terms (full history, 5y, recent year, + behaviour in the defined market episodes, calendar-year pattern) + 3. what is driving it (fit quality -> whose exposures vs idiosyncratic + alpha; the economic meaning of the top loadings; weight stability) + 4. what we do NOT know (unexplained share, alpha steadiness, sample + size, holdings opacity) - stated plainly + 5. bottom line (account placement, best cluster peers, role) + +Run: .venv/bin/python -m fundlab.narrative (prints all narratives) +""" +from __future__ import annotations + +import json +from pathlib import Path + +import numpy as np + +from fundlab import report as _r +from fundlab.searchlist import BROAD_SLEEVES +from fundlab import decompose as _dc + +HERE = Path(__file__).parent + +_XC = json.loads((HERE / "xcheck_report.json").read_text()) \ + if (HERE / "xcheck_report.json").exists() else {} + + +def _pct(x, nd=1) -> str: + if x is None or (isinstance(x, float) and np.isnan(x)): + return "no data" + return f"{100 * x:+.{nd}f}%" + + +def _num(x, nd=2) -> str: + if x is None or (isinstance(x, float) and np.isnan(x)): + return "n/a" + return f"{x:.{nd}f}" + + +# short prose names for sleeves whose SLEEVE_DESC lead-in references a +# sibling ticker ("same HY exposure as VWEAX via a different fund") +_PROSE_NAME = { + "vea": "international developed markets", + "efa": "international developed markets", + "hyg": "high-yield corporate bonds", + "vweax": "high-yield corporate bonds", + "dbb": "broad commodities", + "gsg": "broad commodities", + "vblix": "crash vol (VIX futures)", + "bil": "cash (T-bills)", + "shv": "cash (T-bills)", +} + + +def _sleeve_prose(sym: str, beta: float) -> str: + nm, desc = _r.sleeve_desc(sym) + short = _PROSE_NAME.get(sym) + head = short.split(";")[0].strip() if short else \ + desc.split(";")[0].strip() + head = head[0].upper() + head[1:] if head else nm + if abs(beta) < 0.15: + return f"a small tilt to {head.lower()}" + if beta > 0: + return f"exposure to {head.lower()}" + return f"short-like exposure to {head.lower()} (it rises when that " \ + f"asset falls)" + + +def _top_sleeves(fr: dict, window: str, n: int = 3) -> list[tuple[str, float]]: + f = fr.get(window) or {} + betas = f.get("betas") or {} + top = sorted(betas.items(), key=lambda kv: -abs(kv[1]))[:n] + return [(s, b) for s, b in top if abs(b) >= 0.08] + + +def _fit(fr: dict, srow: dict) -> tuple[float | None, float | None, + float | None]: + """(r2, alpha_ann_5y, t5) preferring the reference-model numbers.""" + r2 = srow.get("r2_5y") + a5 = srow.get("alpha_ann_5y") + t5 = srow.get("alpha_t_5y") + if not isinstance(r2, (int, float)): + r2 = (fr.get("rec5") or fr.get("full") or {}).get("r2") + if not isinstance(a5, (int, float)): + a5 = (fr.get("rec5") or fr.get("full") or {}).get("alpha_ann") + if not isinstance(t5, (int, float)): + t5 = (fr.get("rec5") or fr.get("full") or {}).get("alpha_t") + return (r2 if isinstance(r2, (int, float)) else None, + a5 if isinstance(a5, (int, float)) else None, + t5 if isinstance(t5, (int, float)) else None) + + +def narrate(sym: str) -> list[str]: + sym = sym.lower() + srow = _r.search_row(sym) + fr = _r.factor_row(sym) + dr = _r.decomp_row(sym) + tax = _r.tax_row(sym) + ddr = _r.dd_row(sym) + xcr = _XC.get(sym, {}) + p = _r.price(sym) + name = (srow.get("name") or _r.FONDS.get(sym, {}).get("name") + or (fr or {}).get("name") or sym.upper()) + out: list[str] = [] + + # ------------------------------------------------------------ 1. what + who = [] + who.append(f"{sym.upper()} is the {name}.") + obj = (_r.FONDS.get(sym) or {}).get("objective") + if obj: + first = obj.split(". ")[0].strip().rstrip(".") + who.append(f"Stated strategy: {first}.") + if xcr and not xcr.get("error"): + b = [x["name"] for x in xcr.get("buckets", [])[:2]] + if b: + who.append(f"Per its latest N-PORT its actual holdings are " + f"mostly {b[0]}" + + (f" and {b[1]}" if len(b) > 1 else "") + ".") + out.append(" ".join(who)) + + if p is None: + # no return history: do NOT claim a cluster - the loading vector + # is all zeros and would park the fund in the cash cluster + out.append("There is no local return history for this share " + "class yet (new class), so performance and drivers " + "cannot be measured from returns. The N-PORT shows " + "what it holds: " + ((dr or {}).get("note") + or "see holdings")) + return out + if sym in _r.MEMBER: + _cid, clabel, cn = _r.MEMBER[sym] + who = out[0] + out[0] = (who + f" In the return-driver analysis it sits in the " + f"'{clabel}' cluster with {cn - 1} other funds.") + else: + try: + load = _r.fund_loading_row(fr) + _cid, clabel = _r.cluster_of_row(load) + out[0] = (out[0] + f" Its return-driver signature is closest " + f"to the '{clabel}' cluster.") + except Exception: + pass + + st = _r.perf_stats(p) + r2, a5, t5 = _fit(fr, srow) + tf = (fr.get("full") or {}).get("alpha_t") + a_f = (fr.get("full") or {}).get("alpha_ann") + refs = _r.ref_components(sym, fr) + r2f_ = (fr.get("rec5") or fr.get("full") or {}).get("r2") + fit = r2 if r2 is not None else r2f_ + r25 = srow.get("r2_5y") if isinstance(srow.get("r2_5y"), (int, float)) \ + else r2f_ + refs_curve = refs if (fit is not None and fit >= 0.5) else [] + mix = _r.mix_series(refs_curve) if refs_curve else _r.price("bil") + ivv = _r.price("ivv") + t5y = _r.window_ret(p, "2021-01-01", st["end"]) + t1y = _r.window_ret(p, "2025-09-01", st["end"]) + ref5y = _r.window_ret(mix, "2021-01-01", st["end"]) if mix is not None \ + else None + iv5y = _r.window_ret(ivv, "2021-01-01", st["end"]) if ivv is not None \ + else None + + # --------------------------------------------------------- 2. performance + perf = [] + tot = p.iloc[-1] / p.iloc[0] - 1 + perf.append( + f"Since {st['start'][:4]} it has compounded at " + f"{_pct(st['cagr'])} per year (a {_pct(tot, 0)} total return) " + f"with {_pct(st['vol'], 0)} annualized volatility and a maximum " + f"drawdown of {100 * abs(st['mdd']):.0f}%.") + perf.append( + f"Over the last five years it returned {_pct(t5y)} versus " + f"{_pct(ref5y)} for its reference" + + (f" and {_pct(iv5y)} for the S&P 500" if iv5y is not None else "") + + f". The last twelve months have done {_pct(t1y)}.") + yrs = _r.annual_table(p, 7) + pos = sum(1 for _y, v in yrs if v > 0) + perf.append(f"It was positive in {pos} of the last {len(yrs)} " + f"calendar years (including the partial current year).") + # episodes + rets = ddr.get("rets", {}) + if rets: + best = max(rets.items(), key=lambda kv: kv[1]) + worst = min(rets.items(), key=lambda kv: kv[1]) + perf.append( + f"Across the defined market episodes it did best in " + f"'{best[0]}' ({_pct(best[1])}) and worst in '{worst[0]}' " + f"({_pct(worst[1])}) - the peak-to-trough windows when " + f"equities fell hardest.") + _dv = ddr.get("verdict") or "" + if _dv and not _dv.startswith("CANDIDATE"): + perf.append(f"Its drawdown screen verdict: {_dv}.") + out.append(" ".join(perf)) + + # --------------------------------------------------------- 3. drivers + drv = [] + tops = _top_sleeves(fr, "rec5", 3) + if fit is not None and fit >= 0.7: + tops_txt = ", ".join(_sleeve_prose(s, b) for s, b in tops) + drv.append( + f"The fit is strong: {100 * fit:.0f}% of its excess returns " + f"over the last five years are explained by its measured " + f"exposures" + + (f" - {tops_txt}" if tops else "") + ".") + drv.append( + f"In other words, most of what this fund does is charge you " + f"for those exposures in the form of a fund; what is left - " + f"an alpha of {_pct(a5)} per year (t = {_num(t5, 1)}) - is " + + ("statistically indistinguishable from zero, i.e. the " + "strategy is not clearly adding anything on top of the " + "mix it owns." + if (t5 or 0) < 1.75 else + "small relative to the beta, but it is what separates " + "this fund from the equivalent index mix.") + ) + elif fit is not None and fit >= 0.5: + tops_txt = ", ".join(_sleeve_prose(s, b) for s, b in tops) + drv.append( + f"The measured exposures explain a good share but not all " + f"({100 * fit:.0f}% R²) of its excess returns." + + (f" The main ones: {tops_txt}." if tops else "")) + drv.append( + f"The residual is {_pct(a5)} per year (t = {_num(t5, 1)}): " + + ("a real edge on top of the mix, but a material part of " + "the performance IS the mix - judge the exposures and " + "the skill separately." + if (t5 or 0) >= 1.75 else + "not statistically distinct from noise at the five-year " + "horizon, so treat the outperformance as part beta, part " + "luck until it accumulates more history.") + ) + else: + tops_txt = ", ".join(_sleeve_prose(s, b) for s, b in tops) + drv.append( + f"The broad sleeve framework explains little of its excess " + f"returns (R² = {_num(fit) if fit is not None else 'n/a'}). " + + (f"The loadings that do show up - {tops_txt} - are minor " + "tilts, not the story." + if tops else + "No benchmark loading is even large.")) + long_ctx = "" + if isinstance(a_f, (int, float)): + long_ctx = (f" Over the full history the same estimate is " + f"{_pct(a_f)} per year (t = {_num(tf, 1)}).") + if (t5 or 0) >= 2: + drv.append( + f"The bulk of the performance is therefore " + f"idiosyncratic: it comes from the fund's own holdings " + f"and decisions, which nothing in our 34-sleeve space " + f"replicates. The estimate of that idiosyncratic return " + f"is {_pct(a5)} per year over the last five years " + f"(t = {_num(t5, 1)}) - statistically significant, so " + f"the outperformance is real; what the returns alone " + f"cannot tell us is its source.{long_ctx}") + elif (a5 or 0) >= 0: + drv.append( + f"The bulk of the performance is therefore " + f"idiosyncratic: it comes from the fund's own holdings " + f"and decisions, which nothing in our 34-sleeve space " + f"replicates. The estimate of that idiosyncratic return " + f"is {_pct(a5)} per year over the last five years " + f"(t = {_num(t5, 1)}) - at that t-stat it is not yet " + f"clearly different from a lucky streak.{long_ctx}") + else: + drv.append( + f"Most of the performance is therefore idiosyncratic " + f"rather than benchmark-like. The honest five-year read " + f"of its excess return is {_pct(a5)} per year " + f"(t = {_num(t5, 1)}): over the recent window this fund " + f"is NOT measurably adding to its fitted reference.{long_ctx}") + if (a5 or 0) < 0 and isinstance(tf, (int, float)) and tf >= 1.25: + drv.append( + f"The longer record is better: over the full history the " + f"alpha is {_pct(a_f)} per year (t = {_num(tf, 1)}), " + f"which IS significant - so the five-year estimate " + f"looks like a flat patch in a longer positive trend, " + f"not evidence the edge has gone.") + if xcr and not xcr.get("error"): + b = [x["name"] for x in xcr.get("buckets", [])[:2]] + if b: + drv.append( + f"The N-PORT is consistent with that: the alpha is " + f"plausibly coming from {b[0]}" + + (f" and {b[1]}" if len(b) > 1 else "") + + " - asset classes our sleeve set either does not " + "model directly or models too coarsely. That is an " + "interpretation from the holdings, not something " + "the returns prove.") + if obj and (a5 or 0) > 0: + _o = obj.split(". ")[0].rstrip(".") + if len(_o) > 110: + _o = _o[:110].rsplit(" ", 1)[0] + "..." + drv.append( + f"The stated strategy ('{_o}') is a plausible mechanism " + f"as well, but that is a hypothesis - the returns only " + f"tell us the outperformance exists, not why.") + # stability (curated funds) + roll = (dr or {}).get("rolling") or {} + if "verdict" in (dr or {}) and roll.get("max_drift") is not None: + md = roll["max_drift"] + drv.append( + f"The curated decomposition's read: {dr['verdict']}. " + f"Weight stability: the one-year rolling betas move by at " + f"most {md:.2f} (relative to the full-sample weights) - " + + ("so the weights are roughly stable over time." + if md < 0.5 else + "so treat this as a tactically active fund, not a " + "static mix.")) + out.append(" ".join(drv)) + + # ------------------------------------------------- 4. what we do NOT know + unk = [] + if fit is not None: + if (t5 or 0) >= 2: + unk.append( + f"To be explicit about the limits: " + f"{100 * (1 - fit):.0f}% of this fund's excess return " + f"is NOT attributed to any measured exposure by our " + f"analysis - we know the outperformance is real, we do " + f"not know from returns what produces it; the source " + f"has to be read from the filings before sizing up.") + else: + unk.append( + f"To be explicit about the limits: the excess return is " + f"not measurable against noise at the current sample " + f"size, so there is not even a reliable alpha to " + f"attribute; {100 * (1 - fit):.0f}% of the return is " + f"outside our sleeve model either way.") + pf = srow.get("alpha_pos_frac") + if isinstance(pf, (int, float)) and pf < 0.60: + unk.append( + f"The alpha is also not steady: in {100 * (1 - pf):.0f}% of " + f"rolling six-month windows the fund trailed its reference, " + f"so the five-year number is carrying periods of " + f"underperformance.") + if (t5 or 0) >= 2 and isinstance(tf, (int, float)) and tf < 1.25: + unk.append( + "The significance is recent - over the full history the " + "alpha is not significant, so this is a provisional read on " + "a ~5-year sample.") + if st["years"] < 3: + unk.append( + f"The whole sample is short ({st['years']:.1f} years), so " + f"every number above has a wide error bar.") + if xcr.get("note"): + unk.append(xcr["note"].rstrip(".") + ".") + note = (dr or {}).get("note") + if note and "N-PORT" in note: + # holdings cross-check sentence (shortlist) + for sentence in note.split(". "): + if "N-PORT" in sentence or "holdings" in sentence.lower(): + unk.append(sentence.strip().rstrip(".") + + " - the holdings check is the ground truth " + "here, and the return model is only a " + "description of it.") + break + if not unk: + unk.append("No major caveats beyond the usual: the past fit may " + "not persist, and the sleeves are models of the " + "fund, not the fund itself.") + out.append(" ".join(unk)) + + # --------------------------------------------------------- 5. bottom line + bot = [] + loc = tax.get("location") + if loc: + basis = "N-PORT holdings" if tax.get("basis") == "N-PORT" \ + else "the sleeve model" + if "MIXED" in loc: + bot.append( + f"Account placement: not settled by the model - the " + f"sleeve model calls it MIXED, so the last 1099-DIV is " + f"the arbiter before choosing taxable vs IRA.") + else: + bot.append( + f"Account placement: " + f"{'taxable' if 'TAXABLE' in loc else 'IRA'} " + f"({basis} says {loc}).") + # peers + if sym in _r.MEMBER: + cid = _r.MEMBER[sym][0] + members = _r.KMEANS["clusters"][cid]["syms"] + peers = [] + for s in members: + if s == sym: + continue + v = _r.search_row(s) + if isinstance(v.get("alpha_t_5y"), (int, float)) and \ + v["alpha_t_5y"] > 0: + pp = _r.price(s) + if pp is None: + continue + peers.append((v["alpha_t_5y"], v.get("alpha_ann_5y"), s)) + peers.sort(reverse=True) + if peers: + b_t, b_a, b_s = peers[0] + bp = _r.price(b_s) + bt5 = _r.window_ret(bp, "2021-01-01", + _r.perf_stats(bp)["end"]) + bot.append( + f"Within its cluster the strongest alternative is " + f"{b_s.upper()} (5y {_pct(bt5)}, alpha t = {b_t:+.1f}); " + f"the choice between them should turn on the conviction " + f"in the strategy, the tax fit and the price paid, not " + f"on the statistics - they are the same kind of " + f"position.") + cp = srow.get("corr_portfolio") + if isinstance(cp, (int, float)): + bot.append( + f"Its correlation with your current portfolio is " + f"{cp:+.2f} - " + + ("a genuine diversifier" if cp < 0.3 + else "not very diversified vs what you already own".rstrip(".")) + + ".") + out.append(" ".join(bot)) + return out + + +def main() -> None: + from fundlab.nport import FUND_TOKENS + from fundlab.decompose import ALIAS + cand = [v["sym"] for v in _r.SEARCH.values() + if isinstance(v, dict) + and str(v.get("verdict", "")).startswith("CANDIDATE")] + short = [s for s in FUND_TOKENS if s not in ALIAS] + for s in cand + short: + print(f"=== {s.upper()} ===") + for para in narrate(s): + print(para) + print() + + +if __name__ == "__main__": + main() diff --git a/fundlab/report.py b/fundlab/report.py index ebe2a77..5d303f9 100644 --- a/fundlab/report.py +++ b/fundlab/report.py @@ -308,18 +308,16 @@ def mix_series(refs: list[dict]) -> pd.Series | None: if b is None: return None return 100 * b / b.iloc[0] - cols = [] + idx = None for c in refs: p = price(c["sym"]) if p is None: continue - cols.append((c["beta"], p.pct_change())) - if not cols: - return None - idx = None - for _b, r in cols: + r = c["beta"] * p.pct_change() idx = r if idx is None else idx.combine(r, lambda x, y: x + y, fill_value=0.0) + if idx is None: + return None idx = idx.fillna(0.0) path = (1 + idx).cumprod() return 100 * path / path.iloc[0] @@ -665,6 +663,11 @@ def strategy_block(sym: str) -> str: return out +def narrative_block(sym: str) -> str: + from fundlab.narrative import narrate + return "".join(f"

{html.escape(par)}

" for par in narrate(sym)) + + # ------------------------------------------------------------------ build def fund_section(sym: str, title: str, subtitle: str) -> str: fr = factor_row(sym) @@ -688,6 +691,8 @@ def fund_section(sym: str, title: str, subtitle: str) -> str:

{html.escape(title)} {html.escape(subtitle)}

{strategy_block(sym)} +

Discussion

+{narrative_block(sym)} {equity_figure(sym, name, refs_curve)}

Performance

{perf_table(price(sym), refs_curve) if price(sym) is not None else '

no price data

'} diff --git a/fundlab/reportdata.py b/fundlab/reportdata.py new file mode 100644 index 0000000..49cb284 --- /dev/null +++ b/fundlab/reportdata.py @@ -0,0 +1,345 @@ +"""Build reports/report_data.json: everything the web app (and the HTML +report) need for the per-fund report pages, in one structured file. + +One entry per fund (11 alpha candidates + 13 shortlist funds): + meta name, verdict, group, order + narrative list of prose paragraphs (fundlab.narrative) + chart {dates, fund, ref, ivv} (downsampled price paths, rebased) + perf rows [period, fund, reference, ivv, fund-ref] (strings) + drivers markdown lines (reference model, signature, verdicts) + reference {rows [[loading, name, exposure]], weak, netcash, text} + tax plain text + peers {cluster, rows, proscons} + +Run: .venv/bin/python -m fundlab.reportdata +""" +from __future__ import annotations + +import json +import time +from pathlib import Path + +import numpy as np +import pandas as pd + +from fundlab import report as _r +from fundlab import decompose +from fundlab.narrative import narrate + +HERE = Path(__file__).parent +OUT = HERE.parent / "reports" / "report_data.json" + +MAX_PTS = 1500 + + +def _down(s: pd.Series) -> tuple[list, list]: + if s is None: + return [], [] + if len(s) > MAX_PTS: + s = s.iloc[:: (len(s) // MAX_PTS + 1)] + return [str(d.date()) for d in s.index], [float(v) for v in s] + + +def _chart(sym: str, refs_curve: list[dict]) -> dict: + p = _r.price(sym) + fd, fv = _down(100 * p / p.iloc[0] if p is not None else None) + if refs_curve: + mix = _r.mix_series(refs_curve) + else: + b = _r.price("bil") + mix = 100 * b / b.iloc[0] if b is not None else None + md, mv = _down(mix) + iv = _r.price("ivv") + id_, ivv = _down(100 * iv / iv.iloc[0] if iv is not None else None) + return {"dates": fd, "fund": fv, "ref_dates": md, "ref": mv, + "ivv_dates": id_, "ivv": ivv} + + +def _perf_rows(sym: str, refs_curve: list[dict]) -> list[list[str]]: + p = _r.price(sym) + if p is None: + return [] + mix = (_r.mix_series(refs_curve) if refs_curve + else _r.price("bil")) + iv = _r.price("ivv") + st = _r.perf_stats(p) + rows = [] + + def add(label, a, b): + fr_ = _r.window_ret(p, a, b) + mr = _r.window_ret(mix, a, b) + ir = _r.window_ret(iv, a, b) + gap = (fr_ - mr) if (fr_ is not None and mr is not None) else None + rows.append([label, _r.fmt_pct(fr_), _r.fmt_pct(mr), + _r.fmt_pct(ir), _r.fmt_pct(gap)]) + + add("Full history", st["start"], st["end"]) + add("Last 5y", "2021-01-01", st["end"]) + add("Last 1y", "2025-09-01", st["end"]) + for lab, a, b in _r.EPISODES: + add(lab, a, b) + yr = p.resample("YE").last().dropna() + years = [str(yr.index[i].year) for i in range(1, len(yr))][-6:] + for i, y in enumerate(years): + a = f"{y}-01-01" + b = f"{int(y) + 1}-01-01" if i < len(years) - 1 else st["end"] + add(y, a, b) + return rows + + +def _drivers(sym: str, fr: dict, srow: dict, dr: dict) -> list[str]: + out = [] + a5 = srow.get("alpha_ann_5y") + t5 = srow.get("alpha_t_5y") + r25 = srow.get("r2_5y") + if isinstance(r25, (int, float)) or isinstance(a5, (int, float)): + out.append( + f"**Reference model, last 5 years:** R² = " + f"{_r.fmt_r2(r25 if isinstance(r25, (int, float)) else None)}, " + f"alpha = " + f"{_r.fmt_pct(a5) if isinstance(a5, (int, float)) else '—'}" + f"{' (t = ' + format(t5, '+.1f') + ')' if isinstance(t5, (int, float)) else ''}") + a_f = (fr.get("full") or {}).get("alpha_ann") + t_f = (fr.get("full") or {}).get("alpha_t") + r2f = srow.get("r2_full") + if isinstance(r2f, (int, float)) or isinstance(a_f, (int, float)): + out.append( + f"**Reference model, full history:** R² = " + f"{_r.fmt_r2(r2f if isinstance(r2f, (int, float)) else None)}, " + f"alpha = " + f"{_r.fmt_pct(a_f) if isinstance(a_f, (int, float)) else '—'}" + f"{' (t = ' + format(t_f, '+.1f') + ')' if isinstance(t_f, (int, float)) else ''}.") + f = fr.get("rec5") or {} + betas = f.get("betas") or {} + top = sorted(betas.items(), key=lambda kv: -abs(kv[1]))[:6] + if top: + out.append("**Return-driver signature (34 sleeves):** " + + ", ".join(f"`{s}` {b:+.2f}" for s, b in top) + + f" - net cash {1 - sum(betas.values()):+.2f}.") + if dr and "verdict" in dr: + out.append(f"**Decomposition verdict:** {dr['verdict']}") + roll = dr.get("rolling") or {} + if roll.get("max_drift") is not None: + out.append(f"Weight stability: max 1y β-drift = " + f"{roll['max_drift']:.2f} (0 = perfectly stable, " + f">1 = unstable).") + if dr.get("note"): + out.append(f"*{dr['note']}*") + if srow.get("verdict"): + out.append(f"**Screen verdict:** {srow['verdict']}") + return out + + +def _reference(sym: str, fr: dict, refs: list[dict], + refs_curve: list[dict]) -> dict: + rows = [] + for c in refs: + nm, desc = _r.sleeve_desc(c["sym"]) + rows.append([f"{c['sym'].upper()} {c['beta']:+.2f}", nm, desc]) + sb = sum(c["beta"] for c in refs) + cash = 1 - sb + if cash > 0.05: + net = f"loadings sum to {sb:.2f} → ~{cash:.0%} net cash" + elif cash < -0.05: + net = f"loadings sum to {sb:.2f} → ~{-cash:.0%} net levered" + else: + net = f"loadings sum to {sb:.2f} → fully invested" + weak = bool(refs) and not refs_curve + text = ("**Weak fit - read with care.** These loadings are each " + "individually significant but collectively explain little; " + "the performance table anchors to cash, not this mix." + if weak else + "The reference is the fund's own fitted sleeve mix; " + "'alpha' everywhere means outperformance vs this mix, in " + "excess of the T-bill rate.") + return {"rows": rows, "weak": weak, "netcash": net, "text": text} + + +def _tax(sym: str) -> str: + t = _r.tax_row(sym) + if not t: + return "No tax classification on file." + loc = t.get("location", "?") + basis = t.get("basis", "?") + conf = "from actual N-PORT holdings (high confidence)" \ + if basis == "N-PORT" else "from the return-sleeve mix (model)" + s = (f"Character score {t.get('score', '?')} ({conf}). ") + if "MIXED" in loc: + s += "Model says MIXED - the last 1099-DIV is the arbiter." + elif "TAXABLE" in loc: + if "munis" in loc: + s += ("Keep in the **taxable** account " + "(munis - tax-exempt interest is wasted in an IRA).") + elif "defers" in loc: + s += ("Keep in the **taxable** account " + "(income mostly defers to the LTCG/ROC rate).") + else: + s += "Keep in the **taxable** account." + else: + s += f"Recommended account: **{loc}**." + if t.get("notes"): + s += f" _{t['notes']}_" + return s + + +def _peers(sym: str, fr: dict) -> dict: + if sym in _r.MEMBER: + cid, clabel, cn = _r.MEMBER[sym] + members = _r.KMEANS["clusters"][cid]["syms"] + else: + try: + cid, clabel = _r.cluster_of_row(_r.fund_loading_row(fr)) + except Exception: + return {} + cn = _r.KMEANS["clusters"].get(cid, {}).get("n", 0) + members = _r.KMEANS["clusters"].get(cid, {}).get("syms", []) + peers = [] + for s in members: + if s == sym: + continue + v = _r.search_row(s) + if isinstance(v.get("alpha_t_5y"), (int, float)): + peers.append((v["alpha_t_5y"], s)) + peers.sort(key=lambda x: (x[0] > 0, x[0]), reverse=True) + pos = [p for p in peers if p[0] > 0] + peers = [s for _t, s in (pos + [p for p in peers if p[0] <= 0])[:4]] + + def statline(s: str) -> list[str]: + p = _r.price(s) + if p is None: + return ["—", "—", "—", "—", "—", "n/a"] + st = _r.perf_stats(p) + v = _r.search_row(s) + t5 = _r.window_ret(p, "2021-01-01", st["end"]) + a5 = v.get("alpha_ann_5y") + tt = v.get("alpha_t_5y") + tax = (_r.tax_row(s) or {}).get("location") or "n/a" + return [_r.fmt_pct(t5), _r.fmt_pct(st["cagr"]), + _r.fmt_pct(st["mdd"]), + _r.fmt_r2(v.get("r2_5y")), + (f"{_r.fmt_pct(a5)} (t={tt:+.1f})" + if isinstance(a5, (int, float)) else "—"), tax] + + rows = [[f"{sym.upper()} (this fund)", *statline(sym)]] + for s in peers: + rows.append([s.upper(), *statline(s)]) + + # pros / cons + adv, dis = [], [] + p = _r.price(sym) + if p is not None and peers: + st = _r.perf_stats(p) + t5 = _r.window_ret(p, "2021-01-01", st["end"]) + vals = {} + for s in peers: + pp = _r.price(s) + if pp is None: + continue + ss = _r.perf_stats(pp) + vals[s] = (_r.window_ret(pp, "2021-01-01", ss["end"]), + ss["mdd"], ss["vol"]) + if vals: + best_t5 = max(v[0] for v in vals.values() if v[0] is not None) + best_dd = max(v[1] for v in vals.values()) + low_vol = min(v[2] for v in vals.values()) + if t5 is not None and t5 >= best_t5 - 0.02: + adv.append("5y return at the top of the cluster") + elif t5 is not None and t5 < best_t5 - 0.10: + dis.append(f"5y return trails the best peer by " + f"{100 * (best_t5 - t5):.0f}pp") + if st["mdd"] < best_dd - 0.05: + adv.append("sharpest drawdown in the cluster") + elif st["mdd"] > best_dd + 0.05: + dis.append(f"deeper drawdown than the calmest peer " + f"({_r.fmt_pct(st['mdd'])} vs " + f"{_r.fmt_pct(best_dd)})") + if st["vol"] < low_vol - 0.02: + adv.append("lowest volatility in the cluster") + elif st["vol"] > low_vol + 0.05: + dis.append("meaningfully more volatile than the " + "calmest peer") + if adv: + pc = "**Advantages vs peers:** " + "; ".join(adv) + "." + elif dis: + pc = "**Disadvantages vs peers:** " + "; ".join(dis) + "." + else: + pc = ("Middle of the cluster - no decisive edge on return, " + "drawdown or volatility; the choice comes down to alpha " + "quality (t), tax fit and conviction in the strategy.") + if dis and adv: + pc += " **Disadvantages:** " + "; ".join(dis) + "." + return {"cluster": clabel, "n": cn, + "rows": rows, "proscons": pc} + + +def build_fund(sym: str, group: str, order: int) -> dict: + t0 = time.time() + fr = _r.factor_row(sym) + dr = _r.decomp_row(sym) + srow = _r.search_row(sym) + name = (srow.get("name") or _r.FONDS.get(sym, {}).get("name") + or (fr or {}).get("name") or sym.upper()) + refs = _r.ref_components(sym, fr) + r2f_ = (fr.get("rec5") or fr.get("full") or {}).get("r2") + r25 = srow.get("r2_5y") if isinstance(srow.get("r2_5y"), (int, float)) \ + else None + fit = r25 if r25 is not None else r2f_ + refs_curve = refs if (fit is not None and fit >= 0.5) else [] + p = _r.price(sym) + st_ = _r.perf_stats(p) if p is not None else {} + stats = {"t5y": _r.fmt_pct(_r.window_ret(p, "2021-01-01", + st_["end"])) if p is not None + else "—", + "cagr": _r.fmt_pct(st_.get("cagr")), + "mdd": _r.fmt_pct(st_.get("mdd")), + "r2_5y": _r.fmt_r2(srow.get("r2_5y")), + "alpha_5y": (f"{_r.fmt_pct(srow['alpha_ann_5y'])} " + f"(t={srow['alpha_t_5y']:+.1f})" + if isinstance(srow.get("alpha_ann_5y"), + (int, float)) else "—")} + d = { + "meta": {"sym": sym, "name": name, "group": group, "order": order, + "verdict": srow.get("verdict", ""), + "objective": (_r.FONDS.get(sym) or {}).get("objective", + "")}, + "stats": stats, + "narrative": narrate(sym), + "chart": _chart(sym, refs_curve), + "perf": _perf_rows(sym, refs_curve), + "drivers": _drivers(sym, fr, srow, dr), + "reference": _reference(sym, fr, refs, refs_curve), + "tax": _tax(sym), + "peers": _peers(sym, fr), + } + print(f" {sym:8s} {time.time() - t0:5.1f}s", flush=True) + return d + + +def main() -> None: + from fundlab.decompose import ALIAS + from fundlab.nport import FUND_TOKENS + t0 = time.time() + cand = [v["sym"] for v in _r.SEARCH.values() + if isinstance(v, dict) + and str(v.get("verdict", "")).startswith("CANDIDATE")] + cand.sort(key=lambda s: -(_r.SEARCH[s]["alpha_t_5y"] + if isinstance(_r.SEARCH[s].get("alpha_t_5y"), + (int, float)) else -9)) + short = [s for s in FUND_TOKENS if s not in ALIAS] + data = {"generated": time.strftime("%Y-%m-%d %H:%M"), + "funds": {}} + i = 0 + for s in cand: + i += 1 + data["funds"][s] = build_fund(s, "candidate", i) + for s in short: + i += 1 + data["funds"][s] = build_fund(s, "shortlist", i) + OUT.parent.mkdir(exist_ok=True) + OUT.write_text(json.dumps(data)) + print(f"wrote {OUT} in {time.time() - t0:.0f}s " + f"({OUT.stat().st_size / 1e6:.1f} MB, {len(data['funds'])} funds)") + + +if __name__ == "__main__": + main() diff --git a/reports/fund_report.html b/reports/fund_report.html index 0d649b8..9315454 100644 --- a/reports/fund_report.html +++ b/reports/fund_report.html @@ -3911,7 +3911,7 @@ return Plotly;