Factor screen (v2, 35 drivers) + return-driver clusters

factors.py: full OLS of all 2,384 funds on a 35-driver basis
(overinclusive, no portfolio-corr screening - corr is a replacement
signal, not a rejection). v1's 21 + lqd/hyg/prefs/emb/tip/shy/vtv/
8 sectors/CTA/commodities. Basis fixes: drop vea/vug (dupes of
efa/qqq), drop finux (TERMINATED 2017 - silently zeroed the
complete-case mask; v1 forward selection never hit this), drop bil
(shv/bil near-null -> offsetting shv+64/bil-54 noise fits),
residualize vblix on ivv+tlt (pure vol axis), ridge 0.02.

cluster.py: hierarchical tree saved but fixed-k cuts degenerate
(most funds are blends -> one 2250-fund blob); k-means++ (deterministic)
is the useful grouping, re-run live in-app for any k.

app: Return-driver clusters expander (k slider 10-60, summary table,
member table sorted by alpha-t). Findings at k=30 in RESEARCH.md.

app.py

@@ -679,6 +679,105 @@ with tab_fundlab:
                 "alpha), t5y ≥ 2, t-full ≥ 1.25, ≥ 45% positive windows, "
                 "portfolio correlation < 0.3.")
 
+    # --- return-driver clusters: k-means on 35-sleeve loading vectors ---
+    with st.expander(
+            "Return-driver clusters — funds grouped by what drives "
+            "their returns"):
+        _ck = st.slider("clusters", 10, 60, 30, step=2, key="fl_clus_k")
+        try:
+            import numpy as _np
+            from fundlab import cluster as _cl
+            from fundlab import factors as _fac
+            _syms, _V = _cl.loading_matrix()
+            _fr = json.loads((_dc.RESULTS.parent /
+                              "factor_results.json").read_text())
+            _lab = _cl.kmeans(_V, _ck)
+            _sumrows = []
+            for c in range(_ck):
+                idx = [i for i, l in enumerate(_lab) if l == c]
+                if not idx:
+                    continue
+                med = _np.median(_V[idx], 0)
+                top = _np.argsort(-_np.abs(med))[:4]
+                prof = {_fac.DRIVERS[t]: float(med[t]) for t in top
+                        if abs(med[t]) >= 0.08}
+                lab = _cl.label(prof)
+                t5s = [_fr[_syms[i]].get("alpha_t_5y") for i in idx]
+                t5s = [t for t in t5s if isinstance(t, (int, float))]
+                n_sig = sum(1 for t in t5s if t >= 2)
+                best = max(idx, key=lambda i: abs(
+                    _fr[_syms[i]].get("alpha_t_5y") or 0))
+                _sumrows.append({
+                    "cluster": lab, "n": len(idx),
+                    "t5≥2": n_sig,
+                    "top by |t5|": (f"{_syms[best].upper()} "
+                                    f"({_fr[_syms[best]].get('name','')[:40]})")})
+            # remember cluster id with each summary row for the selectbox
+            for _i, _r in enumerate(_sumrows):
+                _r["_id"] = _i
+            _sumrows.sort(key=lambda r: -r["n"])
+            st.dataframe(pd.DataFrame(
+                {k: r[k] for k in ("cluster", "n", "t5≥2", "top by |t5|")}
+                for r in _sumrows), width="stretch")
+            _cpick = st.selectbox(
+                "expand a cluster",
+                [f"{r['cluster']}  (n={r['n']}, t5≥2: {r['t5≥2']})"
+                 for r in _sumrows], key="fl_clus_pick")
+            _wanted = _cpick.split("  (")[0]
+            _crows = [r for r in _sumrows if r["cluster"] == _wanted]
+            _cn = (_crows[0]["n"] if _crows else None)
+            _crows = [r for r in _sumrows
+                      if r["cluster"] == _wanted and r["n"] == _cn]
+            _target = _crows[0]["cluster"] if _crows else None
+            _members = []
+            for c in range(_ck):
+                idx = [i for i, l in enumerate(_lab) if l == c]
+                if not idx:
+                    continue
+                med = _np.median(_V[idx], 0)
+                top = _np.argsort(-_np.abs(med))[:4]
+                prof = {_fac.DRIVERS[t]: float(med[t]) for t in top
+                        if abs(med[t]) >= 0.08}
+                if _cl.label(prof) == _target:
+                    _members = [(i, idx[0]) for i in idx]
+                    break
+            _rows = []
+            for i, _b in _members:
+                v = _fr.get(_syms[i], {})
+                _a5 = v.get("alpha_ann_5y")
+                _t5 = (v.get("alpha_t_5y")
+                       if isinstance(v.get("alpha_t_5y"), (int, float))
+                       else -99)
+                _rows.append({
+                    "fund": f"{_syms[i].upper()} — {v.get('name','')[:48]}",
+                    "R²": (f"{v['full']['r2']:.2f}"
+                           if isinstance(v.get("full"), dict) else "—"),
+                    "alpha 5y": (f"{_a5*100:+.1f}% (t={_t5:+.1f})"
+                                 if isinstance(_a5, (int, float)) else "—"),
+                    "corr port": (f"{v['corr_portfolio']:.2f}"
+                                  if isinstance(
+                                      v.get("corr_portfolio"),
+                                      (int, float)) else "—"),
+                    "verdict": (v.get("verdict") or "")[:40],
+                    "src": "local" if v.get("local") else "new",
+                    "_t": _t5,
+                })
+            _rows.sort(key=lambda r: -r["_t"])
+            for r in _rows:
+                r.pop("_t")
+            st.dataframe(pd.DataFrame(_rows), width="stretch")
+            st.caption(
+                "Each fund's daily returns are regressed on 35 sleeve axes "
+                "(equity styles/sizes/intl/EM, the duration ladder, IG/HY/"
+                "muni/MBS/preferred/EM-debt credit, sectors, gold/oil/"
+                "commodities, CTA); the 35-d loading vector is the fund's "
+                "'return-driver signature' and k-means groups similar "
+                "signatures. 'corr port' is shown for reference only - "
+                "high-corr funds are REPLACEMENTS for current holdings, "
+                "not rejections.")
+        except Exception as e:  # noqa: BLE001
+            st.warning(f"cluster view unavailable: {e}")
+
     _f = _FUNDS.get(_fl_pick, {})
     _man = _MAN.get(_fl_pick, {})
     st.subheader(f"{_f.get('name', _fl_pick)}  ·  {_fl_pick.upper()}")

fundlab/RESEARCH.md

@@ -120,16 +120,62 @@ not a gate.
   RECENT (last 5y), not full-history. Good thing the screen keeps both
   stats.
 
-### Next iterations (not done)
+### Next iterations
-1. Add missing sleeves to the model (short-duration/floating-rate,
+1. [x] **Add missing factors + cluster by return driver** (this
-   muni, preferreds, EM debt, merger-arb proxy) to split true alpha
+      iteration - fundlab/factors.py + fundlab/cluster.py).
-   from missing-factor exposure; re-screen (0.3s/fund, ~12 min).
 2. N-PORT holdings cross-check on the top ~15 candidates (the v1
    16-fund pipeline: edgar NPORT fetch + category buckets) to confirm
    what the alpha funds actually hold.
 3. CEF universe (485/N-2 filers) - separate pass; CEFs have
    premium/discount dynamics the NAV screen can't see.
 
+### Factor screen + clusters (iteration 2, 2026-08-27)
+Design (per user: overinclusive, NO portfolio-corr screening - high
+corr funds are REPLACEMENTS; group funds by return driver):
+- fundlab/factors.py: full OLS of every fund's daily total returns on
+  a 35-driver basis (was 21): +lqd (IG corp), +hyg, +pff (prefs),
+  +emb (EM debt), +tip, +shy, +vtv (value), +8 sectors (xl*), +dbmf
+  (CTA), +dbb (commodities). Dropped from the regression basis:
+  vea/vug (>0.95 dupes of efa/qqq), **finux (TERMINATED 2017 - it
+  silently zeroed the complete-case mask for every fund overlapping its
+  lifetime; the v1 forward selector never hit this because sleeves
+  were optional, full OLS needs a common sample)**, bil (its shv/bil
+  pair let OLS express tiny net cash exposure as huge offsetting
+  loadings shv+64/bil-54 - noise fits with R2~0.01, |t|<2).
+  vblix residualized on [ivv, tlt] (tlt~vblix 0.963) -> pure vol axis.
+  Small ridge (0.02, slopes only) stabilizes loadings; winsorize at
+  +/-4 for clustering. Output: factor_results.json (per-fund
+  full+5y loadings, R2, alpha, t + carried screen metadata).
+- fundlab/cluster.py: (a) hierarchical avg-linkage tree (saved,
+  4-8s) - but **fixed-k tree cuts are degenerate here**: most funds
+  are multi-sleeve blends, so the tree lumps ~2250 of them into one
+  blob and only peels pure single-sleeve clusters. (b) **k-means
+  (k-means++ init, numpy, deterministic) is the useful grouping** -
+  the app re-runs it live for any k (2384x35 is milliseconds).
+  Labels = top |median loading| per cluster (>=0.30 gate, else
+  "no dominant driver (balanced/idio)").
+- App Fund Lab: "Return-driver clusters" expander - k slider
+  (10-60, default 30), summary table (label, n, t5>=2 count, top
+  fund by |t5|), selectbox -> member table sorted by alpha t desc
+  (corr port as info column only).
+
+Findings (k=30): the universe decomposes into readable driver groups:
+US large blend (232), short T 1-3y (194), cash/ultra-short (185),
+US small (128+70+47), US value (115), intl developed (109), growth-
+tilted blend (79+27, loadings ivv+1.07/vtv-0.68), healthcare (15),
+plus a 686-fund "no dominant driver (balanced/idio)" blob - the
+largest single group, where the real candidate hunting happens (84
+funds with t5>=2). The screen's 250 candidates now have a driver
+address: e.g. hmezx (merger arb) and egrix (macro) sit in the
+balanced/idio blob with no sleeve to blame; rctix/dflex sit in
+short-duration; aguax/femdx in the EM group.
+
+Caveats: loadings are FULL-history (5y fallback) and ridge-shrunk -
+they describe the driver, not a tradable weight; the balanced/idio
+blob is big by construction (most funds ARE mixes); Japan/China/India
+single-country funds have no country sleeve and read as weak/idio -
+a known coverage gap.
+
 Infra lesson: /tmp gets cleaned mid-run - keep logs + caches in the
 project (fundlab/overnight.log, fundlab/universe_cache/), and use
 `setsid nohup ... < /dev/null &` so a closed shell can't kill the job.

fundlab/cluster.py

@@ -0,0 +1,220 @@
+"""Cluster funds by RETURN DRIVER.
+
+Input: fundlab/factor_results.json (per-fund loading vectors from the
+39-sleeve full OLS). Distance: squared Euclidean on the full-window
+loading vector (missing sleeves -> 0). Agglomerative, AVERAGE linkage
+(Lance-Armstrong update), pure numpy (no scipy in the venv).
+
+Output: fundlab/cluster_tree.json
+  syms   - fund symbols (rows of the loading matrix)
+  merges - [(a, b, dist), ...] in merge order, a/b ORIGINAL fund
+           indices, b absorbed into a
+  n      - number of funds
+
+Cutting the tree at any k is O(n) (parent array over the first n-k
+merges), so the app can offer a k slider with instant re-grouping.
+Label a cluster by its top |median loading| sleeves; all-small ->
+"idiosyncratic / alpha".
+
+Run: python -m fundlab.cluster     (builds the tree, ~1-2 min)
+"""
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+
+import numpy as np
+
+HERE = Path(__file__).parent
+RESULTS = HERE / "factor_results.json"
+TREE = HERE / "cluster_tree.json"
+KMEANS = HERE / "cluster_kmeans.json"
+from fundlab.factors import DRIVERS  # noqa: E402
+
+
+def log(msg: str) -> None:
+    print(time.strftime("%H:%M:%S"), msg, flush=True)
+
+
+def loading_matrix() -> tuple[list[str], np.ndarray]:
+    d = json.loads(RESULTS.read_text())
+    syms, rows = [], []
+    for sym, v in d.items():
+        # full window preferred; funds without 250 complete-case rows
+        # there fall back to the 5y window
+        f = v.get("full")
+        if not (isinstance(f, dict) and "betas" in f):
+            f = v.get("rec5")
+        if not (isinstance(f, dict) and "betas" in f):
+            continue
+        row = np.array([f["betas"].get(s, 0.0) or 0.0 for s in DRIVERS],
+                       dtype=float)
+        # winsorize at +/-4: everything beyond that is an OLS noise fit
+        # (R2 ~ 0.01, |t| < 2 - e.g. fyhtx shv +63.8), never a real
+        # exposure (no open-end fund runs 400% of a sleeve)
+        row = np.clip(row, -4.0, 4.0)
+        syms.append(sym)
+        rows.append(row)
+    return syms, np.vstack(rows)
+
+
+def hclust(V: np.ndarray) -> list[list]:
+    """Average-linkage agglomerative. Returns [(a, b, dist), ...] with
+    ORIGINAL row indices, b absorbed into a, merge order."""
+    n = len(V)
+    ss = np.sum(V * V, axis=1)
+    D = ss[:, None] + ss[None, :] - 2 * (V @ V.T)
+    np.clip(D, 0, None)
+    np.fill_diagonal(D, np.inf)
+    order = np.arange(n)
+    size = np.ones(n)
+    merges: list[list] = []
+    for step in range(n - 1):
+        iu, ju = np.unravel_index(np.argmin(D), D.shape)
+        i, j = int(iu), int(ju)
+        if i > j:
+            i, j = j, i
+        a, b = int(order[i]), int(order[j])
+        merges.append([a, b, float(D[i, j])])
+        # Lance-Armstrong: new distance from i to every survivor k
+        size_ij = size[i] + size[j]
+        D[i, :] = (size[i] * D[i, :] + size[j] * D[j, :]) / size_ij
+        D[:, i] = D[i, :]
+        size[i] = size_ij
+        D = np.delete(D, j, axis=0)
+        D = np.delete(D, j, axis=1)
+        order = np.delete(order, j)
+        np.fill_diagonal(D, np.inf)
+        if step % 500 == 499:
+            log(f"  hclust: {step + 1}/{n - 1} merges")
+    return merges
+
+
+def cut(merges: list, n: int, k: int) -> dict[int, int]:
+    """Cluster id (0..k-1 by size order) for each original index."""
+    parent = list(range(n))
+
+    def find(x: int) -> int:
+        while parent[x] != x:
+            parent[x] = parent[parent[x]]
+            x = parent[x]
+        return x
+    for a, _b, _d in merges[: n - k]:
+        a, b = int(a), int(_b)
+        parent[find(b)] = find(a)
+    roots = [find(i) for i in range(n)]
+    # map roots to dense ids, largest cluster -> id 0
+    import collections
+    cnt = collections.Counter(roots)
+    rank = {r: i for i, r in enumerate(sorted(cnt, key=cnt.get, reverse=True))}
+    return {i: rank[r] for i, r in enumerate(roots)}
+
+
+def cluster_means(syms: list[str], V: np.ndarray,
+                  assign: dict[int, int]) -> list[dict]:
+    """Per-cluster median loading profile (for labels)."""
+    k = max(assign.values()) + 1
+    out = []
+    for c in range(k):
+        idx = [i for i, s in assign.items() if s == c]
+        med = np.median(V[idx], axis=0)
+        top = np.argsort(-np.abs(med))[:4]
+        prof = {DRIVERS[t]: float(med[t]) for t in top
+                if abs(med[t]) >= 0.05}
+        out.append({"n": len(idx), "top": prof,
+                    "syms": [syms[i] for i in idx]})
+    return out
+
+
+SLEEVE_NAME = {
+    "qqq": "US growth", "ivv": "US large blend", "iwm": "US small",
+    "efa": "intl developed", "vwo": "EM equity", "vnq": "REIT",
+    "shv": "cash/ultra-short", "shy": "short T 1-3y",
+    "ief": "T 7-10y", "tlt": "long T", "agg": "core bond (AGG)",
+    "vblix": "vol (pure)", "vweax": "HY corporate", "vmbix": "MBS",
+    "lqd": "IG corporate", "hyg": "high yield", "pff": "preferreds",
+    "emb": "EM debt", "tip": "TIPS/inflation", "vtv": "US value",
+    "djp": "gas", "gsg": "oil", "gld": "gold", "fxe": "EUR",
+    "fxy": "JPY", "xlk": "tech", "xlf": "financials", "xle": "energy",
+    "xlv": "healthcare", "xlp": "staples", "xlu": "utilities",
+    "xly": "consumer disc", "xlb": "materials", "dbmf": "CTA/futures",
+    "dbb": "commodities",
+}
+
+
+def label(prof: dict) -> str:
+    # a real driver shows up as >=0.3 on a median loading; below that the
+    # cluster is balanced mixes with no single dominant driver
+    if not prof or max(abs(v) for v in prof.values()) < 0.30:
+        return "no dominant driver (balanced/idio)"
+    names = sorted(prof, key=lambda s: -abs(prof[s]))
+    return " + ".join(f"{SLEEVE_NAME.get(s, s)} {prof[s]:+.2f}"
+                      for s in names[:3])
+
+
+def kmeans(V: np.ndarray, k: int, seed: int = 0,
+            iters: int = 100) -> np.ndarray:
+    """k-means++ init, deterministic seed. Returns labels (len n)."""
+    rng = np.random.default_rng(seed)
+    n = len(V)
+    # k-means++ init
+    C = np.empty((k, V.shape[1]))
+    C[0] = V[rng.integers(n)]
+    d2 = ((V - C[0]) ** 2).sum(1)
+    for j in range(1, k):
+        probs = d2 / d2.sum()
+        C[j] = V[rng.choice(n, p=probs)]
+        d2 = np.minimum(d2, ((V - C[j]) ** 2).sum(1))
+    for _ in range(iters):
+        dist = ((V[:, None, :] - C[None, :, :]) ** 2).sum(2)   # n x k
+        lab = dist.argmin(1)
+        newC = C.copy()
+        for j in range(k):
+            m = lab == j
+            if m.any():
+                newC[j] = V[m].mean(0)
+        if np.allclose(newC, C):
+            break
+        C = newC
+    return dist.argmin(1)
+
+
+def run_kmeans(k: int = 30) -> dict:
+    syms, V = loading_matrix()
+    lab = kmeans(V, k)
+    out: dict[int, dict] = {}
+    for c in range(k):
+        idx = [i for i, l in enumerate(lab) if l == c]
+        if not idx:
+            continue
+        med = np.median(V[idx], axis=0)
+        top = np.argsort(-np.abs(med))[:4]
+        prof = {DRIVERS[t]: float(med[t]) for t in top
+                if abs(med[t]) >= 0.08}
+        out[c] = {"n": len(idx), "top": prof, "label": label(prof),
+                  "syms": [syms[i] for i in idx]}
+    return {"k": k, "clusters": out}
+
+
+def run() -> None:
+    t0 = time.time()
+    syms, V = loading_matrix()
+    log(f"cluster: {len(syms)} funds x {V.shape[1]} drivers")
+    merges = hclust(V)
+    TREE.write_text(json.dumps({"syms": syms, "merges": merges,
+                                "n": len(syms), "sleeves": DRIVERS}))
+    log(f"hierarchical tree written in {time.time() - t0:.0f}s -> {TREE.name}")
+    # NOTE: fixed-k cuts of the tree peel small pure-sleeve clusters off a
+    # giant blend blob (most funds are multi-sleeve mixes) - the useful
+    # grouping is k-means on the loading vectors instead.
+    res = run_kmeans(30)
+    KMEANS.write_text(json.dumps(res))
+    log(f"k-means k=30 written in {time.time() - t0:.0f}s -> {KMEANS.name}")
+    for c in sorted(res["clusters"], key=lambda c: -res["clusters"][c]["n"]):
+        v = res["clusters"][c]
+        log(f"  n={v['n']:4d}  {v['label']}")
+
+
+if __name__ == "__main__":
+    run()

fundlab/factors.log

@@ -0,0 +1,14 @@
+07:30:21 factor screen: 2384 funds, 2384 to do
+07:30:21 building driver panel (37 axes)
+07:30:23   200/2384
+07:30:26   400/2384
+07:30:29   600/2384
+07:30:33   800/2384
+07:30:36   1000/2384
+07:30:40   1200/2384
+07:30:44   1400/2384
+07:30:47   1600/2384
+07:30:51   1800/2384
+07:30:54   2000/2384
+07:30:57   2200/2384
+07:31:00 factor screen done: 2384 funds -> factor_results.json

fundlab/factors.py

@@ -0,0 +1,197 @@
+"""Factor screen v2: full OLS of every screened fund on the EXPANDED
+sleeve set -> per-fund loading vectors.
+
+This is the "benchmark-neutral" representation: each fund is expressed
+as a vector of factor loadings (how much of its return each driver
+explains) + an unexplained alpha. Clustering the loadings groups funds
+by RETURN DRIVER (fundlab/cluster.py), which is what you want when
+shopping for replacements - not a yes/no alpha verdict.
+
+Deliberately OVERINCLUSIVE vs the v1 21-sleeve set: short/long rates,
+IG/HY credit, munis, TIPS, preferreds, EM debt, style factors, all 8
+major sectors, CTA, commodities. Missing sleeves (JNK, MBS, MUB, IJM,
+HFR, XLI, XLC) - no clean long-history proxy in the local DB.
+
+No portfolio-correlation screening anywhere in v2: corr_port is kept
+as an information column (the user is considering replacing holdings,
+so high-corr funds are REPLACEMENTS, not rejections).
+
+Output: fundlab/factor_results.json  (one entry per screened fund:
+full + 5y loadings, R2, alpha, t, plus carried screen metadata).
+"""
+from __future__ import annotations
+
+import json
+import time
+from functools import lru_cache
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+from fundlab import decompose
+from fundlab.searchlist import BROAD_SLEEVES, PORTFOLIO
+
+HERE = Path(__file__).parent
+RESULTS = HERE / "factor_results.json"
+SEARCH_ALL = HERE / "search_all.json"
+SELECTED = HERE / "universe_cache" / "selected.json"
+
+# expanded sleeve set: the v1 21 + rates/credit/style/sectors/CTA
+EXTRA_SLEEVES = [
+    # short rates (carry axis, separate from T-bills)
+    "shy",                       # 1-3y Treasuries
+    # IG corporate (agg is ~half Treasuries; lqd isolates credit)
+    "lqd",
+    # high yield / credit spreads
+    "hyg", "pff", "emb",
+    # TIPS (real rates)
+    "tip",
+    # US style factors (qqq/ivv blend growth+value)
+    "vug", "vtv",
+    # sectors (1998+, overinclusive)
+    "xlk", "xlf", "xle", "xlv", "xlp", "xlu", "xly", "xlb",
+    # CTA / managed futures
+    "dbmf",
+    # broad commodities (gsg/djp/gld are single-commodity)
+    "dbb",
+]
+SLEEVES_V2 = list(dict.fromkeys(BROAD_SLEEVES + EXTRA_SLEEVES))
+
+# regression/cluster basis: drop the >0.95 duplicates (2010+ correlations:
+# vea~efa 0.995, qqq~vug 0.976, vug~xlk 0.955). bil/shv/shy stay - they
+# finux (Fidelity Intl Bond) TERMINATED 2017 - it breaks the complete-
+# case mask for every fund with post-2017 history (the v1 forward
+# selector never hit this because sleeves were optional; full OLS needs
+# a common sample). Residualize vblix on [ivv, tlt] (tlt~vblix 0.963)
+# -> a PURE vol axis.
+# bil dropped too: for near-cash funds, plain OLS expresses a tiny net
+# short-rate exposure as huge offsetting shv/bil coefficients (shv +64 /
+# bil -54) - shv (1-3m) alone carries the cash/ultra-short axis, shy
+# keeps the 1-3y axis.
+DROP_FROM_REGRESSION = {"vea", "vug", "finux", "bil"}
+DRIVERS = [s for s in SLEEVES_V2 if s not in DROP_FROM_REGRESSION]
+
+
+def _resid(y: pd.Series, X: pd.DataFrame) -> pd.Series:
+    m = y.notna() & X.notna().all(axis=1)
+    Xa = np.column_stack([np.ones(m.sum())] + [X[c][m].to_numpy()
+                                                for c in X.columns])
+    beta, *_ = np.linalg.lstsq(Xa, y[m].to_numpy(), rcond=None)
+    full = np.column_stack([np.ones(len(y))] +
+                           [np.where(m, X[c], np.nan) for c in X.columns])
+    r = y.to_numpy() - full @ beta
+    return pd.Series(np.where(m, r, np.nan), index=y.index, name=y.name)
+
+
+def log(msg: str) -> None:
+    print(time.strftime("%H:%M:%S"), msg, flush=True)
+
+
+@lru_cache(maxsize=1)
+def _panel() -> pd.DataFrame:
+    log(f"building driver panel ({len(DRIVERS)} axes)")
+    p = decompose.returns_panel(SLEEVES_V2).sort_index()
+    p = p[~p.index.duplicated(keep="last")][DRIVERS]
+    # pure vol axis: vblix residualized on the core equity/duration axes
+    core = [c for c in ("ivv", "tlt") if c in p.columns]
+    if "vblix" in p.columns and core:
+        p["vblix"] = _resid(p["vblix"], p[core])
+    return p
+
+
+# small ridge on the sleeve columns (NOT the intercept): without it,
+# near-collinear rate sleeves (shv/bil/shy) leave a near-null direction
+# in which cash-like funds get huge offsetting loadings (shv +64, bil -54)
+# along an arbitrary axis - garbage for clustering.
+RIDGE = 0.02
+
+
+def _beta_window(y: pd.Series, X: pd.DataFrame, start: str) -> dict | None:
+    idx = y.index.intersection(X.index[X.index >= pd.to_datetime(start)])
+    y, X = y.loc[idx], X.loc[idx]
+    m = y.notna() & X.notna().all(axis=1)
+    y, X = y[m], X[m]
+    if len(y) < 250:
+        return None
+    cols = [X[c].to_numpy() for c in X.columns]
+    Xa = np.column_stack([np.ones(len(y))] + cols)
+    yv = y.to_numpy()
+    lam = np.eye(len(Xa.T)) * RIDGE
+    lam[0, 0] = 0.0                    # don't shrink the intercept
+    beta, *_ = np.linalg.lstsq(Xa.T @ Xa + lam, Xa.T @ yv, rcond=None)
+    resid = yv - Xa @ beta
+    ssr = float(resid @ resid)
+    yss = float(((yv - yv.mean()) ** 2).sum())
+    r2 = 1.0 - ssr / yss if yss > 0 else 0.0
+    # t for the intercept (unshrunken OLS inference on the raw fit)
+    fit = decompose.ols(yv, Xa)
+    return {
+        "n": int(len(y)),
+        "betas": {c: float(beta[i + 1]) for i, c in enumerate(X.columns)},
+        "r2": float(r2),
+        "alpha_ann": float(fit["beta"][0]) * 252,
+        "alpha_t": float(fit["t"][0]),
+    }
+    return {
+        "n": int(len(y)),
+        "betas": {c: float(fit["beta"][i + 1]) for i, c in enumerate(X.columns)},
+        "r2": float(fit["r2"]),
+        "alpha_ann": float(fit["beta"][0]) * 252,   # same as decompose()
+        "alpha_t": float(fit["t"][0]),
+    }
+
+
+def factor_screen(sym: str, start: str = "1990-01-01") -> dict | None:
+    fund = decompose.adj_close(sym)
+    if fund is None or len(fund) < 250:
+        return None
+    y = fund.pct_change().dropna()
+    X = _panel().reindex(y.index)
+    # sleeves must have history for most of the fund's window
+    avail = [c for c in X.columns if X[c].notna().mean() > 0.3]
+    full = _beta_window(y, X[avail], start)
+    rec = _beta_window(y, X[avail], "2021-01-01")
+    return {"full": full, "rec5": rec}
+
+
+def run() -> None:
+    def _ok(v: dict) -> bool:
+        f = v.get("full") or v.get("rec5")
+        return isinstance(f, dict) and "betas" in f
+
+    sel = json.loads(SELECTED.read_text())
+    search_all = (json.loads(SEARCH_ALL.read_text())
+                  if SEARCH_ALL.exists() else {})
+    out = json.loads(RESULTS.read_text()) if RESULTS.exists() else {}
+    # resume: skip only GOOD entries (broken/None ones get redone when
+    # the model changes - this bit us: finux-era None entries were
+    # skipped forever after finux was dropped)
+    todo = [r for r in sel if not _ok(out.get(r["sym"], {}))]
+    log(f"factor screen: {len(sel)} funds, {len(todo)} to do")
+    for i, r in enumerate(todo):
+        sym = r["sym"]
+        try:
+            res = factor_screen(sym)
+        except Exception as e:
+            res = {"error": str(e)}
+        row = out.get(sym, {})
+        row.update({"name": r["name"], "local": r["local"],
+                    "alpha_name": r["alpha_name"], **res})
+        # carry screen metadata (information columns, no screening)
+        sa = search_all.get(sym, {})
+        for k in ("corr_portfolio", "corr_benchmark", "verdict",
+                  "alpha_ann_5y", "alpha_t_5y", "alpha_t_full",
+                  "r2_5y", "max_dd", "alpha_pos_frac"):
+            if isinstance(sa.get(k), (int, float, str)):
+                row[k] = sa[k]
+        out[sym] = row
+        if (i + 1) % 200 == 0:
+            RESULTS.write_text(json.dumps(out, default=str))
+            log(f"  {i + 1}/{len(todo)}")
+    RESULTS.write_text(json.dumps(out, default=str))
+    log(f"factor screen done: {len(out)} funds -> {RESULTS.name}")
+
+
+if __name__ == "__main__":
+    run()

fundlab/streamlit.log

@@ -0,0 +1,11 @@
+
+Collecting usage statistics. To deactivate, set browser.gatherUsageStats to false.
+
+2026-08-27 06:56:40.082 Uvicorn server started on :::8599
+
+  You can now view your Streamlit app in your browser.
+
+  Local URL: http://localhost:8599
+  Network URL: http://192.168.3.6:8599
+  External URL: http://100.33.61.109:8599
+