diff --git a/app.py b/app.py index 153b328..8981121 100644 --- a/app.py +++ b/app.py @@ -617,9 +617,11 @@ with tab_fundlab: # --- alpha search: all screened funds (shortlist + longlist + harvest) with st.expander("Alpha search — all screened funds, ranked"): - _all_rows: list[dict] = [] - for _src in ("search_results.json", "search_mined.json", - "search_external.json"): + # search_all (comprehensive overnight screen) is the superset; + # read it first so its rows win any sym collision. + _by_sym: dict[str, dict] = {} + for _src in ("search_all.json", "search_results.json", + "search_mined.json", "search_external.json"): try: _j = json.loads((_dc.RESULTS.parent / _src).read_text()) except Exception: @@ -627,7 +629,8 @@ with tab_fundlab: for _k, _v in _j.items(): if isinstance(_v, dict) and _v.get("sym"): _v.setdefault("source", _src) - _all_rows.append(_v) + _by_sym.setdefault(_v["sym"], _v) + _all_rows = list(_by_sym.values()) if _all_rows: def _tier(r) -> int: v = str(r.get("verdict", "")) diff --git a/fundlab/RESEARCH.md b/fundlab/RESEARCH.md new file mode 100644 index 0000000..4e0df7c --- /dev/null +++ b/fundlab/RESEARCH.md @@ -0,0 +1,122 @@ +# Fund discovery research log + +Running log of attempts, dead ends, and learnings for finding candidate +funds (alpha-driven, portfolio-complementing) with the `fundlab` +pipeline. Newest entries at the bottom of each section. Update as you +go — this file IS the knowledge base. + +## Sources & what works + +| Source | Status | Notes | +|---|---|---| +| Local stocks DB (`~/prog/fin/stocks/`, 8,250 syms) | ✅ primary | ~100 open-end alt families already present; `dbmine.py` mines by name pattern | +| SEC full-index `Archives/edgar/full-index/YYYY/QTRn/company.gz` | ✅ **discovery workhorse** | lists EVERY filing; 497/497K filers = all active open-end funds. 4-quarter union = 1,668 CIKs, 33,188 series, 10,372 class tickers | +| SEC full-submission `.txt` (per accession) | ✅ | ~10–50KB; line-based SGML cover with `` (UNCLOSED tag) + `` per class; often several series per filing | +| Yahoo chart API (`query1.../v8/finance/chart/`) | ✅ | no crumb needed; meta has instrumentType/exchange/longName; `range=20y` gives history length | +| `goget` (`~/go/bin/goget`) | ✅ | batch downloader, idempotent, ~2–5s/sym | +| EDGAR FTS (`efts.sec.gov/LATEST/search-index`) | ⚠️ fragile | phrase queries w/ hyphens fail; AND-semantics; 100-hit cap per query → incomplete for common phrases. OK for rare phrases only | +| SEC `browse-edgar` company listing | ❌ dead | now JS-rendered, no data in HTML | +| SEC `/files/investment-company-tickers.json` | ❌ doesn't exist | 404 (misremembered) | +| SEC `company_tickers.json` | ❌ for our purpose | only exchange-listed (ETFs/CEFs/stocks); open-end OTC funds absent | +| Yahoo search/crumb API | ❌ throttled | IP-level "Too Many Requests" on `fc.yahoo.com`/`getcrumb`; chart API unaffected | +| stockanalysis.com/funds/ | ❌ 404 | path guessed wrong, not pursued | + +## Learnings (hard-won) + +1. **Famous multi-strategy/macro funds are private/offshore** — Millennium, + Balyasny, Schonfeld, ExodusPoint, Two Sigma, Winton, Marshall Wace, + Brevan Howard, AQR Event-Driven: no US open-end class, no EDGAR 497, + no Yahoo OTC ticker. Structural, not a search failure. +2. **Precision over recall for name→ticker resolution**: a guessed + ticker that "looks right" is worse than no answer. Chart-API name + gate + 2/3 token overlap rejected 23/24 memory-based guesses. +3. **Yahoo exchange name is a USELESS fund/ETF discriminator**: OTC + mutual funds report `fullExchangeName="Nasdaq"`. Use + `instrumentType` (MUTUALFUND vs ETF). +4. **497 SGML cover uses UNCLOSED tags, one per line** — + `Foo Fund\nTNMAX\n`. + Closed-tag regexes find nothing; `to_text()` (tag stripping) + destroys the data. Parse line-based, BEFORE any tag stripping. +5. **Full-index columns drift** — don't trust fixed widths; the header + line and data rows don't align. Regex the whole line. +6. **One quarter ≠ the universe**: each fund's annual base-497 + re-filing lands in a random quarter; union of 4 consecutive + quarters is the full active universe (1,209 in Q2 alone → 1,668 + union). +7. **Accession paths in the index are relative to `/Archives/`**, not + `/Archives/edgar/data/` — doubling the prefix 404s. +8. **Amendments (497A/497VPU) may lack the series cover** — base 497 / + 497K carry it. (497A inclusion pending — see below.) +9. **Family CIKs repeat across the index** (e.g. AB under 2 CIKs, + same series listed twice) — dedupe by series name, not CIK. +10. **Large-n BIC is knife-edge** (ΔBIC=2 ≈ ΔR²=0.0008 at n=2500) — + the |t|>2 gate on added regressors is essential (decompose.py). +11. **The portfolio is 50% market-neutral (qspnx)** — MN/L-S-equity + funds show the strongest alpha on screen (+12–17%/yr) but corr + 0.35–0.76 with the portfolio. "Alpha" ≠ "diversifying for YOU". +12. **Near-duplicate sleeves make OLS knife-edge** — distinct-axis + candidate sets (one rep per sleeve family) or the betas split + arbitrarily between ivv/vti/vt. +13. **Wrong-fund objectives are worse than none** — all EDGAR + resolution stages gate on name match before accepting. + +## Pipeline stages (current) + +``` +full-index (4 qtrs) → per-CIK latest 497/497K .txt (cached, 4-thread) + → parse_cover (line-based SGML) + → [name filter — REMOVED in v2, kept as a tag] + → Yahoo chart verify (instrumentType, 20y length) + → share-class dedupe (longest history) + → goget missing (batched) + → screen_fund (sleeve OLS, BIC fwd-select, 5y alpha t, persistence, + corr vs portfolio & benchmark) [0.3s/fund] + → search_*.json → app Fund Lab "Alpha search" table +``` + +## Overnight comprehensive screen (v2, started 2026-08-26) + +Goal: screen EVERY OTC open-end fund with ≥5y history from the 497 +universe — no name pre-filter. Name match becomes a tag/cross-check, +not a gate. + +### Attempts & progress +- [x] Survey: 10,372 unique class tickers in covers cache; screen + costs 0.3s/fund → local screen of ~5k families ≈ 25 min. + Bottlenecks: Yahoo verify (~20 min threaded) + goget + (~3–5 h for ~4k missing syms) — the overnight part. +- [ ] Yahoo verify all non-local tickers → `universe_cache/yahoo_meta.json` +- [ ] goget in resumable batches +- [ ] screen all → `search_all.json` (streamed, resumable) +- [ ] app reads search_all.json; cross-check: alpha funds whose names + DON'T match the alpha pattern (what the v1 filter missed) + +### Notes while running +(empty — append as the run progresses) +- 2026-08-26 16:01 === stage verify === +- 2026-08-26 16:01 verify: 10372 tickers, 9149 to hit Yahoo +- 2026-08-26 16:01 verify: 500/9149 +- 2026-08-26 16:02 verify: 1000/9149 +- 2026-08-26 16:03 verify: 1500/9149 +- 2026-08-26 16:03 verify: 2000/9149 +- 2026-08-26 16:04 verify: 2500/9149 +- 2026-08-26 16:04 verify: 3000/9149 +- 2026-08-26 16:05 verify: 3500/9149 +- 2026-08-26 16:05 verify: 4000/9149 +- 2026-08-26 16:06 verify: 4500/9149 +- 2026-08-26 16:06 verify: 5000/9149 +- 2026-08-26 16:07 verify: 5500/9149 +- 2026-08-26 16:07 verify: 6000/9149 +- 2026-08-26 16:08 verify: 6500/9149 +- 2026-08-26 16:09 verify: 7000/9149 +- 2026-08-26 16:09 verify: 7500/9149 +- 2026-08-26 16:10 verify: 8000/9149 +- 2026-08-26 16:10 verify: 8500/9149 +- 2026-08-26 16:11 verify: 9000/9149 +- 2026-08-26 16:11 verify done: 10260/10372 with data +- 2026-08-26 16:11 === overnight run finished in 0.2h === +- 2026-08-26 21:31 === stage select === +- 2026-08-26 21:31 select: 2384 funds (407 local, 1977 external) +- 2026-08-26 21:31 === overnight run finished in 0.0h === +- 2026-08-26 21:31 === stage download === +- 2026-08-26 21:31 download: 1960 missing symbols diff --git a/fundlab/overnight.log b/fundlab/overnight.log new file mode 100644 index 0000000..d1ee95b --- /dev/null +++ b/fundlab/overnight.log @@ -0,0 +1,2 @@ +21:31:57 === stage download === +21:31:57 download: 1960 missing symbols diff --git a/fundlab/overnight.py b/fundlab/overnight.py new file mode 100644 index 0000000..aed777b --- /dev/null +++ b/fundlab/overnight.py @@ -0,0 +1,245 @@ +"""Overnight comprehensive screen (v2): EVERY OTC open-end fund in the +497 universe, no name pre-filter. + +Everything is cached and resumable - safe to kill and restart: + universe_cache/covers.json CIK -> series+tickers (from edgar_universe) + universe_cache/yahoo_meta.json ticker -> {name, type, exch, days} + universe_cache/selected.json final (sym, name, days) work list + search_all.json screen results, streamed + resumed + +Stages: + 1. verify - Yahoo chart call per non-local ticker (threaded, 429 + backoff); local tickers measured from CSV row counts + 2. select - one class per series (longest history), >=5y, + instrumentType MUTUALFUND; name-tag (not a filter) + 3. download- goget the missing symbols in resumable batches + 4. screen - screen_fund per fund, skip already-screened, save often + 5. finalize- summary stats into RESEARCH.md + +Usage: python -m fundlab.overnight [stage] (default: all stages) +""" +from __future__ import annotations + +import json +import re +import subprocess +import time +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + +from fundlab import search +from fundlab.dbmine import PATTERN as ALPHA_KW + +HERE = Path(__file__).parent +CACHE = HERE / "universe_cache" +COVERS = CACHE / "covers.json" +META = CACHE / "yahoo_meta.json" +SELECTED = CACHE / "selected.json" +RESULTS = HERE / "search_all.json" +RESEARCH = HERE / "RESEARCH.md" +DATA = Path.home() / "prog/fin/stocks" +GOGET = Path.home() / "go/bin/goget" +MIN_DAYS = 1250 # ~5y of daily bars +UA = {"User-Agent": "research test@example.com"} + + +def log(msg: str) -> None: + print(time.strftime("%H:%M:%S"), msg, flush=True) + with RESEARCH.open("a") as f: + f.write(f"- {time.strftime('%Y-%m-%d %H:%M')} {msg}\n") + + +def all_tickers() -> dict[str, str]: + """ticker -> series name (first series seen wins).""" + d = json.loads(COVERS.read_text()) + out: dict[str, str] = {} + for v in d.values(): + for s in v.get("series", []): + name = s.get("name", "") + for t in s.get("tickers", []): + out.setdefault(t, name) + return out + + +# ---------------------------------------------------------------- stage 1 +def _yahoo_one(t: str) -> tuple[str, dict]: + url = (f"https://query1.finance.yahoo.com/v8/finance/chart/{t}" + f"?range=20y&interval=1d") + for attempt in range(4): + try: + req = urllib.request.Request(url, headers=search.UA) + d = json.load(urllib.request.urlopen(req, timeout=30)) + res = (d.get("chart") or {}).get("result") + if not res: + err = ((d.get("chart") or {}).get("error") or {}) + return t, {"error": str(err.get("code", "no result"))} + meta = res[0].get("meta", {}) + return t, {"name": meta.get("longName") or + meta.get("shortName") or "", + "type": (meta.get("instrumentType") or "").upper(), + "exch": meta.get("fullExchangeName") or "", + "days": len(res[0].get("timestamp", []))} + except urllib.error.HTTPError as e: + if e.code == 429 and attempt < 3: + time.sleep(5 * (attempt + 1)) + continue + return t, {"error": f"HTTP {e.code}"} + except Exception as e: + return t, {"error": str(e)} + return t, {"error": "retries exhausted"} + + +def stage_verify(workers: int = 4) -> dict: + tickers = all_tickers() + local = {p.name[:-5].lower() for p in DATA.glob("*.json")} + meta: dict = (json.loads(META.read_text()) if META.exists() else {}) + # local tickers: measure from CSV row counts (no Yahoo call) + for t in tickers: + if t.lower() in local and t not in meta: + csv = DATA / f"{t.lower()}-history.csv" + if not csv.exists(): + meta[t] = {"local": True, "days": 0, + "type": "MUTUALFUND", + "name": tickers[t], "exch": "local"} + continue + with csv.open() as f: + days = max(0, sum(1 for _ in f) - 1) + meta[t] = {"local": True, "days": days, "type": "MUTUALFUND", + "name": tickers[t], "exch": "local"} + todo = [t for t in tickers if t not in meta + and t.lower() not in local] + log(f"verify: {len(tickers)} tickers, {len(todo)} to hit Yahoo") + done = 0 + with ThreadPoolExecutor(max_workers=workers) as ex: + futs = {ex.submit(_yahoo_one, t): t for t in todo} + for fut in as_completed(futs): + t, res = fut.result() + meta[t] = res + done += 1 + if done % 500 == 0: + META.write_text(json.dumps(meta)) + log(f"verify: {done}/{len(todo)}") + META.write_text(json.dumps(meta)) + ok = sum(1 for v in meta.values() + if isinstance(v.get("days"), int)) + log(f"verify done: {ok}/{len(meta)} with data") + return meta + + +# ---------------------------------------------------------------- stage 2 +def select_rows(tickers: dict[str, str], meta: dict, min_days: int = MIN_DAYS) -> list[dict]: + """Pure: (ticker->series-name, ticker->meta) -> one class per fund. + + Keeps MUTUALFUND classes with >= min_days bars; collapses share + classes (same series name) to the longest-history one; tags each + fund with whether its NAME matches the alpha pattern (a label, not + a filter in v2). + """ + out = [] + for t, name in tickers.items(): + m = meta.get(t) or {} + if not isinstance(m.get("days"), int): + continue + if m.get("type", "MUTUALFUND") != "MUTUALFUND": + continue + if m["days"] < min_days: + continue + out.append({"sym": t.lower(), "ticker": t, "name": name or + m.get("name", ""), "days": m["days"], + "local": bool(m.get("local")), + "alpha_name": bool(ALPHA_KW.search(name or ""))}) + by_name: dict[str, dict] = {} + for row in out: + key = row["name"] or row["sym"] + cur = by_name.get(key) + if cur is None or row["days"] > cur["days"]: + by_name[key] = row + return sorted(by_name.values(), key=lambda r: r["sym"]) + + +def stage_select() -> list[dict]: + sel = select_rows(all_tickers(), json.loads(META.read_text())) + SELECTED.write_text(json.dumps(sel, indent=1)) + log(f"select: {len(sel)} funds " + f"({sum(1 for r in sel if r['local'])} local, " + f"{sum(1 for r in sel if not r['local'])} external)") + return sel + + +# ---------------------------------------------------------------- stage 3 +def stage_download(batch: int = 200) -> None: + sel = json.loads(SELECTED.read_text()) + missing = [r["sym"] for r in sel + if not (DATA / f"{r['sym']}-history.csv").exists()] + log(f"download: {len(missing)} missing symbols") + for i in range(0, len(missing), batch): + chunk = missing[i:i + batch] + subprocess.run([str(GOGET), *chunk], cwd=DATA, capture_output=True, + timeout=7200) + got = sum(1 for s in chunk if (DATA / f"{s}-history.csv").exists()) + log(f"download: batch {i//batch + 1} -> {got}/{len(chunk)}") + still = sum(1 for s in missing + if not (DATA / f"{s}-history.csv").exists()) + log(f"download done: {still} still missing (no Yahoo data?)") + + +# ---------------------------------------------------------------- stage 4 +def stage_screen() -> None: + sel = json.loads(SELECTED.read_text()) + results = (json.loads(RESULTS.read_text()) if RESULTS.exists() else {}) + todo = [r for r in sel if r["sym"] not in results] + log(f"screen: {len(sel)} funds, {len(todo)} to do") + for i, r in enumerate(todo): + try: + row = search.screen_fund(r["sym"], r["name"], "all") + except Exception as e: + row = {"sym": r["sym"], "name": r["name"], "source": "all", + "error": str(e)} + row["alpha_name"] = r["alpha_name"] + results[r["sym"]] = row + if (i + 1) % 100 == 0: + RESULTS.write_text(json.dumps(results, default=str)) + log(f"screen: {i + 1}/{len(todo)}") + RESULTS.write_text(json.dumps(results, default=str)) + log(f"screen done: {len(results)} funds") + + +# ---------------------------------------------------------------- stage 5 +def stage_finalize() -> None: + results = json.loads(RESULTS.read_text()) + rows = [v for v in results.values() if isinstance(v, dict)] + from collections import Counter + verdicts = Counter(str(v.get("verdict", v.get("error", "?"))).split(" -")[0] + .split(" (")[0] for v in rows) + log(f"finalize: {len(rows)} funds screened") + for k, n in verdicts.most_common(): + log(f" {n:5d} {k}") + # alpha funds whose names did NOT match the pattern (v1 blind spot) + missed = [v for v in rows if v.get("alpha_name") is False + and str(v.get("verdict", "")).startswith("CANDIDATE")] + log(f" candidates v1 name-filter would have missed: " + f"{[v['sym'] for v in missed]}") + + +def run(stages: list[str] | None = None) -> None: + stages = stages or ["verify", "select", "download", "screen", "finalize"] + t0 = time.time() + for s in stages: + log(f"=== stage {s} ===") + if s == "verify": + stage_verify() + elif s == "select": + stage_select() + elif s == "download": + stage_download() + elif s == "screen": + stage_screen() + elif s == "finalize": + stage_finalize() + log(f"=== overnight run finished in {(time.time() - t0) / 3600:.1f}h ===") + + +if __name__ == "__main__": + import sys + run(sys.argv[1:] or None) diff --git a/tests/test_fundlab.py b/tests/test_fundlab.py index 8e3b552..a785410 100644 --- a/tests/test_fundlab.py +++ b/tests/test_fundlab.py @@ -338,6 +338,36 @@ def test_universe() -> None: ["tickers"] == [], "") +def test_overnight() -> None: + print("overnight select", flush=True) + from fundlab import overnight as ov + tickers = {"AAAAX": "Foo Market Neutral Fund", + "AAAIJ": "Foo Market Neutral Fund", # 2nd class + "BBBAX": "Bar Growth Fund", + "CCCAX": "Baz ETF", + "DDDAX": "Qux Short History"} + meta = {"AAAAX": {"days": 3000, "type": "MUTUALFUND", "name": "x"}, + "AAAIJ": {"days": 2500, "type": "MUTUALFUND", "name": "x"}, + "BBBAX": {"days": 3000, "type": "MUTUALFUND", "name": "y"}, + "CCCAX": {"days": 3000, "type": "ETF", "name": "z"}, + "DDDAX": {"days": 500, "type": "MUTUALFUND", "name": "q"}, + "EEEAX": {"error": "404"}} + sel = ov.select_rows(tickers, meta, min_days=1250) + check("ETF dropped", [r["sym"] for r in sel].count("cccax") == 0, + str(sel)) + check("short history dropped", + [r["sym"] for r in sel].count("dddax") == 0, "") + check("classes collapse to longest", + [r["sym"] for r in sel].count("foo") == 0 and + [r for r in sel if r["name"] == "Foo Market Neutral Fund"] + [0]["sym"] == "aaaax", str(sel)) + check("alpha_name tagged", sel[0]["alpha_name"] is True, + str(sel[0])) + check("non-alpha name not tagged (but kept)", + any(r["sym"] == "bbabx" or r["sym"] == "bbbax" and + r["alpha_name"] is False for r in sel), str(sel)) + + def test_curated() -> None: print("curated", flush=True) import fundlab.fundinfo as fi @@ -361,6 +391,7 @@ def main() -> int: test_decompose() test_search() test_universe() + test_overnight() test_curated() test_edgar_live() print(f"\n{PASS} passed, {FAIL} failed")