Greg Pomerantz
592d12958f
fundlab/edgar_universe.py - the 'search' for funds we don't have:
1. SEC full-index (Archives/edgar/full-index/YYYY/QTRn/company.gz)
lists every filing; CIKs that filed a base 497/497K in the past 4
quarters = every currently-active US open-end fund (1,668).
2. one small fetch per CIK: the full-submission .txt carries the
line-based SGML prospectus cover (<SERIES-NAME> ... unclosed
<CLASS-CONTRACT-TICKER-SYMBOL> tags) - fund name + every class
ticker, often several funds per filing.
3. alpha-leaning name filter (expanded dbmine PATTERN: +relative
value, risk allocation, dynamic global, real return, hedged),
drop local-DB + shortlist tickers,
4. Yahoo chart verify: instrumentType MUTUALFUND (OTC open-end;
exchange name is useless - OTC funds report 'Nasdaq'),
>=5y daily history,
5. share-class dedupe (longest history), goget download, same
screen_fund engine.
Resumable (per-CIK covers cache), 4-thread, Range-free small files.
First pass results (46 funds screened, 5 NEW candidates):
egrix/ecgmx Eaton Vance Global Macro Absolute Return: R2 0.07,
+7.9%/+4.8% 5y alpha, t 4.9/4.6, corr-port 0.22 - pure macro idio
dmszx Destinations Multi-Strategy Alternatives: R2 0.57, +3.3%, t3.5
cbhax Victory Market Neutral Income: R2 0.07, +4.6%, t2.9, corr 0.11
pdinx Putnam Diversified Income: semi-alpha (full t5.8, 62% 6m+)
(+ wmnux/gioax = 2nd share classes of already-known candidates)
vmnix Vanguard MN: alpha but corr 0.35 (portfolio already 50% MN)
app Fund Lab alpha table now also reads search_external.json.
tests: parse_cover unit tests (unclosed-tag SGML, ticker series
attach, malformed rejected). 65/65 fundlab, 32/32 app.
130 lines
4.6 KiB
Python
130 lines
4.6 KiB
Python
"""Mine the local stocks DB for alpha-leaning open-end funds and screen
|
|
them with the same engine (fundlab.search.screen_fund).
|
|
|
|
The DB already holds ~8k symbols incl. a rich set of US open-end
|
|
alternatives (AQR, PIMCO, JPM, Principal, Calamos, GMO, Franklin K2, ...).
|
|
The miner:
|
|
1. scans every <sym>.json for alpha-leaning fund names
|
|
(market-neutral / long-short / multi-strategy / macro / managed
|
|
futures / absolute return / multi-asset TA / risk premia /
|
|
alternatives),
|
|
2. dedupes share classes (same fund, A/I/N/R6/Z/Instl...) keeping the
|
|
class with the longest history,
|
|
3. screens each survivor (requires >= 5y of data).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import glob
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
DATA = Path.home() / "prog/fin/stocks"
|
|
OUT = Path(__file__).parent / "search_mined.json"
|
|
|
|
PATTERN = re.compile(
|
|
r"(market neutral|long.?/?.?short|multi.?strateg|managed futures|"
|
|
r"macro opportunit|global macro|alternative (strateg|strats|risk|asset|"
|
|
r"income|core|allocation)|alternatives fund|absolute return|"
|
|
r"risk premia|multi.?asset (income|absolute|ult|balanced)|"
|
|
r"tactical allocation|trends fund|market trend|opportunistic (equity|long)|"
|
|
r"multi.?manager|diversified (alternatives|income)|relative value|"
|
|
r"risk allocation|dynamic global|real return|hedge?d (allocation|strateg))", re.I)
|
|
|
|
# skip anything that is really an ETF wrapper or index
|
|
SKIP = re.compile(
|
|
r"(exchange.?traded|etf trust|index fund|s&p 500|nasdaq 100|"
|
|
r"real estate trust|reit\b|grayscale|liquidation|royalty|bitcoin|"
|
|
r"litecoin|ethereum|multimanager (20|lifestyle)|core plus)", re.I)
|
|
|
|
# trailing share-class tokens for family dedupe
|
|
CLASS_TOK = re.compile(
|
|
r"\b(a|i|n|c|b|z|x|r6|r5|r4|svc|inst|instl|institutional|advisor|"
|
|
r"retail|investor|plus|class [a-z]?\d?|series [a-z]?)\b\.?$", re.I)
|
|
|
|
KNOWN = set(json.loads(
|
|
(Path(__file__).parent.parent / "funds.json").read_text()))
|
|
|
|
|
|
def _known_families() -> set[str]:
|
|
"""Family keys of the shortlist, so other share classes of the SAME
|
|
fund (qspix vs the shortlist's qspnx) are not mined as new funds."""
|
|
out = set()
|
|
for name in json.loads(
|
|
(Path(__file__).parent.parent / "funds.json").read_text()):
|
|
out.add(family_key(name))
|
|
return out
|
|
|
|
|
|
def family_key(name: str) -> str:
|
|
t = re.sub(r"[^a-z0-9 ]+", " ", name.lower())
|
|
prev = None
|
|
while prev != t:
|
|
prev = t
|
|
t = CLASS_TOK.sub(" ", t)
|
|
t = re.sub(r"\s+", " ", t).strip()
|
|
# also strip leading trust/series wrappers ("trust for professional...")
|
|
t = re.sub(r"^(trust for |investment managers series [a-z0-9 ]*-?\s*)",
|
|
"", t)
|
|
return t
|
|
|
|
|
|
KNOWN_FAMS = _known_families()
|
|
|
|
|
|
def mine() -> list[dict]:
|
|
fams: dict[str, dict] = {}
|
|
for f in glob.glob(str(DATA / "*.json")):
|
|
sym = Path(f).name[:-5].lower()
|
|
# KNOWN = the shortlist in funds.json; eigmx is the I class of the
|
|
# shortlist's eagmx (EV Global Macro) - don't double-count it
|
|
if sym in KNOWN or sym == "eigmx":
|
|
continue
|
|
try:
|
|
d = json.load(open(f))
|
|
res = d["chart"]["result"][0]
|
|
meta = res["meta"]
|
|
except Exception:
|
|
continue
|
|
itype = (meta.get("instrumentType") or "").upper()
|
|
if itype == "ETF":
|
|
continue
|
|
name = meta.get("longName") or meta.get("shortName") or ""
|
|
if not PATTERN.search(name) or SKIP.search(name):
|
|
continue
|
|
days = len(res.get("timestamp", []) or [])
|
|
if days < 1250: # need >= 5y
|
|
continue
|
|
key = family_key(name)
|
|
if len(key) < 12 or key in KNOWN_FAMS:
|
|
continue
|
|
if key not in fams or days > fams[key]["days"]:
|
|
fams[key] = {"sym": sym, "name": name, "days": days}
|
|
out = sorted(fams.values(), key=lambda x: x["name"])
|
|
return out
|
|
|
|
|
|
def run(screen: bool = True) -> dict:
|
|
fams = mine()
|
|
print(f"mined {len(fams)} distinct fund families", flush=True)
|
|
results = {}
|
|
if screen:
|
|
from fundlab import search
|
|
for c in fams:
|
|
row = search.screen_fund(c["sym"], c["name"], "mined")
|
|
results[c["sym"]] = row
|
|
print(f"{c['sym']:7} {c['name'][:44]:44} "
|
|
f"{row.get('verdict', row.get('error'))[:44]}",
|
|
flush=True)
|
|
else:
|
|
for c in fams:
|
|
print(f" {c['sym']:7} {c['days']:5}d {c['name'][:60]}")
|
|
OUT.write_text(json.dumps(results or fams, indent=1, default=str))
|
|
print(f"wrote {OUT}")
|
|
return results
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(screen="--no-screen" not in sys.argv)
|