f/fundlab/dbmine.py

"""Mine the local stocks DB for alpha-leaning open-end funds and screen
them with the same engine (fundlab.search.screen_fund).

The DB already holds ~8k symbols incl. a rich set of US open-end
alternatives (AQR, PIMCO, JPM, Principal, Calamos, GMO, Franklin K2, ...).
The miner:
  1. scans every <sym>.json for alpha-leaning fund names
     (market-neutral / long-short / multi-strategy / macro / managed
     futures / absolute return / multi-asset TA / risk premia /
     alternatives),
  2. dedupes share classes (same fund, A/I/N/R6/Z/Instl...) keeping the
     class with the longest history,
  3. screens each survivor (requires >= 5y of data).
"""
from __future__ import annotations

import glob
import json
import re
import sys
from pathlib import Path

DATA = Path.home() / "prog/fin/stocks"
OUT = Path(__file__).parent / "search_mined.json"

PATTERN = re.compile(
    r"(market neutral|long.?/?.?short|multi.?strateg|managed futures|"
    r"macro opportunit|global macro|alternative (strateg|strats|risk|asset|"
    r"income|core|allocation)|alternatives fund|absolute return|"
    r"risk premia|multi.?asset (income|absolute|ult|balanced)|"
    r"tactical allocation|trends fund|market trend|opportunistic (equity|long)|"
    r"multi.?manager|diversified (alternatives|income))", re.I)

# skip anything that is really an ETF wrapper or index
SKIP = re.compile(
    r"(exchange.?traded|etf trust|index fund|s&p 500|nasdaq 100|"
    r"real estate trust|reit\b|grayscale|liquidation|royalty|bitcoin|"
    r"litecoin|ethereum|multimanager (20|lifestyle)|core plus)", re.I)

# trailing share-class tokens for family dedupe
CLASS_TOK = re.compile(
    r"\b(a|i|n|c|b|z|x|r6|r5|r4|svc|inst|instl|institutional|advisor|"
    r"retail|investor|plus|class [a-z]?\d?|series [a-z]?)\b\.?$", re.I)

KNOWN = set(json.loads(
    (Path(__file__).parent.parent / "funds.json").read_text()))


def _known_families() -> set[str]:
    """Family keys of the shortlist, so other share classes of the SAME
    fund (qspix vs the shortlist's qspnx) are not mined as new funds."""
    out = set()
    for name in json.loads(
            (Path(__file__).parent.parent / "funds.json").read_text()):
        out.add(family_key(name))
    return out


def family_key(name: str) -> str:
    t = re.sub(r"[^a-z0-9 ]+", " ", name.lower())
    prev = None
    while prev != t:
        prev = t
        t = CLASS_TOK.sub(" ", t)
    t = re.sub(r"\s+", " ", t).strip()
    # also strip leading trust/series wrappers ("trust for professional...")
    t = re.sub(r"^(trust for |investment managers series [a-z0-9 ]*-?\s*)",
               "", t)
    return t


KNOWN_FAMS = _known_families()


def mine() -> list[dict]:
    fams: dict[str, dict] = {}
    for f in glob.glob(str(DATA / "*.json")):
        sym = Path(f).name[:-5].lower()
        # KNOWN = the shortlist in funds.json; eigmx is the I class of the
        # shortlist's eagmx (EV Global Macro) - don't double-count it
        if sym in KNOWN or sym == "eigmx":
            continue
        try:
            d = json.load(open(f))
            res = d["chart"]["result"][0]
            meta = res["meta"]
        except Exception:
            continue
        itype = (meta.get("instrumentType") or "").upper()
        if itype == "ETF":
            continue
        name = meta.get("longName") or meta.get("shortName") or ""
        if not PATTERN.search(name) or SKIP.search(name):
            continue
        days = len(res.get("timestamp", []) or [])
        if days < 1250:            # need >= 5y
            continue
        key = family_key(name)
        if len(key) < 12 or key in KNOWN_FAMS:
            continue
        if key not in fams or days > fams[key]["days"]:
            fams[key] = {"sym": sym, "name": name, "days": days}
    out = sorted(fams.values(), key=lambda x: x["name"])
    return out


def run(screen: bool = True) -> dict:
    fams = mine()
    print(f"mined {len(fams)} distinct fund families", flush=True)
    results = {}
    if screen:
        from fundlab import search
        for c in fams:
            row = search.screen_fund(c["sym"], c["name"], "mined")
            results[c["sym"]] = row
            print(f"{c['sym']:7} {c['name'][:44]:44} "
                  f"{row.get('verdict', row.get('error'))[:44]}",
                  flush=True)
    else:
        for c in fams:
            print(f"  {c['sym']:7} {c['days']:5}d  {c['name'][:60]}")
    OUT.write_text(json.dumps(results or fams, indent=1, default=str))
    print(f"wrote {OUT}")
    return results


if __name__ == "__main__":
    run(screen="--no-screen" not in sys.argv)