f/fundlab/harvest.py

"""Harvest candidate multi-asset / alternatives fund tickers from the SEC
company_tickers file, verify each resolves on Yahoo with real history,
download the survivors with goget, and screen them with the same engine.

The SEC file lists exchange symbols; Yahoo's chart API resolves many of
them to the underlying mutual fund (instrumentType + longName) AND serves
full NAV history for them (verified: PDI -> 2012..). CEFs (", Inc.",
"Trust, Inc.") are excluded - the screen targets open-end share classes.
"""
from __future__ import annotations

import json
import re
import subprocess
import urllib.request
from pathlib import Path

from fundlab import decompose, search
from fundlab.searchlist import BROAD_SLEEVES

SEC_TICKERS = Path("/tmp/company_tickers.json")
DATA = Path.home() / "prog/fin/stocks"
GOGET = Path.home() / "go/bin/goget"
OUT = Path(__file__).parent / "search_harvest.json"

FUND_KW = re.compile(r"\b(fund|funds|trust)\b", re.I)
CEF_KW = re.compile(r",\s*Inc\.|Trust,\s*Inc\.|, Inc\b|TRUST\s+[IVX]+,?\s*$",
                    re.I)


def harvest_candidates() -> list[dict]:
    d = json.loads(SEC_TICKERS.read_text())
    out = []
    for v in d.values():
        t, title = v.get("ticker", ""), v.get("title", "")
        if not t or len(t) < 4 or not FUND_KW.search(title):
            continue
        if CEF_KW.search(title):
            continue
        out.append({"ticker": t, "title": title})
    return out


def history_length(ticker: str) -> int:
    """Days of history Yahoo serves for a ticker; 0 if none, or if it's an
    exchange-listed instrument (ETF/stock - we want OTC fund classes)."""
    url = (f"https://query1.finance.yahoo.com/v8/finance/chart/"
           f"{ticker}?range=20y&interval=1d")
    req = urllib.request.Request(url, headers=search.UA)
    try:
        d = json.load(urllib.request.urlopen(req, timeout=30))
        res = (d.get("chart") or {}).get("result")
        if not res:
            return 0
        meta = res[0].get("meta") or {}
        exch = (meta.get("fullExchangeName") or "").upper()
        if any(e in exch for e in ("NASDAQ", "NYSE", "ARCA")):
            return 0
        return len(res[0].get("timestamp", []))
    except Exception:
        return 0


def run(min_days: int = 1250) -> dict:
    cands = harvest_candidates()
    print(f"harvested {len(cands)} fund-like SEC tickers", flush=True)
    verified = []
    for c in cands:
        n = history_length(c["ticker"])
        if n >= min_days:
            verified.append({**c, "days": n})
            print(f"  keep {c['ticker']:6} {n:5}d  {c['title'][:46]}",
                  flush=True)
    print(f"{len(verified)} with >= {min_days}d history", flush=True)

    missing = [c["ticker"].lower() for c in verified
               if not (DATA / f"{c['ticker'].lower()}-history.csv").exists()]
    if missing and GOGET.exists():
        print(f"goget downloading {len(missing)} symbols...", flush=True)
        subprocess.run([str(GOGET), *missing], cwd=DATA,
                       capture_output=True, timeout=1800)

    results = {}
    for c in verified:
        sym = c["ticker"].lower()
        if not (DATA / f"{sym}-history.csv").exists():
            results[sym] = {"sym": sym, "title": c["title"],
                            "error": "no history after download"}
            continue
        row = search.screen_fund(sym, c["title"], "harvest")
        results[sym] = row
        print(f"{sym:7} {row.get('verdict', row.get('error'))[:60]}",
              flush=True)
    OUT.write_text(json.dumps(results, indent=1, default=str))
    print(f"wrote {OUT}")
    return results


if __name__ == "__main__":
    run()