diff --git a/fundlab/edgar.py b/fundlab/edgar.py index 21d8d26..d233999 100644 --- a/fundlab/edgar.py +++ b/fundlab/edgar.py @@ -375,6 +375,56 @@ def _eval_doc(text: str, name: str, ticker: str | None = None) -> tuple[float, s return None +# section headings that terminate an investment-strategies excerpt +_STRATEGY_STOP = re.compile( + r"\b(?:principal risks|fees and expenses|fund performance|portfolio " + r"turnover|fund fees|purchasing|how to buy|share class)", re.I) + + +def extract_strategy(text: str, limit: int = 2600) -> str | None: + """The fund's investment-strategies section, if present. + + Prospectus supplements amend strategies inside amendment clauses, so the + best 'Investment Strategies' occurrence is chosen: one immediately + followed by actual strategy prose ("seeks", "invests", "Under normal + circumstances") wins.""" + cands = list(re.finditer( + r"(?:[Pp]rincipal )?Investment [Ss]trateg(?:y|ies)" + r"|main investment strategies\??", + text, re.I)) + best, best_score = None, -1 + for m in cands: + seg = text[m.end(): m.end() + 1000] + score = 0 + if re.search(r"\b(seeks|invests|investing|invest \w|" + r"under normal circumstances)\b", seg, re.I): + score += 3 + if "?" in seg[:80]: # a Q&A section heading + score += 2 + if re.search(r"\?\s*\d{1,3}\s", seg[:200]): + score -= 5 # table of contents ("... ? 8 Who ...") + if re.match(r"\s+risk\b", seg, re.I): + score -= 5 # a "...Investment Strategy Risk" subheading + if _STRATEGY_STOP.search(seg[:400]): + score -= 2 + if score > best_score: + best, best_score = m, score + if best is None or best_score < 0: + # no heading: take the prose following the objective sentence + # (Fund Summary boxes run objective -> strategies in sequence) + m = re.search(_SEEK, text[:15000], re.I) + if not m: + return None + seg = text[m.end(): m.end() + limit + 400] + else: + seg = text[best.end(): best.end() + limit + 400] + stop = _STRATEGY_STOP.search(seg) + if stop: + seg = seg[:stop.start()] + out = re.sub(r"\s+", " ", seg).strip(" .") + return out[:limit] if len(out) > 100 else None + + def _name_queries(name: str) -> list[str]: """FTS queries for a fund name: all significant words, then dropping the SHORTEST words first (Yahoo abbreviations like Mgd/Glbl/Macr are diff --git a/fundlab/fundinfo.py b/fundlab/fundinfo.py index 4ff6bc9..b7251ad 100644 --- a/fundlab/fundinfo.py +++ b/fundlab/fundinfo.py @@ -24,7 +24,7 @@ import sys from datetime import date from pathlib import Path -from .edgar import fetch_fund +from .edgar import extract_strategy, fetch_fund, sec_get, to_text DATA_ROOT = Path("~/prog/fin/stocks").expanduser() FUNDS_FILE = Path(__file__).resolve().parent.parent / "funds.json" @@ -53,16 +53,52 @@ def long_name(sym: str, root: Path) -> str | None: def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(prog="fundlab.fundinfo") - ap.add_argument("symbols", nargs="+", help="fund ticker(s), e.g. vtsax vwo") + ap.add_argument("symbols", nargs="*", + help="fund ticker(s), e.g. vtsax vwo (required unless --strategy)") ap.add_argument("--refresh", action="store_true", help="re-fetch from EDGAR even if present in funds.json") ap.add_argument("--no-curate", action="store_true", help="ignore funds_curated.json and use EDGAR for everything") + ap.add_argument("--strategy", action="store_true", + help="fill in the 'strategy' field for funds.json entries " + "that have an EDGAR url (no objective fetching)") ap.add_argument("--data-root", default=str(DATA_ROOT)) ap.add_argument("--max-docs", type=int, default=10, help="max EDGAR documents to try per fund") args = ap.parse_args(argv) + if args.strategy: + if not FUNDS_FILE.exists(): + print("no funds.json", file=sys.stderr) + return 1 + funds = json.loads(FUNDS_FILE.read_text()) + rc = 0 + for sym, rec in funds.items(): + url = rec.get("url") + if not url: + continue + if rec.get("strategy") and not args.refresh: + print(f"= {sym}: strategy cached") + continue + try: + txt = to_text(sec_get(url)) + except Exception as e: + print(f"= {sym}: fetch failed: {type(e).__name__}", file=sys.stderr) + rc = 1 + continue + strat = extract_strategy(txt) + if strat: + rec["strategy"] = strat + print(f"= {sym}: strategy {len(strat)} chars") + else: + print(f"= {sym}: no strategy section found", file=sys.stderr) + FUNDS_FILE.write_text(json.dumps(funds, indent=2)) + print(f"\n{FUNDS_FILE}") + return rc + + if not args.strategy and not args.symbols: + ap.error("symbols are required (or use --strategy)") + root = Path(args.data_root) curated = {} if args.no_curate else load_curated() funds = {} diff --git a/tests/test_fundlab.py b/tests/test_fundlab.py index 330b927..444894e 100644 --- a/tests/test_fundlab.py +++ b/tests/test_fundlab.py @@ -158,6 +158,31 @@ def test_edgar_live() -> None: and res["form"], repr(res)) +def test_strategy() -> None: + print("strategy extraction", flush=True) + doc = ("Fund Cover\nThe Fund seeks total return.\nFees and Expenses " + "of the Fund: see table.\nInvestment Strategies The Fund invests " + "in a diversified portfolio of US and international equity " + "securities and may use derivatives to manage risk. It also " + "invests in fixed income of varying credit quality.\nPrincipal " + "Risks Market Risk. Values may go down.\nMore text.") + s = edgar.extract_strategy(doc) + check("strategy section extracted", + s is not None and "diversified portfolio" in s + and "Market Risk" not in s, repr(s)) + check("no strategy section -> None", + edgar.extract_strategy("Fund Cover\nThe Fund seeks X.\nFees.") is None) + # supplement amendment: the REAL strategy follows a replaced-clause heading + doc2 = ("Supplement\nInvestment Strategies of the section of the " + "Prospectus entitled X is deleted and replaced with the " + "following: Under normal circumstances, the Fund invests in " + "investment grade bonds with maturities under one year and " + "cash equivalents.\nFund Management\nPortfolio Manager: Bob.") + s2 = edgar.extract_strategy(doc2) + check("amendment clause strategy extracted", + s2 is not None and "investment grade bonds" in s2, repr(s2)) + + def test_curated() -> None: print("curated", flush=True) import fundlab.fundinfo as fi @@ -176,6 +201,7 @@ def main() -> int: test_pool() test_text_and_objective() test_classify() + test_strategy() test_curated() test_edgar_live() print(f"\n{PASS} passed, {FAIL} failed")