fundlab: extract strategy sections from EDGAR docs; --strategy CLI mode

- extract_strategy(): finds the 'Principal Investment Strategies' /
  'main investment strategies' section, scores candidates (strategy prose
  +3, Q&A heading +2, TOC -5, risk subheading -5, stop-heading -2),
  truncates at the next section heading; falls back to the prose after
  the objective sentence when no heading exists
- fundinfo --strategy [--refresh]: populates the strategy field of
  funds.json from each fund's EDGAR document
- funds.json now carries objective + strategy for 20 funds (the 9
  curated index funds have no strategy: their objective is the strategy)

fundlab/edgar.py

@@ -375,6 +375,56 @@ def _eval_doc(text: str, name: str, ticker: str | None = None) -> tuple[float, s
     return None
 
 
+# section headings that terminate an investment-strategies excerpt
+_STRATEGY_STOP = re.compile(
+    r"\b(?:principal risks|fees and expenses|fund performance|portfolio "
+    r"turnover|fund fees|purchasing|how to buy|share class)", re.I)
+
+
+def extract_strategy(text: str, limit: int = 2600) -> str | None:
+    """The fund's investment-strategies section, if present.
+
+    Prospectus supplements amend strategies inside amendment clauses, so the
+    best 'Investment Strategies' occurrence is chosen: one immediately
+    followed by actual strategy prose ("seeks", "invests", "Under normal
+    circumstances") wins."""
+    cands = list(re.finditer(
+        r"(?:[Pp]rincipal )?Investment [Ss]trateg(?:y|ies)"
+        r"|main investment strategies\??",
+        text, re.I))
+    best, best_score = None, -1
+    for m in cands:
+        seg = text[m.end(): m.end() + 1000]
+        score = 0
+        if re.search(r"\b(seeks|invests|investing|invest \w|"
+                     r"under normal circumstances)\b", seg, re.I):
+            score += 3
+        if "?" in seg[:80]:            # a Q&A section heading
+            score += 2
+        if re.search(r"\?\s*\d{1,3}\s", seg[:200]):
+            score -= 5                  # table of contents ("... ? 8 Who ...")
+        if re.match(r"\s+risk\b", seg, re.I):
+            score -= 5                  # a "...Investment Strategy Risk" subheading
+        if _STRATEGY_STOP.search(seg[:400]):
+            score -= 2
+        if score > best_score:
+            best, best_score = m, score
+    if best is None or best_score < 0:
+        # no heading: take the prose following the objective sentence
+        # (Fund Summary boxes run objective -> strategies in sequence)
+        m = re.search(_SEEK, text[:15000], re.I)
+        if not m:
+            return None
+        seg = text[m.end(): m.end() + limit + 400]
+    else:
+        seg = text[best.end(): best.end() + limit + 400]
+    stop = _STRATEGY_STOP.search(seg)
+    if stop:
+        seg = seg[:stop.start()]
+    out = re.sub(r"\s+", " ", seg).strip(" .")
+    return out[:limit] if len(out) > 100 else None
+
+
 def _name_queries(name: str) -> list[str]:
     """FTS queries for a fund name: all significant words, then dropping
     the SHORTEST words first (Yahoo abbreviations like Mgd/Glbl/Macr are

fundlab/fundinfo.py

@@ -24,7 +24,7 @@ import sys
 from datetime import date
 from pathlib import Path
 
-from .edgar import fetch_fund
+from .edgar import extract_strategy, fetch_fund, sec_get, to_text
 
 DATA_ROOT = Path("~/prog/fin/stocks").expanduser()
 FUNDS_FILE = Path(__file__).resolve().parent.parent / "funds.json"
@@ -53,16 +53,52 @@ def long_name(sym: str, root: Path) -> str | None:
 
 def main(argv: list[str] | None = None) -> int:
     ap = argparse.ArgumentParser(prog="fundlab.fundinfo")
-    ap.add_argument("symbols", nargs="+", help="fund ticker(s), e.g. vtsax vwo")
+    ap.add_argument("symbols", nargs="*",
+                    help="fund ticker(s), e.g. vtsax vwo (required unless --strategy)")
     ap.add_argument("--refresh", action="store_true",
                     help="re-fetch from EDGAR even if present in funds.json")
     ap.add_argument("--no-curate", action="store_true",
                     help="ignore funds_curated.json and use EDGAR for everything")
+    ap.add_argument("--strategy", action="store_true",
+                    help="fill in the 'strategy' field for funds.json entries "
+                         "that have an EDGAR url (no objective fetching)")
     ap.add_argument("--data-root", default=str(DATA_ROOT))
     ap.add_argument("--max-docs", type=int, default=10,
                     help="max EDGAR documents to try per fund")
     args = ap.parse_args(argv)
 
+    if args.strategy:
+        if not FUNDS_FILE.exists():
+            print("no funds.json", file=sys.stderr)
+            return 1
+        funds = json.loads(FUNDS_FILE.read_text())
+        rc = 0
+        for sym, rec in funds.items():
+            url = rec.get("url")
+            if not url:
+                continue
+            if rec.get("strategy") and not args.refresh:
+                print(f"= {sym}: strategy cached")
+                continue
+            try:
+                txt = to_text(sec_get(url))
+            except Exception as e:
+                print(f"= {sym}: fetch failed: {type(e).__name__}", file=sys.stderr)
+                rc = 1
+                continue
+            strat = extract_strategy(txt)
+            if strat:
+                rec["strategy"] = strat
+                print(f"= {sym}: strategy {len(strat)} chars")
+            else:
+                print(f"= {sym}: no strategy section found", file=sys.stderr)
+        FUNDS_FILE.write_text(json.dumps(funds, indent=2))
+        print(f"\n{FUNDS_FILE}")
+        return rc
+
+    if not args.strategy and not args.symbols:
+        ap.error("symbols are required (or use --strategy)")
+
     root = Path(args.data_root)
     curated = {} if args.no_curate else load_curated()
     funds = {}

tests/test_fundlab.py

@@ -158,6 +158,31 @@ def test_edgar_live() -> None:
           and res["form"], repr(res))
 
 
+def test_strategy() -> None:
+    print("strategy extraction", flush=True)
+    doc = ("Fund Cover\nThe Fund seeks total return.\nFees and Expenses "
+           "of the Fund: see table.\nInvestment Strategies The Fund invests "
+           "in a diversified portfolio of US and international equity "
+           "securities and may use derivatives to manage risk. It also "
+           "invests in fixed income of varying credit quality.\nPrincipal "
+           "Risks Market Risk. Values may go down.\nMore text.")
+    s = edgar.extract_strategy(doc)
+    check("strategy section extracted",
+          s is not None and "diversified portfolio" in s
+          and "Market Risk" not in s, repr(s))
+    check("no strategy section -> None",
+          edgar.extract_strategy("Fund Cover\nThe Fund seeks X.\nFees.") is None)
+    # supplement amendment: the REAL strategy follows a replaced-clause heading
+    doc2 = ("Supplement\nInvestment Strategies of the section of the "
+            "Prospectus entitled X is deleted and replaced with the "
+            "following: Under normal circumstances, the Fund invests in "
+            "investment grade bonds with maturities under one year and "
+            "cash equivalents.\nFund Management\nPortfolio Manager: Bob.")
+    s2 = edgar.extract_strategy(doc2)
+    check("amendment clause strategy extracted",
+          s2 is not None and "investment grade bonds" in s2, repr(s2))
+
+
 def test_curated() -> None:
     print("curated", flush=True)
     import fundlab.fundinfo as fi
@@ -176,6 +201,7 @@ def main() -> int:
     test_pool()
     test_text_and_objective()
     test_classify()
+    test_strategy()
     test_curated()
     test_edgar_live()
     print(f"\n{PASS} passed, {FAIL} failed")