fundlab: extract strategy sections from EDGAR docs; --strategy CLI mode
- extract_strategy(): finds the 'Principal Investment Strategies' / 'main investment strategies' section, scores candidates (strategy prose +3, Q&A heading +2, TOC -5, risk subheading -5, stop-heading -2), truncates at the next section heading; falls back to the prose after the objective sentence when no heading exists - fundinfo --strategy [--refresh]: populates the strategy field of funds.json from each fund's EDGAR document - funds.json now carries objective + strategy for 20 funds (the 9 curated index funds have no strategy: their objective is the strategy)
This commit is contained in:
parent
5c6d95a8b4
commit
7b27920a77
|
|
@ -375,6 +375,56 @@ def _eval_doc(text: str, name: str, ticker: str | None = None) -> tuple[float, s
|
|||
return None
|
||||
|
||||
|
||||
# section headings that terminate an investment-strategies excerpt
|
||||
_STRATEGY_STOP = re.compile(
|
||||
r"\b(?:principal risks|fees and expenses|fund performance|portfolio "
|
||||
r"turnover|fund fees|purchasing|how to buy|share class)", re.I)
|
||||
|
||||
|
||||
def extract_strategy(text: str, limit: int = 2600) -> str | None:
|
||||
"""The fund's investment-strategies section, if present.
|
||||
|
||||
Prospectus supplements amend strategies inside amendment clauses, so the
|
||||
best 'Investment Strategies' occurrence is chosen: one immediately
|
||||
followed by actual strategy prose ("seeks", "invests", "Under normal
|
||||
circumstances") wins."""
|
||||
cands = list(re.finditer(
|
||||
r"(?:[Pp]rincipal )?Investment [Ss]trateg(?:y|ies)"
|
||||
r"|main investment strategies\??",
|
||||
text, re.I))
|
||||
best, best_score = None, -1
|
||||
for m in cands:
|
||||
seg = text[m.end(): m.end() + 1000]
|
||||
score = 0
|
||||
if re.search(r"\b(seeks|invests|investing|invest \w|"
|
||||
r"under normal circumstances)\b", seg, re.I):
|
||||
score += 3
|
||||
if "?" in seg[:80]: # a Q&A section heading
|
||||
score += 2
|
||||
if re.search(r"\?\s*\d{1,3}\s", seg[:200]):
|
||||
score -= 5 # table of contents ("... ? 8 Who ...")
|
||||
if re.match(r"\s+risk\b", seg, re.I):
|
||||
score -= 5 # a "...Investment Strategy Risk" subheading
|
||||
if _STRATEGY_STOP.search(seg[:400]):
|
||||
score -= 2
|
||||
if score > best_score:
|
||||
best, best_score = m, score
|
||||
if best is None or best_score < 0:
|
||||
# no heading: take the prose following the objective sentence
|
||||
# (Fund Summary boxes run objective -> strategies in sequence)
|
||||
m = re.search(_SEEK, text[:15000], re.I)
|
||||
if not m:
|
||||
return None
|
||||
seg = text[m.end(): m.end() + limit + 400]
|
||||
else:
|
||||
seg = text[best.end(): best.end() + limit + 400]
|
||||
stop = _STRATEGY_STOP.search(seg)
|
||||
if stop:
|
||||
seg = seg[:stop.start()]
|
||||
out = re.sub(r"\s+", " ", seg).strip(" .")
|
||||
return out[:limit] if len(out) > 100 else None
|
||||
|
||||
|
||||
def _name_queries(name: str) -> list[str]:
|
||||
"""FTS queries for a fund name: all significant words, then dropping
|
||||
the SHORTEST words first (Yahoo abbreviations like Mgd/Glbl/Macr are
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ import sys
|
|||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from .edgar import fetch_fund
|
||||
from .edgar import extract_strategy, fetch_fund, sec_get, to_text
|
||||
|
||||
DATA_ROOT = Path("~/prog/fin/stocks").expanduser()
|
||||
FUNDS_FILE = Path(__file__).resolve().parent.parent / "funds.json"
|
||||
|
|
@ -53,16 +53,52 @@ def long_name(sym: str, root: Path) -> str | None:
|
|||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
ap = argparse.ArgumentParser(prog="fundlab.fundinfo")
|
||||
ap.add_argument("symbols", nargs="+", help="fund ticker(s), e.g. vtsax vwo")
|
||||
ap.add_argument("symbols", nargs="*",
|
||||
help="fund ticker(s), e.g. vtsax vwo (required unless --strategy)")
|
||||
ap.add_argument("--refresh", action="store_true",
|
||||
help="re-fetch from EDGAR even if present in funds.json")
|
||||
ap.add_argument("--no-curate", action="store_true",
|
||||
help="ignore funds_curated.json and use EDGAR for everything")
|
||||
ap.add_argument("--strategy", action="store_true",
|
||||
help="fill in the 'strategy' field for funds.json entries "
|
||||
"that have an EDGAR url (no objective fetching)")
|
||||
ap.add_argument("--data-root", default=str(DATA_ROOT))
|
||||
ap.add_argument("--max-docs", type=int, default=10,
|
||||
help="max EDGAR documents to try per fund")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
if args.strategy:
|
||||
if not FUNDS_FILE.exists():
|
||||
print("no funds.json", file=sys.stderr)
|
||||
return 1
|
||||
funds = json.loads(FUNDS_FILE.read_text())
|
||||
rc = 0
|
||||
for sym, rec in funds.items():
|
||||
url = rec.get("url")
|
||||
if not url:
|
||||
continue
|
||||
if rec.get("strategy") and not args.refresh:
|
||||
print(f"= {sym}: strategy cached")
|
||||
continue
|
||||
try:
|
||||
txt = to_text(sec_get(url))
|
||||
except Exception as e:
|
||||
print(f"= {sym}: fetch failed: {type(e).__name__}", file=sys.stderr)
|
||||
rc = 1
|
||||
continue
|
||||
strat = extract_strategy(txt)
|
||||
if strat:
|
||||
rec["strategy"] = strat
|
||||
print(f"= {sym}: strategy {len(strat)} chars")
|
||||
else:
|
||||
print(f"= {sym}: no strategy section found", file=sys.stderr)
|
||||
FUNDS_FILE.write_text(json.dumps(funds, indent=2))
|
||||
print(f"\n{FUNDS_FILE}")
|
||||
return rc
|
||||
|
||||
if not args.strategy and not args.symbols:
|
||||
ap.error("symbols are required (or use --strategy)")
|
||||
|
||||
root = Path(args.data_root)
|
||||
curated = {} if args.no_curate else load_curated()
|
||||
funds = {}
|
||||
|
|
|
|||
|
|
@ -158,6 +158,31 @@ def test_edgar_live() -> None:
|
|||
and res["form"], repr(res))
|
||||
|
||||
|
||||
def test_strategy() -> None:
|
||||
print("strategy extraction", flush=True)
|
||||
doc = ("Fund Cover\nThe Fund seeks total return.\nFees and Expenses "
|
||||
"of the Fund: see table.\nInvestment Strategies The Fund invests "
|
||||
"in a diversified portfolio of US and international equity "
|
||||
"securities and may use derivatives to manage risk. It also "
|
||||
"invests in fixed income of varying credit quality.\nPrincipal "
|
||||
"Risks Market Risk. Values may go down.\nMore text.")
|
||||
s = edgar.extract_strategy(doc)
|
||||
check("strategy section extracted",
|
||||
s is not None and "diversified portfolio" in s
|
||||
and "Market Risk" not in s, repr(s))
|
||||
check("no strategy section -> None",
|
||||
edgar.extract_strategy("Fund Cover\nThe Fund seeks X.\nFees.") is None)
|
||||
# supplement amendment: the REAL strategy follows a replaced-clause heading
|
||||
doc2 = ("Supplement\nInvestment Strategies of the section of the "
|
||||
"Prospectus entitled X is deleted and replaced with the "
|
||||
"following: Under normal circumstances, the Fund invests in "
|
||||
"investment grade bonds with maturities under one year and "
|
||||
"cash equivalents.\nFund Management\nPortfolio Manager: Bob.")
|
||||
s2 = edgar.extract_strategy(doc2)
|
||||
check("amendment clause strategy extracted",
|
||||
s2 is not None and "investment grade bonds" in s2, repr(s2))
|
||||
|
||||
|
||||
def test_curated() -> None:
|
||||
print("curated", flush=True)
|
||||
import fundlab.fundinfo as fi
|
||||
|
|
@ -176,6 +201,7 @@ def main() -> int:
|
|||
test_pool()
|
||||
test_text_and_objective()
|
||||
test_classify()
|
||||
test_strategy()
|
||||
test_curated()
|
||||
test_edgar_live()
|
||||
print(f"\n{PASS} passed, {FAIL} failed")
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user