Greg Pomerantz
e9f8dfc462
- edgar.py: SEC FTS + submissions API; strict cover-gate extraction (name in title position or (TICKER) on the cover; underlying-reference names like leveraged wrappers rejected); 4-pass fetch (ticker->CIK filings, name search, annual reports, ticker search); keyword category classifier. Returns None rather than a wrong fund's objective. - fundinfo.py CLI: curated -> cached -> EDGAR resolution into funds.json - funds_curated.json: human-verified objectives for 13 benchmark-pool funds (iShares/Vanguard family-trust classes the scraper can't reach) - tests: 26 checks incl. live EDGAR fetch of VTSAX
187 lines
8.1 KiB
Python
187 lines
8.1 KiB
Python
"""fundlab tests: pool parsing, text extraction, classification, EDGAR live.
|
|
|
|
Run: .venv/bin/python tests/test_fundlab.py
|
|
The live EDGAR test needs network + is rate-limited; it degrades to a skip.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
import urllib.error
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
|
|
|
|
from fundlab import edgar, pool # noqa: E402
|
|
|
|
PASS, FAIL = 0, 0
|
|
|
|
|
|
def check(name: str, cond: bool, extra: str = "") -> None:
|
|
global PASS, FAIL
|
|
if cond:
|
|
PASS += 1
|
|
print(f" ok {name}", flush=True)
|
|
else:
|
|
FAIL += 1
|
|
print(f" FAIL {name} {extra}", flush=True)
|
|
|
|
|
|
# ---------------------------------------------------------------- pool
|
|
def test_pool() -> None:
|
|
print("pool", flush=True)
|
|
entries = pool.load_pool()
|
|
check("pool parses many entries", len(entries) > 90, str(len(entries)))
|
|
bysym = {e.symbol: e for e in entries}
|
|
check("original entries present",
|
|
all(s in bysym for s in ("vbtlx", "ewz", "gld", "ijr", "fxe")))
|
|
check("augmentations present",
|
|
all(s in bysym for s in ("agg", "vt", "vwo", "vnq", "tlt", "bil")))
|
|
check("sections preserved",
|
|
bysym["vbtlx"].section.startswith("Treasury")
|
|
and bysym["agg"].section.startswith("Common index ETFs"))
|
|
check("labels carried", "inflation-protected" in bysym["vipix"].label)
|
|
check("no garbage symbols",
|
|
all(re.fullmatch(r"\S{1,6}", e.symbol) for e in entries))
|
|
|
|
|
|
# ---------------------------------------------------------------- text
|
|
_HTML = (b"<html><body><h3>Fund Summary</h3><p>Investment Objective</p>"
|
|
b"<p>The Fund seeks to track the performance of a benchmark index "
|
|
b"that measures the investment return of large-capitalization "
|
|
b"stocks.</p><p>Fees and Expenses</p></body></html>")
|
|
|
|
|
|
def test_text_and_objective() -> None:
|
|
print("text / objective extraction", flush=True)
|
|
txt = edgar.to_text(_HTML)
|
|
check("to_text strips tags",
|
|
"Fund Summary" in txt and "<p>" not in txt and "Investment Objective" in txt)
|
|
doc = ("Random cover page text. "
|
|
"Vanguard 500 Index Fund\nProspectus\nInvestment Objective\n"
|
|
"The Fund seeks to track the performance of a benchmark index "
|
|
"that measures the investment return of large-capitalization "
|
|
"stocks.\nFees and Expenses\nmore text about Vanguard 500 "
|
|
"Index Fund share classes.")
|
|
obj = edgar.extract_objective(doc, "Vanguard 500 Index Fund")
|
|
check("objective extracted",
|
|
obj is not None and obj.startswith("The Fund seeks to track")
|
|
and "large-capitalization stocks" in obj, repr(obj))
|
|
check("objective is bounded",
|
|
obj is not None and len(obj) < 300, repr(obj and len(obj)))
|
|
doc2 = ("Fund ABC\nThe ETF seeks to invest at least 80% of assets in "
|
|
"US investment grade debt securities of emerging markets.")
|
|
obj2 = edgar.extract_objective(doc2, "Fund ABC")
|
|
check("variant: 'The ETF seeks to ...' on the cover",
|
|
obj2 is not None and "debt securities" in obj2 and "US" in obj2,
|
|
repr(obj2))
|
|
check("no objective -> None",
|
|
edgar.extract_objective("no fund here at all", "Ghost Fund") is None)
|
|
# a document that merely MENTIONS the fund deep in its body (family
|
|
# filings, funds investing in it) must not yield that fund's objective:
|
|
# the name is not on the cover (within the first COVER_GATE chars)
|
|
body = ("Section %d discusses portfolio construction, risk management "
|
|
"and liquidity. ")
|
|
filler = ("XYZ Growth Fund\nProspectus\nInvestment Objective\nThe Fund "
|
|
"seeks to provide total return while generating moderate "
|
|
"current income.\n" +
|
|
"".join(body % i for i in range(40)) +
|
|
"Comparison: similar to Vanguard 500 Index Fund.\n")
|
|
check("mention-only doc -> None",
|
|
edgar.extract_objective(filler, "Vanguard 500 Index Fund") is None)
|
|
# the same doc with the name on the cover IS about the fund
|
|
oncover = ("Vanguard 500 Index Fund\nProspectus\nInvestment Objective\n"
|
|
"The Fund seeks to track large-capitalization stocks.\n" +
|
|
"".join(body % i for i in range(40)))
|
|
check("name on cover -> extracted",
|
|
(edgar.extract_objective(oncover, "Vanguard 500 Index Fund")
|
|
or "").startswith("The Fund seeks to track"))
|
|
# a leveraged ETF's cover cites the underlying fund's name; the name
|
|
# is a reference, not a title -> not our fund's document
|
|
doc4 = ("Acme 2X Leveraged QQQ Daily ETF\nProspectus\nThe Fund seeks\n"
|
|
"daily investment results that are 200% of the performance of\n"
|
|
"the Invesco QQQ Trust. The Fund seeks to achieve daily\n"
|
|
"investment results equal to two times the performance of the\n"
|
|
"index, before fees and expenses.")
|
|
check("underlying reference on cover -> None",
|
|
edgar.extract_objective(doc4, "Invesco QQQ Trust") is None)
|
|
|
|
|
|
# ---------------------------------------------------------------- classify
|
|
def test_classify() -> None:
|
|
print("category classification", flush=True)
|
|
c = edgar.classify_category
|
|
check("US index fund -> equity",
|
|
c("The Fund seeks to track large-capitalization stocks.",
|
|
"Vanguard 500 Index Fund") == "equity")
|
|
check("aggregate bond -> fixed_income",
|
|
c("The Fund seeks to track the performance of an index of US "
|
|
"investment grade debt securities.", "iShares Core US Aggregate Bond ETF")
|
|
== "fixed_income")
|
|
check("balanced -> mixed",
|
|
c("The Fund invests in a combination of equity and debt "
|
|
"securities.", "Example Balanced Fund") == "mixed")
|
|
check("gold -> alternatives",
|
|
c("The Fund seeks to track the price of gold bullion.",
|
|
"SPDR Gold Shares") == "alternatives")
|
|
check("money market -> money_market",
|
|
c("The Fund seeks to maintain stability of principal. It "
|
|
"invests in money market instruments.", "XX Money Market Fund")
|
|
== "money_market")
|
|
check("unknown -> other", c("The Fund does unusual things.", "Mystery") == "other")
|
|
|
|
|
|
# ---------------------------------------------------------------- edgar live
|
|
def test_edgar_live() -> None:
|
|
print("EDGAR live (network)", flush=True)
|
|
try:
|
|
name = None
|
|
meta = json.loads((pathlib.Path("~/prog/fin/stocks").expanduser()
|
|
/ "vtsax.json").read_text())["chart"]["result"][0]["meta"]
|
|
name = meta.get("longName")
|
|
if not name:
|
|
raise urllib.error.URLError("no local vtsax.json")
|
|
res = edgar.fetch_fund(name, ticker="vtsax", max_docs=8)
|
|
except Exception as e:
|
|
print(f" SKIP (no network / data): {type(e).__name__}: {e}")
|
|
return
|
|
check("vtsax objective fetched",
|
|
res is not None and "seeks to" in res["objective"]
|
|
and "stock market" in res["objective"].lower(),
|
|
repr(res and res.get("objective")))
|
|
check("vtsax classified equity",
|
|
res is not None and res["category"] == "equity",
|
|
repr(res and res.get("category")))
|
|
check("provenance recorded",
|
|
res is not None and res["url"].startswith("https://www.sec.gov")
|
|
and res["form"], repr(res))
|
|
|
|
|
|
def test_curated() -> None:
|
|
print("curated", flush=True)
|
|
import fundlab.fundinfo as fi
|
|
cur = fi.load_curated()
|
|
check("curated file loads", len(cur) >= 10, str(len(cur)))
|
|
check("curated entries well-formed",
|
|
all(set(v) >= {"objective", "category"} and
|
|
v["category"] in ("equity", "fixed_income", "mixed",
|
|
"alternatives", "money_market", "other")
|
|
for v in cur.values()))
|
|
check("curated objectives look right",
|
|
all("seeks" in v["objective"].lower() for v in cur.values()))
|
|
|
|
|
|
def main() -> int:
|
|
test_pool()
|
|
test_text_and_objective()
|
|
test_classify()
|
|
test_curated()
|
|
test_edgar_live()
|
|
print(f"\n{PASS} passed, {FAIL} failed")
|
|
return 1 if FAIL else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|