gmp/f
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fundlab: EDGAR investment-objective fetcher + curated fallback

Browse Source 
- edgar.py: SEC FTS + submissions API; strict cover-gate extraction
  (name in title position or (TICKER) on the cover; underlying-reference
  names like leveraged wrappers rejected); 4-pass fetch (ticker->CIK
  filings, name search, annual reports, ticker search); keyword category
  classifier. Returns None rather than a wrong fund's objective.
- fundinfo.py CLI: curated -> cached -> EDGAR resolution into funds.json
- funds_curated.json: human-verified objectives for 13 benchmark-pool
  funds (iShares/Vanguard family-trust classes the scraper can't reach)
- tests: 26 checks incl. live EDGAR fetch of VTSAX
This commit is contained in:
Greg Pomerantz 2026-08-25 18:15:48 -04:00
parent 4f36bc7aea
commit e9f8dfc462
7 changed files with 962 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fundlab/__init__.py Normal file
View File

@ -0,0 +1 @@
"""fundlab: automated fund analysis (EDGAR info, benchmark pool, factor tests)."""

371
fundlab/edgar.py Normal file
View File

@ -0,0 +1,371 @@
"""EDGAR fetcher: fund investment objective + category from SEC filings.
Pipeline: fund name -> EDGAR full-text search over prospectus/annual-report
forms (N-1A, 485APOS, 8-A12B, S-1, N-CSR, N-CSRS) -> fetch the document ->
extract the "Investment Objective" sentence -> classify the fund category.
SEC usage policy: identify yourself via USER_AGENT (name + contact), stay
well under 10 requests/second (we insert REQUEST_DELAY between calls).
"""
from __future__ import annotations
import json
import re
import time
import urllib.error
import urllib.parse
import urllib.request
USER_AGENT = "fundlab (personal research) gmp@wow.st"
REQUEST_DELAY = 0.15 # seconds between SEC requests
MAX_DOC_BYTES = 60 * 2 ** 20 # skip documents bigger than this
# forms whose primary document contains a fund's investment objective.
# Per-fund registration statements are tried FIRST — they hold one fund's
# objective in a clean "Fund Summary" box. Family annual reports (N-CSR/
# N-CSRS) cover hundreds of funds across several sub-documents, so the
# target fund's section may be in a part the search doesn't surface.
# forms whose documents contain a fund's investment objective. Per-fund
# registration statements are tried FIRST — they hold one fund's objective
# in a clean "Fund Summary" box. Annual reports (N-CSR/N-CSRS) are a
# fallback; family filings covering many funds are filtered out by the
# cover gate (see _on_cover).
PROSPECTUS_FORMS = "485APOS,485BPOS,497,497K,497F,N-1A,8-A12B,S-1,FWP,424B2,424B3"
ANNUAL_FORMS = "N-CSR,N-CSRS"
# how far into a document the fund's identifier may sit and still count as
# being ON THE COVER (the document is about this fund). Per-fund
# prospectus covers fit in ~1200 chars; title position only (~400) for
# annual reports and ticker-based lookups.
COVER_GATE = 1200
TITLE_GATE = 400
_last_request = 0.0
def sec_get(url: str, timeout: int = 120, retries: int = 3) -> bytes:
"""GET with SEC user agent, politeness delay and retry/backoff."""
global _last_request
for attempt in range(retries):
wait = REQUEST_DELAY - (time.monotonic() - _last_request)
if wait > 0:
time.sleep(wait)
_last_request = time.monotonic()
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.read()
except urllib.error.HTTPError as e:
if e.code == 404:
raise
if e.code in (429, 500, 502, 503) and attempt + 1 < retries:
time.sleep(2.0 * (attempt + 1))
continue
raise
except (urllib.error.URLError, TimeoutError) as e:
if attempt + 1 < retries:
time.sleep(2.0 * (attempt + 1))
continue
raise
raise RuntimeError("unreachable")
def to_text(html: bytes) -> str:
"""Crude but adequate HTML -> text: block ends become newlines, tags go."""
t = re.sub(rb"<\s*(br|/p|/div|/tr|/h[1-6])[^>]*>", b"\n", html, flags=re.I)
t = re.sub(rb"<[^>]+>", b" ", t)
t = re.sub(rb"&#?\w+;", b" ", t)
t = t.decode("latin-1", "ignore")
t = re.sub(r"[ \t]+", " ", t)
t = re.sub(r"\n\s*", "\n", t)
# tag stripping eats apostrophes ("Fund s board") — fix the common ones
t = re.sub(r"\b(Fund|Trust|ETF) s\b", r"\1's", t)
# abbreviations whose dots would break sentence-level matching
for a, b in (("U.S.", "US"), ("U.K.", "UK"), ("E.U.", "EU"),
("e.g.", "eg"), ("i.e.", "ie"), ("No.", "No")):
t = t.replace(a, b)
return t
def ticker_to_company(ticker: str) -> tuple[str, str] | None:
"""ticker -> (cik, conformed company name) via browse-edgar (Atom output)."""
url = ("https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany"
f"&CIK={urllib.parse.quote(ticker.upper())}&type=&dateb="
"&owner=include&count=5&output=atom")
xml = sec_get(url, timeout=30).decode("latin-1", "ignore")
cik = re.search(r"<cik>(\d+)</cik>", xml)
name = re.search(r"<conformed-name>([^<]+)</conformed-name>", xml)
if not cik:
return None
return cik.group(1), (name.group(1) if name else "")
def cik_recent_filings(cik: str, types: str, count: int = 6) -> list[dict]:
"""Recent filings of a registrant (newest first) via the submissions API."""
url = f"https://data.sec.gov/submissions/CIK{int(cik):010d}.json"
d = json.loads(sec_get(url, timeout=60))
f = d.get("filings", {}).get("recent", {})
forms = f.get("form", [])
out = []
for i, form in enumerate(forms):
if form not in types.split(","):
continue
out.append({
"accession": f["accessionNumber"][i],
"form": form,
"filed": f.get("filingDate", [""] * len(forms))[i],
"doc": (f.get("primaryDocument") or [""] * len(forms))[i],
})
if len(out) >= count:
break
return out
def fts_search(query: str, forms: str | None = None, size: int = 20) -> list[dict]:
"""EDGAR full-text search.
Hits are sorted by relevance, then date: a fund's own prospectus ranks
above documents that merely mention the ticker (funds investing in it,
family fund lists). Relevance alone is not enough — section attribution
in extract_objective is the real guard — but ordering by it means the
first match is usually the right fund's.
"""
params = {"q": query, "forms": forms or "", "size": str(size)}
url = f"https://efts.sec.gov/LATEST/search-index?{urllib.parse.urlencode(params)}"
d = json.loads(sec_get(url, timeout=60))
hits = d.get("hits", {}).get("hits", [])
out, seen = [], set()
for h in hits:
s = h.get("_source", {})
key = (s.get("adsh", ""), h.get("_id", "").split(":", 1)[-1])
if key in seen:
continue
seen.add(key)
out.append({
"score": h.get("_score", 0.0),
"accession": s.get("adsh", ""),
"filename": h.get("_id", "").split(":", 1)[-1],
"cik": (s.get("ciks") or [""])[0],
"form": s.get("form", ""),
"file_date": s.get("file_date", ""),
})
out.sort(key=lambda x: (x["score"], x["file_date"]), reverse=True)
return out
def doc_url(cik: str, accession: str, filename: str) -> str:
return (f"https://www.sec.gov/Archives/edgar/data/{int(cik)}"
f"/{accession.replace('-', '')}/{filename}")
_SEEK = (r"((?:the|its) (?:fund|trust|etf|portfolio)\s*\)?[^\.\n]{0,120}?"
r"seeks (?:to |investment results |investment objective)?"
r"[^.]{10,400}\.)")
def _name_re(name: str) -> str:
"""Pattern for a fund name; to_text() turns 'U.S.' into 'US', so the
dots are optional."""
return re.escape(name).replace(r"U\.S\.", r"U\.?S\.?")
# a name preceded by one of these is a REFERENCE to another fund (a
# leveraged ETF's cover says "200% of the performance of the Invesco
# QQQ Trust"), not the title of the document
_REF_BEFORE = re.compile(
r"(?:performance\s+of\s+|underlying\s+|index\s+of\s+|versus\s+|"
r"than\s+|against\s+|of\s+the\s+|the\s+)$", re.I)
def _name_titlelike(text: str, name: str, limit: int) -> bool:
"""True when the name appears in title position on the cover — not as
an underlying reference (see _REF_BEFORE)."""
for m in re.finditer(_name_re(name), text[:limit], re.I):
before = text[max(0, m.start() - 40):m.start()]
if not _REF_BEFORE.search(before):
return True
return False
def _on_cover(text: str, name: str | None, ticker: str | None,
limit: int = COVER_GATE) -> str | None:
"""Cover gate: is this document ABOUT the fund?
A fund's own registration statement puts its name (in title position)
or its ticker in parentheses on the cover page, within the first
~1200 characters. A document that merely mentions the fund elsewhere
(family fund lists, funds investing in it, exhibits) does not. Returns
'name' / 'ticker' / None.
"""
head = text[:limit]
if name and _name_titlelike(text, name, limit):
return "name"
if ticker and re.search(r"\(\s*" + re.escape(ticker.upper()) + r"\s*\)",
head, re.I):
return "ticker"
return None
def extract_objective(text: str, name: str, ticker: str | None = None,
gate: int = COVER_GATE) -> str | None:
"""Extract the fund's investment objective sentence from a document
that is about the fund (see _on_cover).
Returns None when the document is not about this fund (cover gate
fails) or no objective sentence is found — never another fund's.
"""
if not _on_cover(text, name, ticker, gate):
return None
heads = [m.start() for m in re.finditer(r"investment objective", text, re.I)]
if heads:
m = re.search(_SEEK, text[heads[0]: heads[0] + 2500], re.I)
if m:
return _clean(m.group(1))
m = re.search(_SEEK, text[:15000], re.I)
return _clean(m.group(1)) if m else None
def _clean(s: str) -> str:
# captures like "(the Fund) seeks to ..." leave a stray ')'
s = re.sub(r"^\)\s*", "", s)
s = re.sub(r"^((?:the|its) (?:fund|trust|etf|portfolio)"
r"(?: or (?:fund|trust|etf|portfolio))?)\s*\)\s*",
r"\1 ", s, flags=re.I)
s = re.sub(r"\s+", " ", s).strip()
if s[:1].islower():
s = s[0].upper() + s[1:]
return s
def classify_category(objective: str, name: str) -> str:
"""equity / fixed_income / mixed / alternatives / money_market / other.
Keyword rules over objective + fund name; deliberately transparent and
conservative (unknown -> 'other')."""
t = f"{objective} {name}".lower()
if re.search(r"money market", t):
return "money_market"
equity = bool(re.search(
r"equit|stocks?|common stock|capitalization|nasdaq.?100|s&p ?500 "
r"index|total stock market|russell ?2000", t))
bond = bool(re.search(
r"debt securit|bonds?\b|fixed.income|maturit|treasur|credit|yield|"
r"mortgage|interest.rate", t))
alt = bool(re.search(
r"commodit|derivatives|real estate|currenc|private (?:equity|credit)|"
r"hedge|bullion|precious metal|\bgold\b|\bsilver\b|\bcopper\b|"
r"\bnatural gas\b|\bcrude oil\b|\bwheat\b|\bcorn\b", t))
if equity and bond:
return "mixed"
if bond:
return "fixed_income"
if equity:
return "equity"
if alt:
return "alternatives"
return "other"
def accession_docs(cik: str, accession: str) -> list[str]:
"""All .htm files of a filing (family reports are split into parts)."""
url = (f"https://www.sec.gov/Archives/edgar/data/{int(cik)}"
f"/{accession.replace('-', '')}/index.json")
try:
d = json.loads(sec_get(url, timeout=60))
except Exception:
return []
return [it["name"] for it in d.get("directory", {}).get("item", [])
if it["name"].lower().endswith((".htm", ".html"))]
def _try_docs(hits: list[dict], name: str, ticker: str | None,
max_docs: int, gate: int) -> dict | None:
"""Fetch FTS hits (relevance order) and return the first document that
passes the cover gate and yields an objective sentence."""
fetched = 0
for h in hits:
if fetched >= max_docs:
break
# annual reports: the fund's part may be a sibling document of the
# same filing, so enumerate the accession's documents
names = [h["filename"]]
if h["form"] in ANNUAL_FORMS.split(","):
names += [n for n in accession_docs(h["cik"], h["accession"])
if n != h["filename"]]
for fn in names:
if fetched >= max_docs:
break
try:
raw = sec_get(doc_url(h["cik"], h["accession"], fn))
except urllib.error.HTTPError:
continue
fetched += 1
if len(raw) > MAX_DOC_BYTES:
continue
obj = extract_objective(to_text(raw), name, ticker, gate)
if obj:
return {
"objective": obj,
"category": classify_category(obj, name),
"form": h["form"],
"file_date": h["file_date"],
"url": doc_url(h["cik"], h["accession"], fn),
}
return None
def fetch_fund(fund_name: str, ticker: str | None = None,
max_docs: int = 8) -> dict | None:
"""Full pipeline for one fund. Returns metadata dict or None.
Passes, most direct first:
0. ticker -> registrant CIK (standalone trusts like Invesco QQQ
Trust / iShares Silver Trust): their most recent prospectus
filings, cover-gated;
1. fund-name search over prospectus forms — name in title position
or (TICKER) on the cover (COVER_GATE);
2. fund-name search over annual reports — name in title position
(TITLE_GATE);
3. ticker search over prospectus forms — (TICKER) in title position
(TITLE_GATE); catches funds whose registered name changed over
time.
Returns None rather than a wrong fund's objective.
"""
tk = ticker.upper() if ticker else None
if tk:
comp = ticker_to_company(tk)
if comp:
for fl in cik_recent_filings(comp[0], PROSPECTUS_FORMS, count=15):
if not fl["doc"]:
continue
try:
raw = sec_get(doc_url(comp[0], fl["accession"], fl["doc"]))
except urllib.error.HTTPError:
continue
if len(raw) > MAX_DOC_BYTES:
continue
obj = extract_objective(to_text(raw), fund_name, tk, COVER_GATE)
if obj:
return {
"objective": obj,
"category": classify_category(obj, fund_name),
"form": fl["form"],
"file_date": fl["filed"],
"url": doc_url(comp[0], fl["accession"], fl["doc"]),
}
q = f'"{fund_name}" "seeks"'
res = _try_docs(fts_search(q, PROSPECTUS_FORMS, size=100),
fund_name, tk, max_docs, COVER_GATE)
if res:
return res
res = _try_docs(fts_search(q, ANNUAL_FORMS, size=100),
fund_name, tk, max_docs, TITLE_GATE)
if res:
return res
if tk:
res = _try_docs(fts_search(f'"{tk}" "seeks"', PROSPECTUS_FORMS, size=100),
fund_name, tk, max_docs, TITLE_GATE)
if res:
return res
return None

130
fundlab/fundinfo.py Normal file
View File

@ -0,0 +1,130 @@
"""Resolve fund investment objectives + categories.
Usage:
python -m fundlab.fundinfo SYMBOL [SYMBOL ...] [--refresh] [--no-curate] [--max-docs N]
Objective sources, in priority order:
1. funds_curated.json (human-verified, in this package) — used unless
--no-curate. Authoritative for the well-known benchmark funds.
2. SEC EDGAR — the fund's longName (from the Yahoo chart JSON in the
data root, default ~/prog/fin/stocks) is searched over prospectus /
annual-report forms; the objective is extracted from the fund's own
filing and the fund classified.
Everything is merged into funds.json next to the repository root. Cached
EDGAR entries are reused unless --refresh is given. Be polite to the SEC:
one fund takes ~5-15 requests; run a handful at a time.
"""
from __future__ import annotations
import argparse
import json
import sys
from datetime import date
from pathlib import Path
from .edgar import fetch_fund
DATA_ROOT = Path("~/prog/fin/stocks").expanduser()
FUNDS_FILE = Path(__file__).resolve().parent.parent / "funds.json"
CURATED_FILE = Path(__file__).resolve().parent / "funds_curated.json"
def load_curated() -> dict:
if CURATED_FILE.exists():
try:
return json.loads(CURATED_FILE.read_text())
except Exception:
pass
return {}
def long_name(sym: str, root: Path) -> str | None:
f = root / f"{sym}.json"
if not f.exists():
return None
try:
meta = json.loads(f.read_text())["chart"]["result"][0]["meta"]
return meta.get("longName") or meta.get("shortName")
except Exception:
return None
def main(argv: list[str] | None = None) -> int:
ap = argparse.ArgumentParser(prog="fundlab.fundinfo")
ap.add_argument("symbols", nargs="+", help="fund ticker(s), e.g. vtsax vwo")
ap.add_argument("--refresh", action="store_true",
help="re-fetch from EDGAR even if present in funds.json")
ap.add_argument("--no-curate", action="store_true",
help="ignore funds_curated.json and use EDGAR for everything")
ap.add_argument("--data-root", default=str(DATA_ROOT))
ap.add_argument("--max-docs", type=int, default=10,
help="max EDGAR documents to try per fund")
args = ap.parse_args(argv)
root = Path(args.data_root)
curated = {} if args.no_curate else load_curated()
funds = {}
if FUNDS_FILE.exists():
try:
funds = json.loads(FUNDS_FILE.read_text())
except Exception:
pass
rc = 0
for sym in args.symbols:
sym = sym.lower()
# 1) curated (authoritative) — always wins over cache/EDGAR
if sym in curated:
c = curated[sym]
funds[sym] = {
"name": c.get("name", ""),
"objective": c["objective"],
"category": c["category"],
"source": "curated",
}
print(f"= {sym}: curated ({c['category']})")
continue
# 2) cached EDGAR result
if not args.refresh and sym in funds and funds[sym].get("objective"):
print(f"= {sym}: cached ({funds[sym].get('category')})")
continue
# 3) EDGAR
name = long_name(sym, root)
if not name:
print(f"= {sym}: NO chart JSON in {root} — cannot identify fund",
file=sys.stderr)
rc = 1
continue
print(f"= {sym}: fetching '{name}' from EDGAR ...", flush=True)
try:
res = fetch_fund(name, ticker=sym, max_docs=args.max_docs)
except Exception as e:
print(f" ERROR: {type(e).__name__}: {e}", file=sys.stderr)
rc = 1
continue
if not res:
print(f" no investment objective found in EDGAR", file=sys.stderr)
rc = 1
continue
funds[sym] = {
"name": name,
"objective": res["objective"],
"category": res["category"],
"form": res["form"],
"file_date": res["file_date"],
"url": res["url"],
"fetched": date.today().isoformat(),
"source": "edgar",
}
print(f" category: {res['category']} [{res['form']} {res['file_date']}]")
print(f" objective: {res['objective']}")
FUNDS_FILE.write_text(json.dumps(funds, indent=2))
print(f"\n{FUNDS_FILE}")
return rc
if __name__ == "__main__":
sys.exit(main())

67
fundlab/funds_curated.json Normal file
View File

@ -0,0 +1,67 @@
{
"agg": {
"name": "iShares Core U.S. Aggregate Bond ETF",
"objective": "The Fund seeks to track the investment results of an index composed of the total US investment-grade bond market.",
"category": "fixed_income"
},
"bnd": {
"name": "Vanguard Total Bond Market ETF",
"objective": "The Fund seeks to track the performance of a broad, market-weighted bond index.",
"category": "fixed_income"
},
"efa": {
"name": "iShares MSCI EAFE ETF",
"objective": "The Fund seeks to track the investment performance of an index composed of large- and mid-cap stocks from developed markets outside the US.",
"category": "equity"
},
"gld": {
"name": "SPDR Gold Shares",
"objective": "The Trust seeks to reflect, before fees and expenses, the performance of the price of gold bullion.",
"category": "alternatives"
},
"iwm": {
"name": "iShares Russell 2000 ETF",
"objective": "The Fund seeks through investments in the underlying index to provide investment results that, before fees and expenses, generally correspond to the price and yield performance of an index of small-capitalization US stocks.",
"category": "equity"
},
"qqq": {
"name": "Invesco QQQ Trust, Series 1",
"objective": "The Fund seeks to track, before fees and expenses, the investment results of the Nasdaq-100 Index.",
"category": "equity"
},
"schd": {
"name": "Schwab U.S. Dividend Equity ETF",
"objective": "The Fund seeks to track the performance of the Dow Jones US Dividend 100 Index, before fees and expenses.",
"category": "equity"
},
"shv": {
"name": "iShares 0-1 Year Treasury Bond ETF",
"objective": "The Fund seeks to track the investment results of the ICE Short US Treasury Securities Index, which measures the performance of public obligations of the US Treasury that have a remaining maturity of less than or equal to one year.",
"category": "fixed_income"
},
"slv": {
"name": "iShares Silver Trust",
"objective": "The Trust seeks to reflect, before fees and expenses, generally the performance of the price of silver, less the Trust's expenses.",
"category": "alternatives"
},
"vea": {
"name": "Vanguard FTSE Developed Markets Index Fund ETF Shares",
"objective": "The Fund seeks to track, before fees and expenses, the investment performance of an index representative of the large- and mid-capitalization stocks of companies in developed markets.",
"category": "equity"
},
"vtsax": {
"name": "Vanguard Total Stock Market Index Fund",
"objective": "The Fund seeks to track the performance of a benchmark index that measures the investment return of the overall stock market.",
"category": "equity"
},
"vti": {
"name": "Vanguard Total Stock Market Index Fund ETF Shares",
"objective": "The Fund seeks to track the performance of a benchmark index that measures the investment return of the overall stock market.",
"category": "equity"
},
"vwo": {
"name": "Vanguard FTSE Emerging Markets Index Fund ETF Shares",
"objective": "The Fund seeks to track, before fees and expenses, the investment performance of an index representative of stocks of companies in emerging markets.",
"category": "equity"
}
}

49
fundlab/pool.py Normal file
View File

@ -0,0 +1,49 @@
"""Benchmark discovery pool.
A copy of ~/prog/fin/benchmarks.txt (the user's curated benchmark universe),
augmented with a few common index ETFs that exist in the data universe.
The ORIGINAL file outside this repository is never modified.
Format: section header lines (ending in ':' or free-standing), then
'symbol<tab or 2+ spaces>label' lines.
"""
from __future__ import annotations
import re
from dataclasses import dataclass
from pathlib import Path
POOL_FILE = Path(__file__).parent / "pool" / "benchmarks.txt"
_ENTRY = re.compile(r"^(\S{1,6})[\t ]{2,}(.+)$")
@dataclass(frozen=True)
class PoolEntry:
section: str
symbol: str # lowercase
label: str
def load_pool(path: Path = POOL_FILE) -> list[PoolEntry]:
entries: list[PoolEntry] = []
section = ""
for line in path.read_text().splitlines():
line = line.rstrip()
if not line.strip():
continue
m = _ENTRY.match(line)
if m:
entries.append(PoolEntry(section, m.group(1).lower(),
m.group(2).strip()))
else:
section = line.strip().rstrip(":")
return entries
def pool_labels(entries: list[PoolEntry] | None = None) -> dict[str, str]:
"""symbol -> 'section: label' (for keyword matching and display)."""
if entries is None:
entries = load_pool()
return {e.symbol: f"{e.section}: {e.label}" for e in entries if e.section}

158
fundlab/pool/benchmarks.txt Normal file
View File

@ -0,0 +1,158 @@
Treasury/Agency:
vfijx gnma
vipix inflation-protected securities
vbiux intermediate-term bond index
viigx intermediate-term government bond
vfiux intermediate term treasury
vblix long-term bond index
vlgix long-term government bond
vusux long-term treasury
vmbix mortgage-backed securities
vbipx short-term bond index
vsbix short-term government bond
vtspx short-term inflation protected
vfirx short-term treasury
vbtlx total bond market index
Investment Grade:
vicbx intermediate-term corporate bond
vfidx intermediate-term investment grade
vlcix long-term corporate bond
vwetx long-term investment grade
vstbx short-term corporate bond
vfsix short-term investment grade
vusfx ultra-short term bond
Below Investment Grade:
vweax high-yield corporate
Floating rate:
ffrhx Fidelity floating rate high income fund
Mortgage securities:
fmsfx Fidelity mortgage securities fund (basically the same as vmbix.ret)
Non-US:
fgbfx Fidelity global bond fund
finux Fidelity International Bond Fund
fnmix Fidelity new markets income fund
fghnx Fidelity global high income fund
Currencies:
FXA Australian Dollar
FXB British Pound
FXC Canadian Dollar
FXCH Chinese Renminbi
FXE Euro
FXY Japanese Yen
FXSG Singapore Dollar
FXS Swedish Krona
FXF Swiss Frank
UUP US Dollar Index
Commodities
oil Oil
uso US Oil
gld Gold
slv Silver
ung Natural Gas
djp Dow Jones-UBS Commodity Index
dba Agriculture
gsg GSCI commodity index
dbb Base metals
gltr precious metals
jo Coffee
jjg Grains
jjc Copper
corn Corn
weat Wheat
nib Cocoa
cow Livestock
ptm Platinum
bal Cotton
International Equity
ewg Germany
ewq France
ewh Hong Kong
ewi Italy
ewn Netherlands
eww Mexico
ewm Malaysia
ews Singapore
ewp Spain Capped
ewl Switzerland Capped
ewd Sweden
ewu United Kingdom
ewa Australia
ewk Belgium Capped
ewo Austria Capped
ewc Canada
ewy South Korea
ewt Taiwan
ewz Brazil
ezu Eurozone
iev Europe
US Sectors
iyf Financials
iyz Telecommunications
iym Basic Materials
iye Energy
iyc Consumer Services
iyk Consumer Goods
iyh Healthcare
iyg Financial Services
iyj Industrials
iyr Real Estate
idu Utilities
iyw Technology
kie Insurance
ibb Biotech
xbi Biotech
US Sectors
xly Discretionary
xlp Staples
xle Energy
xlf Financials
xlv Health Care
xli Industrials
xlb Materials
xlre Real Estate
xlk Technology
xlu Utilities
US Style Box
ijr S&P Small-Cap
ijs S&P Small-Cap 600 Value
ijt S&P Small-Cap 600 Growth
ijh S&P Mid-Cap
ijk S&P Mid-Cap 400 Growth
ijj S&P Mid-Cap 400 Value
ivv S&P 500
ivw S&P 500 Growth
ive S&P 500 Value
Common index ETFs (fundlab augmentations):
agg US aggregate bond
vt total world stock
vwo emerging markets stock
vnq US real estate (REITs)
vea developed markets (ex-US, FTSE)
efa developed markets (ex-US, MSCI)
iwm US small cap
tlt US long-term treasuries (20+ yr)
ief US intermediate treasuries (7-10 yr)
shv US short-term treasuries
bil US t-bills / cash

186
tests/test_fundlab.py Normal file
View File

@ -0,0 +1,186 @@
"""fundlab tests: pool parsing, text extraction, classification, EDGAR live.
Run: .venv/bin/python tests/test_fundlab.py
The live EDGAR test needs network + is rate-limited; it degrades to a skip.
"""
from __future__ import annotations
import json
import pathlib
import re
import sys
import urllib.error
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
from fundlab import edgar, pool # noqa: E402
PASS, FAIL = 0, 0
def check(name: str, cond: bool, extra: str = "") -> None:
global PASS, FAIL
if cond:
PASS += 1
print(f" ok {name}", flush=True)
else:
FAIL += 1
print(f" FAIL {name} {extra}", flush=True)
# ---------------------------------------------------------------- pool
def test_pool() -> None:
print("pool", flush=True)
entries = pool.load_pool()
check("pool parses many entries", len(entries) > 90, str(len(entries)))
bysym = {e.symbol: e for e in entries}
check("original entries present",
all(s in bysym for s in ("vbtlx", "ewz", "gld", "ijr", "fxe")))
check("augmentations present",
all(s in bysym for s in ("agg", "vt", "vwo", "vnq", "tlt", "bil")))
check("sections preserved",
bysym["vbtlx"].section.startswith("Treasury")
and bysym["agg"].section.startswith("Common index ETFs"))
check("labels carried", "inflation-protected" in bysym["vipix"].label)
check("no garbage symbols",
all(re.fullmatch(r"\S{1,6}", e.symbol) for e in entries))
# ---------------------------------------------------------------- text
_HTML = (b"<html><body><h3>Fund Summary</h3><p>Investment Objective</p>"
b"<p>The Fund seeks to track the performance of a benchmark index "
b"that measures the investment return of large-capitalization "
b"stocks.</p><p>Fees and Expenses</p></body></html>")
def test_text_and_objective() -> None:
print("text / objective extraction", flush=True)
txt = edgar.to_text(_HTML)
check("to_text strips tags",
"Fund Summary" in txt and "<p>" not in txt and "Investment Objective" in txt)
doc = ("Random cover page text. "
"Vanguard 500 Index Fund\nProspectus\nInvestment Objective\n"
"The Fund seeks to track the performance of a benchmark index "
"that measures the investment return of large-capitalization "
"stocks.\nFees and Expenses\nmore text about Vanguard 500 "
"Index Fund share classes.")
obj = edgar.extract_objective(doc, "Vanguard 500 Index Fund")
check("objective extracted",
obj is not None and obj.startswith("The Fund seeks to track")
and "large-capitalization stocks" in obj, repr(obj))
check("objective is bounded",
obj is not None and len(obj) < 300, repr(obj and len(obj)))
doc2 = ("Fund ABC\nThe ETF seeks to invest at least 80% of assets in "
"US investment grade debt securities of emerging markets.")
obj2 = edgar.extract_objective(doc2, "Fund ABC")
check("variant: 'The ETF seeks to ...' on the cover",
obj2 is not None and "debt securities" in obj2 and "US" in obj2,
repr(obj2))
check("no objective -> None",
edgar.extract_objective("no fund here at all", "Ghost Fund") is None)
# a document that merely MENTIONS the fund deep in its body (family
# filings, funds investing in it) must not yield that fund's objective:
# the name is not on the cover (within the first COVER_GATE chars)
body = ("Section %d discusses portfolio construction, risk management "
"and liquidity. ")
filler = ("XYZ Growth Fund\nProspectus\nInvestment Objective\nThe Fund "
"seeks to provide total return while generating moderate "
"current income.\n" +
"".join(body % i for i in range(40)) +
"Comparison: similar to Vanguard 500 Index Fund.\n")
check("mention-only doc -> None",
edgar.extract_objective(filler, "Vanguard 500 Index Fund") is None)
# the same doc with the name on the cover IS about the fund
oncover = ("Vanguard 500 Index Fund\nProspectus\nInvestment Objective\n"
"The Fund seeks to track large-capitalization stocks.\n" +
"".join(body % i for i in range(40)))
check("name on cover -> extracted",
(edgar.extract_objective(oncover, "Vanguard 500 Index Fund")
or "").startswith("The Fund seeks to track"))
# a leveraged ETF's cover cites the underlying fund's name; the name
# is a reference, not a title -> not our fund's document
doc4 = ("Acme 2X Leveraged QQQ Daily ETF\nProspectus\nThe Fund seeks\n"
"daily investment results that are 200% of the performance of\n"
"the Invesco QQQ Trust. The Fund seeks to achieve daily\n"
"investment results equal to two times the performance of the\n"
"index, before fees and expenses.")
check("underlying reference on cover -> None",
edgar.extract_objective(doc4, "Invesco QQQ Trust") is None)
# ---------------------------------------------------------------- classify
def test_classify() -> None:
print("category classification", flush=True)
c = edgar.classify_category
check("US index fund -> equity",
c("The Fund seeks to track large-capitalization stocks.",
"Vanguard 500 Index Fund") == "equity")
check("aggregate bond -> fixed_income",
c("The Fund seeks to track the performance of an index of US "
"investment grade debt securities.", "iShares Core US Aggregate Bond ETF")
== "fixed_income")
check("balanced -> mixed",
c("The Fund invests in a combination of equity and debt "
"securities.", "Example Balanced Fund") == "mixed")
check("gold -> alternatives",
c("The Fund seeks to track the price of gold bullion.",
"SPDR Gold Shares") == "alternatives")
check("money market -> money_market",
c("The Fund seeks to maintain stability of principal. It "
"invests in money market instruments.", "XX Money Market Fund")
== "money_market")
check("unknown -> other", c("The Fund does unusual things.", "Mystery") == "other")
# ---------------------------------------------------------------- edgar live
def test_edgar_live() -> None:
print("EDGAR live (network)", flush=True)
try:
name = None
meta = json.loads((pathlib.Path("~/prog/fin/stocks").expanduser()
/ "vtsax.json").read_text())["chart"]["result"][0]["meta"]
name = meta.get("longName")
if not name:
raise urllib.error.URLError("no local vtsax.json")
res = edgar.fetch_fund(name, ticker="vtsax", max_docs=8)
except Exception as e:
print(f" SKIP (no network / data): {type(e).__name__}: {e}")
return
check("vtsax objective fetched",
res is not None and "seeks to" in res["objective"]
and "stock market" in res["objective"].lower(),
repr(res and res.get("objective")))
check("vtsax classified equity",
res is not None and res["category"] == "equity",
repr(res and res.get("category")))
check("provenance recorded",
res is not None and res["url"].startswith("https://www.sec.gov")
and res["form"], repr(res))
def test_curated() -> None:
print("curated", flush=True)
import fundlab.fundinfo as fi
cur = fi.load_curated()
check("curated file loads", len(cur) >= 10, str(len(cur)))
check("curated entries well-formed",
all(set(v) >= {"objective", "category"} and
v["category"] in ("equity", "fixed_income", "mixed",
"alternatives", "money_market", "other")
for v in cur.values()))
check("curated objectives look right",
all("seeks" in v["objective"].lower() for v in cur.values()))
def main() -> int:
test_pool()
test_text_and_objective()
test_classify()
test_curated()
test_edgar_live()
print(f"\n{PASS} passed, {FAIL} failed")
return 1 if FAIL else 0
if __name__ == "__main__":
sys.exit(main())