Greg Pomerantz
bb8e39605e
Yahoo's 2026 event-feed change (capitalGain events dropped for some funds; no events at all for terminated tickers) can wipe good event history on re-download. Defense in depth: - overrides/event-backup/: last-known-good copy of every dividend/ capitalGain file (8,192 files, 58 MB); refresh with scripts/backup_events.py after each dump update - data.py event_file(): frozen > data-root (while populated) > backup; used by panel reads, verify_official, and the double-listing scanner - README: capital-gain files are mutual-fund-only in this dump; Yahoo has no LT/ST split (tax.py taxes capg at lt_rate; the per-fund split would come from fund-company annual tax statements or commercial feeds) - tests: exact Timestamp .loc keys (pandas 3.x string matching returns a Series on large DatetimeIndex)
864 lines
36 KiB
Python
864 lines
36 KiB
Python
#!/usr/bin/env python3
|
||
"""Tier-3 verification: cross-check local Yahoo distributions against
|
||
official SEC filings (N-CSR / N-CSRS / 10-K / 10-Q financial statements).
|
||
|
||
For each symbol:
|
||
1. EDGAR full-text search for the ticker restricted to annual/semi-annual
|
||
report forms; take the filing with the latest period end;
|
||
2. fetch the document and locate the fund's Financial Highlights tables;
|
||
3. parse every share-class block (per-share dividends from net investment
|
||
income, distributions from realized capital gains, NAV begin/end,
|
||
total return, for the current period + prior years);
|
||
4. match the class to the ticker by the per-share distribution series
|
||
and by NAV magnitude vs the local year-end closes;
|
||
5. compare official vs local (~/prog/fin/stocks CSVs, frozen copies for
|
||
stale tickers) per year and write a verdict.
|
||
|
||
Verdicts:
|
||
ok best-matching class agrees on every bounded fiscal period
|
||
mismatch the class clearly matches but one or more periods differ
|
||
beyond tolerance -> real data finding; review the period
|
||
(per-date local rows vs the filing) and correct via
|
||
overrides/ (see reports/stale-funds.md conventions)
|
||
weak-match best class fit too poor to trust (uncovered document
|
||
format or wrong fund) -> manual review
|
||
not-found fund not located in any candidate filing
|
||
error fetch/parse failure (see the json for details)
|
||
|
||
Notes:
|
||
- The oldest column has an unbounded local window (all earlier history)
|
||
and is displayed as ~ but never drives the verdict.
|
||
- Distributions are compared ex-date based, in the period window
|
||
(prev period end, period end]; this handles non-calendar fiscal years.
|
||
- ETF/annual-report formats are covered only partially (several ETF
|
||
families come back not-found); open-end N-CSR/N-CSRS formats of the
|
||
Vanguard/Victory style are the well-covered case.
|
||
|
||
Usage:
|
||
python3 scripts/verify_official.py [SYM ...] # default: funds.json
|
||
python3 scripts/verify_official.py --stale # + all frozen tickers
|
||
python3 scripts/verify_official.py --json-only # skip SEC, re-emit report
|
||
|
||
Output: reports/xcheck_official/{SYM}.json + SUMMARY.md
|
||
Politely rate-limited (~1 doc per second). Cached results are re-used
|
||
when the requested period is already covered (no re-fetch).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
import re
|
||
import sys
|
||
import time
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import Optional
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||
|
||
import fundlab.edgar as edgar # noqa: E402 (sec_get with UA + delay)
|
||
|
||
import data as D # noqa: E402
|
||
|
||
ROOT = D.DEFAULT_ROOT
|
||
REPORT_DIR = Path(__file__).resolve().parent.parent / "reports" / "xcheck_official"
|
||
FUNDS = Path(__file__).resolve().parent.parent / "funds.json"
|
||
|
||
EFTS_URL = "https://efts.sec.gov/LATEST/search-index?q={q}&forms=N-CSR,N-CSRS,10-K,10-Q,497,497K"
|
||
FORMS = ("N-CSR", "N-CSRS", "10-K", "10-Q", "497", "497K")
|
||
_STOP = {"fund", "funds", "inc", "ltd", "llc", "trust", "company", "the",
|
||
"and", "of", "for", "class", "series", "shares", "share", "in",
|
||
"on", "at", "to", "a", "an", "with", "portfolio", "portfolios",
|
||
"investment", "investments", "admiral", "investor", "investors",
|
||
"institutional", "etf", "plus", "select", "retail", "wholesale",
|
||
"group", "capital", "global", "us", "u.s."}
|
||
|
||
# 'Financial Highlights' often sits in separate table cells
|
||
# ("Financial |Highlights"); tolerate pipes/whitespace between the words
|
||
FH = r"Financial[\s|]+Highlights"
|
||
TOL_REL = 0.015 # 1.5% per year
|
||
TOL_ABS = 0.015 # or $0.015/share (official figures are 3-dp)
|
||
NAV_TOL = 0.25 # class match: |local_close - official_nav| / nav
|
||
|
||
|
||
# ---------------------------------------------------------------- parsing
|
||
MONTHS = ("January|February|March|April|May|June|July|August|September|"
|
||
"October|November|December")
|
||
|
||
|
||
_MON = {m: f"{i + 1:02d}" for i, m in enumerate(
|
||
[x for x in MONTHS.split("|")])}
|
||
|
||
|
||
def _num(window: str) -> Optional[float]:
|
||
c = window.strip().replace("$", "").replace(",", "")
|
||
neg = c.startswith("(") and c.endswith(")")
|
||
c = c.strip("() ")
|
||
try:
|
||
v = float(c)
|
||
except ValueError:
|
||
return None
|
||
return -v if neg else v
|
||
|
||
|
||
# row labels found in fund Financial Highlights tables; used to bound
|
||
# each row's value window so one row's numbers never bleed into the next
|
||
_ROW_LBL = (r"Net Asset Value,\s*(?:Beginning|End) of Period"
|
||
r"|Net Investment Income"
|
||
r"|Capital Gain Distributions Received"
|
||
r"|Net Realized(?:\s+and\s+Unrealized)?"
|
||
r"|Total from Investment Operations"
|
||
r"|Total Distributions?"
|
||
r"|Dividends?\s+from\s+(?:Net\s+)?"
|
||
r"|Distributions?\s+from\s+(?:Net\s+)?"
|
||
r"|Total Return"
|
||
r"|Ratios/Supplemental")
|
||
|
||
|
||
def _decimals(text: str, start: int, end: int, n: int) -> Optional[list]:
|
||
"""The n decimal numbers in `text[start:end]`, in order.
|
||
|
||
Footnote markers (bare integers) and dashes are skipped because only
|
||
values with a decimal point are accepted; negative values may span
|
||
multiple cells ("(.961 | )"), which the pipe-stripping handles.
|
||
"""
|
||
w = re.sub(r"[|\s]+", "", text[start:end])
|
||
w = w.replace("—", "-").replace(" ", "")
|
||
vals = []
|
||
for m in re.finditer(r"\((\d+\.\d+|\.\d+)\)|(\d+\.\d+|\.\d+)", w):
|
||
v = float(m.group(1) or m.group(2))
|
||
vals.append(-v if m.group(1) else v)
|
||
if len(vals) == n:
|
||
return vals
|
||
return None
|
||
|
||
|
||
_MON3 = {"jan": "01", "feb": "02", "mar": "03", "apr": "04", "may": "05",
|
||
"jun": "06", "jul": "07", "aug": "08", "sep": "09", "oct": "10",
|
||
"nov": "11", "dec": "12"}
|
||
|
||
|
||
def _parse_date(s: str) -> Optional[str]:
|
||
"""'June 30, 2026' | '1/31/26' | 'Dec-31-25' -> ISO date."""
|
||
s = re.sub(r"&#\w+;", " ", s).strip()
|
||
m = re.match(rf"((?:{MONTHS}))\.?\s+(\d{{1,2}}),?\s+(\d{{2,4}})", s, re.I)
|
||
if m:
|
||
y = m.group(3) if len(m.group(3)) == 4 else "20" + m.group(3)
|
||
return f"{y}-{_MON[m.group(1).title()]}-{int(m.group(2)):02d}"
|
||
m = re.match(r"(\d{1,2})[/-](\d{1,2})[/-](\d{2,4})", s)
|
||
if m:
|
||
y = m.group(3) if len(m.group(3)) == 4 else "20" + m.group(3)
|
||
if 1 <= int(m.group(1)) <= 12:
|
||
return f"{y}-{int(m.group(1)):02d}-{int(m.group(2)):02d}"
|
||
m = re.match(r"((?:" + "|".join(_MON3) + r"))\.?\s*[-/]\s*(\d{1,2})"
|
||
r"[-/]\s*(\d{2,4})", s, re.I)
|
||
if m:
|
||
y = m.group(3) if len(m.group(3)) == 4 else "20" + m.group(3)
|
||
return f"{y}-{_MON3[m.group(1).lower()]}-{int(m.group(2)):02d}"
|
||
return None
|
||
|
||
|
||
def _header_periods(text: str, i0: int, i_nav: int) -> list[dict]:
|
||
"""Period labels from the header between the class label and the first
|
||
'Net asset value, beginning' row.
|
||
|
||
Handles both styles:
|
||
Six Months Ended | June 30, | 2026 | Year Ended December 31, | 2025
|
||
| 2024 | ... (month-name dates + bare 4-digit years)
|
||
Six Months Ended 1/31/26 | Year Ended 7/31/25 | 7/31/24 | ...
|
||
(M/D/YY dates)
|
||
Bare years inherit the month-day of the preceding dated column.
|
||
"""
|
||
hdr = re.sub(r"&#\w+;", " ", text[i0:i_nav])
|
||
hdr = re.sub(r"[|\s]+", " ", hdr).strip()
|
||
# columns are newest-first and dated/bare-year columns may interleave;
|
||
# scan left to right in a single pass, never sort
|
||
periods = []
|
||
state = {"md": None}
|
||
|
||
def _eat(m):
|
||
g = m.group(0)
|
||
iso = _parse_date(g)
|
||
if iso:
|
||
state["md"] = iso[5:]
|
||
periods.append({"end": iso, "year": iso[:4], "kind": "fy"})
|
||
return " "
|
||
y = g.strip()
|
||
if not (2 <= len(y) <= 4) or not y.isdigit():
|
||
return g
|
||
if len(y) == 2:
|
||
y = "20" + y
|
||
if not (1950 <= int(y) <= 2040):
|
||
return g
|
||
if not state["md"]:
|
||
state["md"] = "12-31"
|
||
periods.append({"end": f"{y}-{state['md']}", "year": y, "kind": "fy"})
|
||
return " "
|
||
|
||
hdr = re.sub(rf"(?:{MONTHS})\.?\s+\d{{1,2}},?\s+\d{{2,4}}"
|
||
rf"|\d{{1,2}}[/-]\d{{1,2}}[/-]\d{{2,4}}"
|
||
r"|\b(19\d{2}|20\d{2}|\d{2})\b",
|
||
_eat, hdr, flags=re.I)
|
||
# drop duplicate ends, keep first
|
||
seen, out = set(), []
|
||
for p in periods:
|
||
if p["end"] in seen:
|
||
continue
|
||
seen.add(p["end"])
|
||
out.append(p)
|
||
if len(out) >= 2 and out[0]["end"] >= out[1]["end"]:
|
||
out[0]["kind"] = "current"
|
||
return out
|
||
|
||
|
||
def parse_highlights(text: str) -> list[dict]:
|
||
"""All share-class Financial Highlights blocks in `text` (pipe-delimited,
|
||
tag-stripped). A block starts at a 'Net asset value, beginning' row
|
||
(both Vanguard's 'Net Asset Value, Beginning of Period' and Victory's
|
||
'Net asset value, beginning of period' style)."""
|
||
# Row labels are often split across table cells ("Net | asset | value,
|
||
# beginning"); pattern searches run on a pipe-flattened copy (1:1
|
||
# offsets). Cell-based label extraction below uses the original.
|
||
flat = text.replace("|", " ")
|
||
starts = [m.start() for m in
|
||
re.finditer(r"net\s+asset\s+value,\s*beginning", flat, re.I)]
|
||
blocks = []
|
||
for i, s in enumerate(starts):
|
||
end = starts[i + 1] if i + 1 < len(starts) else s + 25000
|
||
pre = re.sub(r"&#\w+;", " ", text[max(0, s - 900):s])
|
||
cells = [c.strip() for c in pre.split("|") if c.strip()]
|
||
# class label: nearest "... Shares" / "Class X" cell; the cells
|
||
# right before the row are often column years, not the label
|
||
label = "?"
|
||
for c in reversed(cells):
|
||
if re.fullmatch(r"[A-Za-z][A-Za-z0-9 \-]*\s+Shares", c) or \
|
||
re.fullmatch(r"Class\s+[A-Z0-9]+", c):
|
||
label = re.sub(r"\s+", " ", c).strip()
|
||
break
|
||
# header window: back at most 2000 chars, but never across a
|
||
# "Table of Contents" page footer (its page numbers would be read
|
||
# as bare year columns)
|
||
h0 = max(0, s - 2000)
|
||
toc = flat.rfind("Table of Contents", h0, s)
|
||
fh = max([m.end() for m in re.finditer(FH, flat[h0:s])] or [h0]) + h0
|
||
h0 = max(h0, toc, fh)
|
||
periods = _header_periods(flat, h0, s)
|
||
if len(periods) < 2:
|
||
continue
|
||
n = len(periods)
|
||
i_end = min(end, flat.find("See accompanying", s) if
|
||
flat.find("See accompanying", s) != -1 else end)
|
||
|
||
def row(pattern: str) -> Optional[list]:
|
||
mm = re.search(pattern, flat[s:i_end], re.I)
|
||
if not mm:
|
||
return None
|
||
i0, i1 = s + mm.end(), i_end
|
||
for b in [x.start() for x in re.finditer(_ROW_LBL,
|
||
flat[s:i_end], re.I)]:
|
||
if b > mm.end():
|
||
i1 = s + b
|
||
break
|
||
return _decimals(flat, i0, i1, n)
|
||
|
||
# distribution rows: prefer explicit "from net investment income" /
|
||
# "from realized capital gains" labels (Vanguard); otherwise the
|
||
# "Distributions to shareholders:" sub-table (Victory), where the
|
||
# NII row is the first "Net investment income" after that header
|
||
ni = row(r"dividends?\s+from\s+(?:net\s+)?investment\s+income")
|
||
cg = row(r"(?:distributions?|dividends?)\s+from\s+(?:net\s+)?realized\s+(?:capital\s+)?gains?")
|
||
if ni is None:
|
||
dm = re.search(r"distributions?\s+to\s+shareholders", flat[s:i_end], re.I)
|
||
if dm:
|
||
mm = re.search(r"net\s+investment\s+income", flat[s + dm.end():i_end], re.I)
|
||
if mm:
|
||
i0 = s + dm.end() + mm.end()
|
||
ni = _decimals(flat, i0, i_end, n)
|
||
tot = row(r"total\s+distributions?\b")
|
||
if ni is None and tot is None:
|
||
continue
|
||
if tot is None:
|
||
if ni is None or cg is None:
|
||
continue
|
||
tot = [round(abs(a) + abs(b), 4) for a, b in zip(ni, cg)]
|
||
# block values: this block's NAV rows
|
||
navb = row(r"net\s+asset\s+value,\s*beginning")
|
||
nave = row(r"net\s+asset\s+value,\s*end")
|
||
blocks.append({"class": label, "periods": periods,
|
||
"nii": [abs(v) for v in ni] if ni else None,
|
||
"capg": [abs(v) for v in cg] if cg else None,
|
||
"total": [abs(v) for v in tot],
|
||
"nav_begin": navb, "nav_end": nave})
|
||
return blocks
|
||
|
||
|
||
def _cellnum(c: str) -> Optional[float]:
|
||
"""One table cell -> number. Dashes (a zero with a footnote) count as
|
||
0.0; footnote-only and junk cells are None."""
|
||
c = c.replace("$", "").replace(",", "").replace(" ", " ").strip()
|
||
if not c:
|
||
return None
|
||
if c.startswith("—") or c in ("-", "–"):
|
||
return 0.0
|
||
m = re.fullmatch(r"(\()?(\d+\.\d+|\.\d+)(\))?(\s*\([a-z]\))?", c)
|
||
if m:
|
||
return -float(m.group(2)) if m.group(1) else float(m.group(2))
|
||
return None
|
||
|
||
|
||
def parse_per_share_blocks(text: str) -> list[dict]:
|
||
"""JPMorgan-style shareholder-report layout ('Per share operating
|
||
performance'): each class block lists, per 'Year Ended <date>' column,
|
||
NAV beginning, NII, realized/unrealized gain, total from operations,
|
||
distribution NII, distribution capital gain, total distributions.
|
||
Cell-based (not _decimals): dash columns are real zeros, and the value
|
||
window must not run into the next year's column. Returns blocks
|
||
compatible with parse_highlights()."""
|
||
cells = [re.sub(r"\s+", " ", c).strip().replace("—", "—")
|
||
.replace(" ", "").replace(" ", "")
|
||
for c in text.split("|")]
|
||
events: list[tuple] = []
|
||
for i, c in enumerate(cells):
|
||
m = re.fullmatch(r"Year[s]?\s+Ended\s+(.+)", c)
|
||
if m and _parse_date(m.group(1)):
|
||
events.append((i, "year", m.group(1)))
|
||
elif re.fullmatch(r"Class\s+[A-Z0-9]{1,3}", c):
|
||
events.append((i, "class", c))
|
||
blocks: list[dict] = []
|
||
cur = None
|
||
for j, (i, kind, val) in enumerate(events):
|
||
if kind == "class":
|
||
cur = {"class": val, "periods": [], "nii": [], "capg": [],
|
||
"total": [], "nav_begin": [], "nav_end": None}
|
||
blocks.append(cur)
|
||
continue
|
||
if cur is None:
|
||
continue
|
||
d = _parse_date(val)
|
||
stop = events[j + 1][0] if j + 1 < len(events) else len(cells)
|
||
vals: list[float] = []
|
||
for c in cells[i + 1:stop]:
|
||
if re.fullmatch(r"Year[s]?\s+Ended\s+.+", c):
|
||
break
|
||
v = _cellnum(c)
|
||
if v is None:
|
||
continue
|
||
vals.append(v)
|
||
if len(vals) == 7:
|
||
break
|
||
if len(vals) != 7:
|
||
continue
|
||
cur["periods"].append({"end": d, "year": d[:4], "kind": "fy"})
|
||
cur["nav_begin"].append(abs(vals[0]))
|
||
cur["nii"].append(abs(vals[4]))
|
||
cur["capg"].append(abs(vals[5]))
|
||
cur["total"].append(abs(vals[6]))
|
||
return [b for b in blocks if len(b["periods"]) >= 2]
|
||
|
||
|
||
def html_to_cells_text(html: bytes) -> str:
|
||
t = re.sub(rb"<\s*(br|/p|/div|/tr|/h[1-6])[^>]*>", b"|\n", html, flags=re.I)
|
||
t = re.sub(rb"<[^>]+>", b"|", t)
|
||
t = t.decode("utf-8", "replace")
|
||
# numeric entities (  narrow nbsp,   nbsp, ...) break literal
|
||
# searches like "Financial Highlights"; keep dashes (zero markers),
|
||
# collapse the rest to plain spaces
|
||
t = (t.replace("—", "—").replace("–", "–")
|
||
.replace("—", "—").replace("–", "–"))
|
||
t = re.sub(r"&#\d+;", " ", t)
|
||
t = re.sub(r"&[a-z]+;", " ", t)
|
||
return t
|
||
|
||
|
||
# ---------------------------------------------------------------- EFTS
|
||
def _efits(q: str) -> list[dict]:
|
||
"""EFTS hits -> [{accession, doc, period_end, display, ciks, score}]."""
|
||
import urllib.parse
|
||
d = json.loads(edgar.sec_get(EFTS_URL.format(q=urllib.parse.quote(q)))
|
||
.decode("utf-8", "replace"))
|
||
out, seen = [], set()
|
||
# EFTS returns up to 100; small filings (497K notices) rank above the
|
||
# large combined reports, so keep the whole page and let the
|
||
# find_filings re-ranking pick the real thing
|
||
for h in d.get("hits", {}).get("hits", [])[:100]:
|
||
src = h.get("_source", {})
|
||
acc, _, doc = h.get("_id", "").partition(":")
|
||
if acc in seen or not doc.lower().endswith((".htm", ".html")):
|
||
continue
|
||
seen.add(acc)
|
||
out.append({"accession": acc, "doc": doc,
|
||
# 497s carry no period_ending; file_date keeps them
|
||
# orderable
|
||
"period_end": src.get("period_ending")
|
||
or src.get("file_date") or "",
|
||
"display": src.get("display_names", ["?"])[0],
|
||
"ciks": src.get("ciks", []),
|
||
"score": h.get("_score", 0)})
|
||
out.sort(key=lambda f: f["period_end"], reverse=True)
|
||
return out
|
||
|
||
|
||
def _brand(name: str) -> Optional[str]:
|
||
"""First distinctive word of the fund name, usually the family brand
|
||
(iShares, Fidelity, JPMorgan, SPDR, Invesco...)."""
|
||
for w in re.findall(r"[A-Za-z][A-Za-z\-'.]{2,}", name):
|
||
if w.lower() not in _STOP:
|
||
return w
|
||
return None
|
||
|
||
|
||
def find_filings(ticker: str, name: str, limit: int = 6) -> list[dict]:
|
||
"""Candidate filings, best first. A bare ticker matches hundreds of
|
||
unrelated documents, so: (1) the full fund name as one phrase, (2)
|
||
ticker + family brand, (3) ticker + distinctive name words, (4) the
|
||
bare ticker. Results are then ranked: a registrant whose display name
|
||
contains the ticker (single-fund trusts like 'SPDR GOLD TRUST (GLD)')
|
||
or the brand words (families like iShares Trust) ranks first, newest
|
||
filing first within a registrant. EFTS itself scores bag-of-words,
|
||
so unrelated prospectuses often outrank the fund's own 10-K — this
|
||
re-ranking is what makes ETF verification possible."""
|
||
def merge(f: list[dict]) -> None:
|
||
for x in f:
|
||
if x["accession"] not in {y["accession"] for y in out}:
|
||
out.append(x)
|
||
out: list[dict] = []
|
||
words = [w for w in re.findall(r"[A-Za-z][A-Za-z\-'.]{2,}", name)
|
||
if w.lower() not in _STOP]
|
||
brand = _brand(name)
|
||
if len(words) >= 4:
|
||
merge(_efits('"' + name + '"'))
|
||
if brand:
|
||
merge(_efits('"' + ticker + '" AND "' + brand + '"'))
|
||
if len(out) < limit and words:
|
||
merge(_efits('"' + ticker + '" AND ' +
|
||
" AND ".join('"' + w + '"' for w in words[:3])))
|
||
if len(out) < limit:
|
||
merge(_efits('"' + ticker + '"'))
|
||
name_words = {w.lower() for w in re.findall(r"[A-Za-z][A-Za-z\-'.]{3,}",
|
||
name)
|
||
if w.lower() not in _STOP}
|
||
scored = []
|
||
for f in out:
|
||
disp = f["display"].lower()
|
||
s = 4 * (1 if ticker.lower() in disp else 0) \
|
||
+ sum(1 for w in name_words if w in disp)
|
||
scored.append((s, f["period_end"], f))
|
||
scored.sort(key=lambda t: (t[0], t[1]), reverse=True)
|
||
per_cik: dict = {}
|
||
res = []
|
||
for s, pe, f in scored:
|
||
key = tuple(f["ciks"]) or (f["display"],)
|
||
if per_cik.get(key, 0) >= 2: # at most 2 filings per registrant
|
||
continue
|
||
per_cik[key] = per_cik.get(key, 0) + 1
|
||
res.append(f)
|
||
if len(res) >= limit:
|
||
break
|
||
return res
|
||
|
||
|
||
def _flex(s: str) -> str:
|
||
"""A word pattern that tolerates pipes/whitespace between words
|
||
(fund names often sit in one cell per word)."""
|
||
return r"[\s|]+".join(re.escape(w) for w in s.split())
|
||
|
||
|
||
def _title_runs(seg: str) -> list[str]:
|
||
return re.findall(r"[A-Z][A-Za-z0-9'\u2019\-]*"
|
||
r"(?:\s+[A-Z][A-Za-z0-9'\u2019\-]*)+", seg)
|
||
|
||
|
||
def fund_region(text: str, ticker: str, name: str) -> Optional[str]:
|
||
"""The slice of the document holding this fund's highlights tables.
|
||
|
||
The ticker appears only in the TOC/cover, not in the financial
|
||
statements. The TOC cell "<Class> Shares - TICKER" is preceded by the
|
||
official fund name cell; section headers in the statements drop
|
||
prefixes like "Vanguard ", so anchor on the name or a tail of it.
|
||
"""
|
||
occs = [m.start() for m in re.finditer(rf"(?<![A-Z0-9]){ticker}(?![A-Z0-9])",
|
||
text, re.I)]
|
||
if not occs:
|
||
return None
|
||
cands: list[str] = []
|
||
for occ in occs[:3]:
|
||
cells = [re.sub(r"&#\w+;|\s+", " ", c).strip()
|
||
for c in text[occ - 800:occ + 200].split("|")]
|
||
for i, c in enumerate(cells):
|
||
# the ticker cell: bare "PMAIX", "Shares - VTSAX", "Class A / PMAIX"...
|
||
if not re.search(rf"(?<![A-Z0-9]){re.escape(ticker)}(?![A-Z0-9])",
|
||
c, re.I):
|
||
continue
|
||
for prev in reversed(cells[:i]):
|
||
if prev and "shares" not in prev.lower() \
|
||
and "class" not in prev.lower() \
|
||
and len(prev.split()) >= 3:
|
||
cands.append(prev)
|
||
break
|
||
cands.append(name)
|
||
for cand in dict.fromkeys(cands):
|
||
words = cand.split()
|
||
if len(words) < 4:
|
||
continue
|
||
for tail_n in range(len(words), 3, -1):
|
||
anchor = _flex(" ".join(words[-tail_n:]))
|
||
for m in re.finditer(anchor, text, re.I):
|
||
fhm = re.search(FH, text[m.start():m.start() + 4000])
|
||
if not fhm:
|
||
continue
|
||
fh = m.start() + fhm.start()
|
||
reg = text[fh:fh + 150000]
|
||
if parse_highlights(reg) or parse_per_share_blocks(reg):
|
||
return reg
|
||
return None
|
||
|
||
|
||
def _name_pats(name: str) -> list[str]:
|
||
"""Pipe-tolerant name patterns. Yahoo longNames and filing names differ
|
||
("JPMorgan US Large Cap Core Plus I" vs "JPMorgan U.S. Large Cap Core
|
||
Plus Fund"), so match on runs of 3-4 distinctive words, dots optional.
|
||
Trailing single-letter / RC# class designators are dropped."""
|
||
if not name:
|
||
return []
|
||
words = [re.sub(r"\.", "", w) for w in name.split()]
|
||
while words and (len(words[-1]) <= 2 and words[-1].isalpha()):
|
||
words.pop()
|
||
pats = []
|
||
def mk(ws: list[str]) -> str:
|
||
# optional dots between letters: "US" matches "U.S." and vice versa
|
||
return r"[\s|]+".join(r"(?:" + r"\.?".join(re.escape(ch) for ch in w)
|
||
+ r"\.?)" for w in ws)
|
||
# tail runs (the distinctive part) and the head (brand + next words)
|
||
for n in (4, 3):
|
||
if len(words) >= n:
|
||
pats.append(mk(words[-n:]))
|
||
if len(words) > n:
|
||
pats.append(mk(words[:n]))
|
||
pats.append(mk(words))
|
||
return pats
|
||
|
||
|
||
def region_per_share(text: str, ticker: str, name: str,
|
||
loc: Optional[dict] = None) -> Optional[str]:
|
||
"""Region finder for the 'Per share operating performance' layout
|
||
(JPMorgan-style shareholder reports). Anchors on the fund name or
|
||
ticker; when `loc` (local series) is given, a candidate region is only
|
||
accepted if one of its class blocks actually matches the local
|
||
distributions — name mentions in notes/TOC of a 58 MB combined report
|
||
would otherwise attribute another fund's tables."""
|
||
pats = _name_pats(name)
|
||
pats.append(rf"(?<![A-Z0-9]){re.escape(ticker)}(?![A-Z0-9])")
|
||
for p in pats:
|
||
for m in re.finditer(p, text, re.I):
|
||
reg = text[m.start():m.start() + 400000]
|
||
blocks = parse_per_share_blocks(reg)
|
||
if not blocks:
|
||
continue
|
||
if loc is None:
|
||
return reg
|
||
for b in blocks:
|
||
errs = []
|
||
for i in range(len(b["periods"]) - 1):
|
||
off = abs(b["total"][i])
|
||
lc = _window_sum(loc["dist"], b["periods"][i + 1]["end"],
|
||
b["periods"][i]["end"])
|
||
if off < 0.01 and lc < 0.01:
|
||
continue
|
||
errs.append(abs(off - lc) / max(off, lc, 1e-9))
|
||
if errs and sum(errs) / len(errs) < 0.15:
|
||
return reg
|
||
return None
|
||
|
||
|
||
def _wordset(s: str) -> set:
|
||
return set(re.findall(r"[a-z0-9]{3,}", s.lower())) - _STOP
|
||
|
||
|
||
def region_by_name(text: str, name: str, min_shared: int = 3):
|
||
"""Fallback when the ticker is absent from the document (families that
|
||
don't publish distribution tickers): match the fund by name — the cell
|
||
immediately before each Financial Highlights heading — and keep the
|
||
best word-overlap candidate with a parseable region."""
|
||
target = _wordset(name)
|
||
best = None
|
||
for m in re.finditer(FH, text):
|
||
pre = re.sub(r"&#\w+;", " ", text[max(0, m.start() - 400):m.start()])
|
||
pre = re.sub(r"[\s|]+", " ", pre).strip()
|
||
runs = re.findall(r"[A-Z][A-Za-z0-9'\u2019\-]*"
|
||
r"(?: [A-Z][A-Za-z0-9'\u2019\-]*){2,11}", pre)
|
||
if not runs:
|
||
continue
|
||
for cand in dict.fromkeys(runs): # any run, best overlap wins
|
||
shared = len(_wordset(cand) & target)
|
||
if shared < min_shared:
|
||
continue
|
||
reg = text[m.start():m.start() + 150000]
|
||
if not (parse_highlights(reg) or parse_per_share_blocks(reg)):
|
||
continue
|
||
if best is None or shared > best[0]:
|
||
best = (shared, reg, cand)
|
||
return best
|
||
|
||
|
||
# ---------------------------------------------------------------- compare
|
||
def local_series(sym: str, root: Path, frozen: Path) -> dict:
|
||
"""Local (date, amount) distributions and close series; frozen copies
|
||
shadow the data root, and the overrides/corrections overlay (confirmed
|
||
official fixes, see overrides/corrections/) is applied on top so a
|
||
corrected fund verifies against its filing rather than the raw dump."""
|
||
import pandas as pd
|
||
def read(sfx: str):
|
||
# event files: frozen > data-root (while populated) > event-backup
|
||
p = D.event_file(sym, sfx, root) if sfx in ("dividend", "capitalGain") \
|
||
else (frozen / f"{sym}-{sfx}.csv")
|
||
if not p.exists():
|
||
p = root / f"{sym}-{sfx}.csv"
|
||
if not p.exists():
|
||
return []
|
||
out = []
|
||
for line in p.read_text(errors="replace").splitlines()[1:]:
|
||
parts = line.split(",")
|
||
if len(parts) >= 2:
|
||
out.append((parts[0][:10], float(parts[1])))
|
||
ser = pd.Series({pd.Timestamp(d): v for d, v in out}, dtype=float)
|
||
try:
|
||
ser = D._apply_corrections(sym, sfx, ser)
|
||
except Exception:
|
||
pass
|
||
return [(ts.strftime("%Y-%m-%d"), float(v)) for ts, v in ser.items()]
|
||
div, capg = read("dividend"), read("capitalGain")
|
||
try:
|
||
div, capg = D.dedupe_event_rows(D._load_corrections(sym), div, capg)
|
||
except Exception:
|
||
pass
|
||
dist = sorted(div + capg)
|
||
p = frozen / f"{sym}-history.csv"
|
||
if not p.exists():
|
||
p = root / f"{sym}-history.csv"
|
||
closes = []
|
||
if p.exists():
|
||
for line in p.read_text(errors="replace").splitlines()[1:]:
|
||
parts = line.split(",")
|
||
if len(parts) >= 5:
|
||
closes.append((parts[0][:10], float(parts[4])))
|
||
return {"dist": dist, "close": closes}
|
||
|
||
|
||
def _window_sum(dist, lo, hi):
|
||
"""Sum of distributions with lo < date <= hi (ISO date strings)."""
|
||
return round(sum(v for d, v in dist if (lo is None or d > lo) and d <= hi),
|
||
4)
|
||
|
||
|
||
def compare(sym: str, blocks: list[dict], loc: dict) -> dict:
|
||
"""Pick the best-matching class and compare period by period.
|
||
|
||
Each official column covers (previous period end, this period end];
|
||
local distributions are summed over the same window, so non-calendar
|
||
fiscal years compare correctly.
|
||
"""
|
||
def period_errs(b):
|
||
errs = []
|
||
for i, p in enumerate(b["periods"]):
|
||
if p["kind"] != "fy":
|
||
continue
|
||
if i + 1 >= len(b["periods"]):
|
||
continue # oldest column: unbounded window, unverifiable
|
||
off = abs(b["total"][i])
|
||
lo = b["periods"][i + 1]["end"]
|
||
lc = _window_sum(loc["dist"], lo, p["end"])
|
||
errs.append(abs(off - lc) / max(off, lc, 1e-9))
|
||
return errs
|
||
|
||
best = None
|
||
for b in blocks:
|
||
errs = period_errs(b)
|
||
if not errs:
|
||
continue
|
||
me = sum(errs) / len(errs)
|
||
if best is None or me < best["err"]:
|
||
best = {"err": me, "block": b}
|
||
if best is None:
|
||
return {"verdict": "not-found", "classes": len(blocks), "detail":
|
||
"no class block with overlapping periods"}
|
||
b = best["block"]
|
||
rows = []
|
||
mismatch = False
|
||
for i, p in enumerate(b["periods"]):
|
||
off = abs(b["total"][i])
|
||
# columns are newest-first: the window lower bound is the NEXT
|
||
# (older) column's end; the oldest column is unbounded
|
||
lo = b["periods"][i + 1]["end"] if i + 1 < len(b["periods"]) else None
|
||
lc = _window_sum(loc["dist"], lo, p["end"])
|
||
d = off - lc
|
||
ok = abs(d) <= TOL_ABS or abs(d) / max(off, 1e-9) <= TOL_REL
|
||
mismatch |= p["kind"] == "fy" and lo is not None and not ok
|
||
rows.append({"period": p["kind"], "end": p["end"],
|
||
"official": round(off, 4), "local": lc,
|
||
"diff": round(d, 4), "ok": ok,
|
||
"unbounded": lo is None})
|
||
# NAV check: most recent period with an official ending NAV
|
||
nav = None
|
||
if b.get("nav_end") and loc["close"]:
|
||
closes = dict(loc["close"])
|
||
for i in range(len(b["periods"]) - 1, -1, -1):
|
||
p = b["periods"][i]
|
||
prior = [d for d in closes if d <= p["end"]]
|
||
if not prior:
|
||
continue
|
||
o, l = abs(b["nav_end"][i]), closes[prior[-1]]
|
||
nav = {"date": prior[-1], "official": o, "local": l,
|
||
"rel": round(abs(o - l) / o, 4),
|
||
"ok": abs(o - l) / o <= NAV_TOL}
|
||
break
|
||
# verdict: a good class fit with a bad year is a real mismatch; a bad
|
||
# class fit overall means the document format (or fund) isn't matched
|
||
if best["err"] >= 0.30:
|
||
verdict = "weak-match"
|
||
else:
|
||
verdict = "mismatch" if mismatch else "ok"
|
||
return {"verdict": verdict,
|
||
"class": b["class"], "mean_rel_err": round(best["err"], 4),
|
||
"nav_check": nav, "periods": rows,
|
||
"other_classes": [x["class"] for x in blocks if x is not b]}
|
||
|
||
|
||
# ---------------------------------------------------------------- driver
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("syms", nargs="*")
|
||
ap.add_argument("--stale", action="store_true",
|
||
help="also verify all frozen (stale) tickers")
|
||
ap.add_argument("--json-only", action="store_true")
|
||
ap.add_argument("--limit", type=int, default=0)
|
||
args = ap.parse_args()
|
||
|
||
syms = [s.lower() for s in args.syms]
|
||
if not syms and not args.stale:
|
||
syms = list(json.loads(FUNDS.read_text()).keys())
|
||
if args.stale:
|
||
syms += sorted(f.stem for f in (D.FROZEN_DIR.glob("*.json")))
|
||
syms = list(dict.fromkeys(syms))
|
||
if args.limit:
|
||
syms = syms[:args.limit]
|
||
|
||
REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
||
summary = []
|
||
t0 = time.monotonic()
|
||
for n, sym in enumerate(syms, 1):
|
||
out = REPORT_DIR / f"{sym}.json"
|
||
cached = None
|
||
if out.exists() and not args.json_only:
|
||
cached = json.loads(out.read_text())
|
||
if not (cached.get("filing") or cached.get("period_end")):
|
||
cached = None # not-found from a prior run: retry
|
||
if cached:
|
||
summary.append(cached)
|
||
print(f"[{n}/{len(syms)}] {sym} cached {cached['verdict']}", flush=True)
|
||
continue
|
||
name = ""
|
||
meta_f = ROOT / f"{sym}.json"
|
||
if meta_f.exists():
|
||
try:
|
||
name = json.loads(meta_f.read_text())["chart"]["result"][0] \
|
||
.get("meta", {}).get("longName", "")
|
||
except Exception:
|
||
pass
|
||
if not name:
|
||
mf = D.FROZEN_DIR / f"{sym}.json"
|
||
if mf.exists():
|
||
name = json.loads(mf.read_text()).get("longName", "")
|
||
try:
|
||
if args.json_only:
|
||
raise RuntimeError("json-only")
|
||
cands = find_filings(sym.upper(), name)
|
||
rec = {"symbol": sym, "name": name}
|
||
loc = local_series(sym, ROOT, D.FROZEN_DIR)
|
||
best = None
|
||
for filing in cands:
|
||
try:
|
||
doc = edgar.sec_get(
|
||
f"https://www.sec.gov/Archives/edgar/data/"
|
||
f"{int(filing['ciks'][0]):010d}/{filing['accession'].replace('-', '')}/"
|
||
f"{filing['doc']}")
|
||
except Exception:
|
||
continue
|
||
text = html_to_cells_text(doc)
|
||
region, how = None, None
|
||
if re.search(rf"(?<![A-Z0-9]){re.escape(sym.upper())}(?![A-Z0-9])",
|
||
text):
|
||
region = fund_region(text, sym.upper(), name)
|
||
how = "ticker"
|
||
if region is None:
|
||
rb = region_by_name(text, name)
|
||
if rb:
|
||
region, how = rb[1], f"name:{rb[2]} (shared={rb[0]})"
|
||
if region is None:
|
||
rp = region_per_share(text, sym.upper(), name, loc)
|
||
if rp:
|
||
region, how = rp, "per-share-data"
|
||
if region is None:
|
||
continue # wrong document; try next candidate
|
||
blocks = (parse_highlights(region)
|
||
or parse_per_share_blocks(region))
|
||
res = compare(sym, blocks, loc)
|
||
res.update({"symbol": sym, "name": name, "filing": filing,
|
||
"how": how, "n_bytes": len(doc)})
|
||
# different official docs can disagree by a cent or two
|
||
# (N-CSRS vs 497 rounding); keep the best match but stop
|
||
# early on a clean "ok"
|
||
key = (res["verdict"] != "ok", res.get("mean_rel_err", 1.0))
|
||
if best is None or key < (best["verdict"] != "ok",
|
||
best.get("mean_rel_err", 1.0)):
|
||
best = res
|
||
if res["verdict"] == "ok":
|
||
break
|
||
if best is not None:
|
||
rec.update(best)
|
||
else:
|
||
rec["verdict"] = "not-found"
|
||
rec["detail"] = ("no candidate filing contained this fund "
|
||
f"({len(cands)} tried)")
|
||
except Exception as e:
|
||
rec = {"symbol": sym, "name": name, "verdict": "error",
|
||
"detail": str(e)[:300]}
|
||
out.write_text(json.dumps(rec, indent=1))
|
||
summary.append(rec)
|
||
print(f"[{n}/{len(syms)}] {sym} {rec.get('verdict')} "
|
||
f"({time.monotonic() - t0:.0f}s)", flush=True)
|
||
|
||
# summary covers EVERY record in the report dir, not just this batch
|
||
recs = []
|
||
for f in sorted(REPORT_DIR.glob("*.json")):
|
||
try:
|
||
recs.append(json.loads(f.read_text()))
|
||
except Exception:
|
||
pass
|
||
lines = ["# Official-filing cross-check (SEC N-CSR/N-CSRS/10-K/10-Q)\n",
|
||
"Verdicts: ok = all bounded fiscal periods agree; mismatch = class matched "
|
||
"but some period differs (review the year, then correct via overrides/); "
|
||
"weak-match = best class fit too poor to trust (format/fund mismatch); "
|
||
"not-found = fund not located in any candidate filing.\n",
|
||
"| Symbol | Verdict | Class | Periods (fy end, ~ = unbounded) | Detail |",
|
||
"|---|---|---|---|---|"]
|
||
for rec in recs:
|
||
yrs = ""
|
||
if rec.get("periods"):
|
||
yrs = " ".join(
|
||
((f"{y['end'][:7]}~") if y.get("unbounded")
|
||
else f"{y['end'][:7]}{'✓' if y['ok'] else '✗'}"
|
||
for y in rec["periods"] if y["period"] == "fy"))
|
||
lines.append(f"| {rec['symbol'].upper()} | {rec.get('verdict')} | "
|
||
f"{rec.get('class', '')} | {yrs} | "
|
||
f"{rec.get('detail', '')[:80]} |")
|
||
(REPORT_DIR / "SUMMARY.md").write_text("\n".join(lines) + "\n")
|
||
from collections import Counter
|
||
c = Counter(r.get("verdict", "?") for r in summary)
|
||
print(f"\n{dict(c)} ({time.monotonic() - t0:.0f}s) -> {REPORT_DIR / 'SUMMARY.md'}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|