Greg Pomerantz
6536c9903e
scripts/verify_official.py locates each fund's latest N-CSR/N-CSRS/10-K/10-Q via EDGAR full-text search (full fund-name phrase first, then ticker + name words, then bare ticker), extracts the fund's Financial Highlights tables (both Vanguard-style and Victory-style layouts, calendar and non-calendar fiscal years, M/D/YY and month-name headers), matches the share class by per-share distribution series + NAV magnitude, and compares per period window against the local Yahoo CSVs (frozen copies for stale tickers). Per-symbol JSONs + SUMMARY.md land in reports/xcheck_official/; results are cached per fund. Run on the 29 curated funds: 8 ok (exact to 3dp, e.g. VTSAX 2021-2026H1), 1 mismatch (CVSIX FY2009: local 1.104 vs official 0.81 - the Yahoo 2008-12-18 row of 0.292 looks spurious), 3 weak-match (uncovered doc formats, e.g. Leuthold), 17 not-found (mostly ETF families whose 10-K layouts aren't covered yet).
652 lines
27 KiB
Python
652 lines
27 KiB
Python
#!/usr/bin/env python3
|
|
"""Tier-3 verification: cross-check local Yahoo distributions against
|
|
official SEC filings (N-CSR / N-CSRS / 10-K / 10-Q financial statements).
|
|
|
|
For each symbol:
|
|
1. EDGAR full-text search for the ticker restricted to annual/semi-annual
|
|
report forms; take the filing with the latest period end;
|
|
2. fetch the document and locate the fund's Financial Highlights tables;
|
|
3. parse every share-class block (per-share dividends from net investment
|
|
income, distributions from realized capital gains, NAV begin/end,
|
|
total return, for the current period + prior years);
|
|
4. match the class to the ticker by the per-share distribution series
|
|
and by NAV magnitude vs the local year-end closes;
|
|
5. compare official vs local (~/prog/fin/stocks CSVs, frozen copies for
|
|
stale tickers) per year and write a verdict.
|
|
|
|
Verdicts:
|
|
ok best-matching class agrees on every bounded fiscal period
|
|
mismatch the class clearly matches but one or more periods differ
|
|
beyond tolerance -> real data finding; review the period
|
|
(per-date local rows vs the filing) and correct via
|
|
overrides/ (see reports/stale-funds.md conventions)
|
|
weak-match best class fit too poor to trust (uncovered document
|
|
format or wrong fund) -> manual review
|
|
not-found fund not located in any candidate filing
|
|
error fetch/parse failure (see the json for details)
|
|
|
|
Notes:
|
|
- The oldest column has an unbounded local window (all earlier history)
|
|
and is displayed as ~ but never drives the verdict.
|
|
- Distributions are compared ex-date based, in the period window
|
|
(prev period end, period end]; this handles non-calendar fiscal years.
|
|
- ETF/annual-report formats are covered only partially (several ETF
|
|
families come back not-found); open-end N-CSR/N-CSRS formats of the
|
|
Vanguard/Victory style are the well-covered case.
|
|
|
|
Usage:
|
|
python3 scripts/verify_official.py [SYM ...] # default: funds.json
|
|
python3 scripts/verify_official.py --stale # + all frozen tickers
|
|
python3 scripts/verify_official.py --json-only # skip SEC, re-emit report
|
|
|
|
Output: reports/xcheck_official/{SYM}.json + SUMMARY.md
|
|
Politely rate-limited (~1 doc per second). Cached results are re-used
|
|
when the requested period is already covered (no re-fetch).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sys
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
import fundlab.edgar as edgar # noqa: E402 (sec_get with UA + delay)
|
|
|
|
import data as D # noqa: E402
|
|
|
|
ROOT = D.DEFAULT_ROOT
|
|
REPORT_DIR = Path(__file__).resolve().parent.parent / "reports" / "xcheck_official"
|
|
FUNDS = Path(__file__).resolve().parent.parent / "funds.json"
|
|
|
|
EFTS_URL = "https://efts.sec.gov/LATEST/search-index?q={q}&forms=N-CSR,N-CSRS,10-K,10-Q"
|
|
FORMS = ("N-CSR", "N-CSRS", "10-K", "10-Q")
|
|
_STOP = {"fund", "funds", "inc", "ltd", "llc", "trust", "company", "the",
|
|
"and", "of", "for", "class", "series", "shares", "share", "in",
|
|
"on", "at", "to", "a", "an", "with", "portfolio", "portfolios",
|
|
"investment", "investments", "admiral", "investor", "investors",
|
|
"institutional", "etf", "plus", "select", "retail", "wholesale",
|
|
"group", "capital", "global", "us", "u.s."}
|
|
|
|
TOL_REL = 0.015 # 1.5% per year
|
|
TOL_ABS = 0.015 # or $0.015/share (official figures are 3-dp)
|
|
NAV_TOL = 0.25 # class match: |local_close - official_nav| / nav
|
|
|
|
|
|
# ---------------------------------------------------------------- parsing
|
|
MONTHS = ("January|February|March|April|May|June|July|August|September|"
|
|
"October|November|December")
|
|
|
|
|
|
_MON = {m: f"{i + 1:02d}" for i, m in enumerate(
|
|
[x for x in MONTHS.split("|")])}
|
|
|
|
|
|
def _num(window: str) -> Optional[float]:
|
|
c = window.strip().replace("$", "").replace(",", "")
|
|
neg = c.startswith("(") and c.endswith(")")
|
|
c = c.strip("() ")
|
|
try:
|
|
v = float(c)
|
|
except ValueError:
|
|
return None
|
|
return -v if neg else v
|
|
|
|
|
|
# row labels found in fund Financial Highlights tables; used to bound
|
|
# each row's value window so one row's numbers never bleed into the next
|
|
_ROW_LBL = (r"Net Asset Value,\s*(?:Beginning|End) of Period"
|
|
r"|Net Investment Income"
|
|
r"|Capital Gain Distributions Received"
|
|
r"|Net Realized(?:\s+and\s+Unrealized)?"
|
|
r"|Total from Investment Operations"
|
|
r"|Total Distributions?"
|
|
r"|Dividends?\s+from\s+(?:Net\s+)?"
|
|
r"|Distributions?\s+from\s+(?:Net\s+)?"
|
|
r"|Total Return"
|
|
r"|Ratios/Supplemental")
|
|
|
|
|
|
def _decimals(text: str, start: int, end: int, n: int) -> Optional[list]:
|
|
"""The n decimal numbers in `text[start:end]`, in order.
|
|
|
|
Footnote markers (bare integers) and dashes are skipped because only
|
|
values with a decimal point are accepted; negative values may span
|
|
multiple cells ("(.961 | )"), which the pipe-stripping handles.
|
|
"""
|
|
w = re.sub(r"[|\s]+", "", text[start:end])
|
|
w = w.replace("—", "-").replace(" ", "")
|
|
vals = []
|
|
for m in re.finditer(r"\((\d+\.\d+|\.\d+)\)|(\d+\.\d+|\.\d+)", w):
|
|
v = float(m.group(1) or m.group(2))
|
|
vals.append(-v if m.group(1) else v)
|
|
if len(vals) == n:
|
|
return vals
|
|
return None
|
|
|
|
|
|
_MON3 = {"jan": "01", "feb": "02", "mar": "03", "apr": "04", "may": "05",
|
|
"jun": "06", "jul": "07", "aug": "08", "sep": "09", "oct": "10",
|
|
"nov": "11", "dec": "12"}
|
|
|
|
|
|
def _parse_date(s: str) -> Optional[str]:
|
|
"""'June 30, 2026' | '1/31/26' | 'Dec-31-25' -> ISO date."""
|
|
s = re.sub(r"&#\w+;", " ", s).strip()
|
|
m = re.match(rf"((?:{MONTHS}))\.?\s+(\d{{1,2}}),?\s+(\d{{2,4}})", s, re.I)
|
|
if m:
|
|
y = m.group(3) if len(m.group(3)) == 4 else "20" + m.group(3)
|
|
return f"{y}-{_MON[m.group(1).title()]}-{int(m.group(2)):02d}"
|
|
m = re.match(r"(\d{1,2})[/-](\d{1,2})[/-](\d{2,4})", s)
|
|
if m:
|
|
y = m.group(3) if len(m.group(3)) == 4 else "20" + m.group(3)
|
|
if 1 <= int(m.group(1)) <= 12:
|
|
return f"{y}-{int(m.group(1)):02d}-{int(m.group(2)):02d}"
|
|
m = re.match(r"((?:" + "|".join(_MON3) + r"))\.?\s*[-/]\s*(\d{1,2})"
|
|
r"[-/]\s*(\d{2,4})", s, re.I)
|
|
if m:
|
|
y = m.group(3) if len(m.group(3)) == 4 else "20" + m.group(3)
|
|
return f"{y}-{_MON3[m.group(1).lower()]}-{int(m.group(2)):02d}"
|
|
return None
|
|
|
|
|
|
def _header_periods(text: str, i0: int, i_nav: int) -> list[dict]:
|
|
"""Period labels from the header between the class label and the first
|
|
'Net asset value, beginning' row.
|
|
|
|
Handles both styles:
|
|
Six Months Ended | June 30, | 2026 | Year Ended December 31, | 2025
|
|
| 2024 | ... (month-name dates + bare 4-digit years)
|
|
Six Months Ended 1/31/26 | Year Ended 7/31/25 | 7/31/24 | ...
|
|
(M/D/YY dates)
|
|
Bare years inherit the month-day of the preceding dated column.
|
|
"""
|
|
hdr = re.sub(r"&#\w+;", " ", text[i0:i_nav])
|
|
hdr = re.sub(r"[|\s]+", " ", hdr).strip()
|
|
# columns are newest-first and dated/bare-year columns may interleave;
|
|
# scan left to right in a single pass, never sort
|
|
periods = []
|
|
state = {"md": None}
|
|
|
|
def _eat(m):
|
|
g = m.group(0)
|
|
iso = _parse_date(g)
|
|
if iso:
|
|
state["md"] = iso[5:]
|
|
periods.append({"end": iso, "year": iso[:4], "kind": "fy"})
|
|
return " "
|
|
y = g.strip()
|
|
if not (2 <= len(y) <= 4) or not y.isdigit():
|
|
return g
|
|
if len(y) == 2:
|
|
y = "20" + y
|
|
if not (1950 <= int(y) <= 2040):
|
|
return g
|
|
if not state["md"]:
|
|
state["md"] = "12-31"
|
|
periods.append({"end": f"{y}-{state['md']}", "year": y, "kind": "fy"})
|
|
return " "
|
|
|
|
hdr = re.sub(rf"(?:{MONTHS})\.?\s+\d{{1,2}},?\s+\d{{2,4}}"
|
|
rf"|\d{{1,2}}[/-]\d{{1,2}}[/-]\d{{2,4}}"
|
|
r"|\b(19\d{2}|20\d{2}|\d{2})\b",
|
|
_eat, hdr, flags=re.I)
|
|
# drop duplicate ends, keep first
|
|
seen, out = set(), []
|
|
for p in periods:
|
|
if p["end"] in seen:
|
|
continue
|
|
seen.add(p["end"])
|
|
out.append(p)
|
|
if len(out) >= 2 and out[0]["end"] >= out[1]["end"]:
|
|
out[0]["kind"] = "current"
|
|
return out
|
|
|
|
|
|
def parse_highlights(text: str) -> list[dict]:
|
|
"""All share-class Financial Highlights blocks in `text` (pipe-delimited,
|
|
tag-stripped). A block starts at a 'Net asset value, beginning' row
|
|
(both Vanguard's 'Net Asset Value, Beginning of Period' and Victory's
|
|
'Net asset value, beginning of period' style)."""
|
|
starts = [m.start() for m in
|
|
re.finditer(r"net\s+asset\s+value,\s*beginning", text, re.I)]
|
|
blocks = []
|
|
for i, s in enumerate(starts):
|
|
end = starts[i + 1] if i + 1 < len(starts) else s + 25000
|
|
pre = re.sub(r"&#\w+;", " ", text[max(0, s - 900):s])
|
|
cells = [c.strip() for c in pre.split("|") if c.strip()]
|
|
# class label: nearest "... Shares" / "Class X" cell; the cells
|
|
# right before the row are often column years, not the label
|
|
label = "?"
|
|
for c in reversed(cells):
|
|
if re.fullmatch(r"[A-Za-z][A-Za-z0-9 \-]*\s+Shares", c) or \
|
|
re.fullmatch(r"Class\s+[A-Z0-9]+", c):
|
|
label = re.sub(r"\s+", " ", c).strip()
|
|
break
|
|
# header window: back at most 2000 chars, but never across a
|
|
# "Table of Contents" page footer (its page numbers would be read
|
|
# as bare year columns)
|
|
h0 = max(0, s - 2000)
|
|
toc = text.rfind("Table of Contents", h0, s)
|
|
fh = text.rfind("Financial Highlights", h0, s)
|
|
h0 = max(h0, toc, fh)
|
|
periods = _header_periods(text, h0, s)
|
|
if len(periods) < 2:
|
|
continue
|
|
n = len(periods)
|
|
i_end = min(end, text.find("See accompanying", s) if
|
|
text.find("See accompanying", s) != -1 else end)
|
|
|
|
def row(pattern: str) -> Optional[list]:
|
|
mm = re.search(pattern, text[s:i_end], re.I)
|
|
if not mm:
|
|
return None
|
|
i0, i1 = s + mm.end(), i_end
|
|
for b in [x.start() for x in re.finditer(_ROW_LBL,
|
|
text[s:i_end], re.I)]:
|
|
if b > mm.end():
|
|
i1 = s + b
|
|
break
|
|
return _decimals(text, i0, i1, n)
|
|
|
|
# distribution rows: prefer explicit "from net investment income" /
|
|
# "from realized capital gains" labels (Vanguard); otherwise the
|
|
# "Distributions to shareholders:" sub-table (Victory), where the
|
|
# NII row is the first "Net investment income" after that header
|
|
ni = row(r"dividends?\s+from\s+(?:net\s+)?investment\s+income")
|
|
cg = row(r"(?:distributions?|dividends?)\s+from\s+(?:net\s+)?realized\s+capital\s+gains?")
|
|
if ni is None:
|
|
dm = re.search(r"distributions?\s+to\s+shareholders", text[s:i_end], re.I)
|
|
if dm:
|
|
mm = re.search(r"net\s+investment\s+income", text[s + dm.end():i_end], re.I)
|
|
if mm:
|
|
i0 = s + dm.end() + mm.end()
|
|
ni = _decimals(text, i0, i_end, n)
|
|
tot = row(r"total\s+distributions?\b")
|
|
if ni is None and tot is None:
|
|
continue
|
|
if tot is None:
|
|
if ni is None or cg is None:
|
|
continue
|
|
tot = [round(abs(a) + abs(b), 4) for a, b in zip(ni, cg)]
|
|
# block values: this block's NAV rows
|
|
navb = row(r"net\s+asset\s+value,\s*beginning")
|
|
nave = row(r"net\s+asset\s+value,\s*end")
|
|
blocks.append({"class": label, "periods": periods,
|
|
"nii": [abs(v) for v in ni] if ni else None,
|
|
"capg": [abs(v) for v in cg] if cg else None,
|
|
"total": [abs(v) for v in tot],
|
|
"nav_begin": navb, "nav_end": nave})
|
|
return blocks
|
|
|
|
|
|
def html_to_cells_text(html: bytes) -> str:
|
|
t = re.sub(rb"<\s*(br|/p|/div|/tr|/h[1-6])[^>]*>", b"|\n", html, flags=re.I)
|
|
t = re.sub(rb"<[^>]+>", b"|", t)
|
|
t = t.decode("utf-8", "replace")
|
|
t = re.sub(r" ?;", " ", t)
|
|
return t
|
|
|
|
|
|
# ---------------------------------------------------------------- EFTS
|
|
def _efits(q: str) -> list[dict]:
|
|
"""EFTS hits -> [{accession, doc, period_end, display, ciks, score}]."""
|
|
import urllib.parse
|
|
d = json.loads(edgar.sec_get(EFTS_URL.format(q=urllib.parse.quote(q)))
|
|
.decode("utf-8", "replace"))
|
|
out, seen = [], set()
|
|
for h in d.get("hits", {}).get("hits", [])[:20]:
|
|
src = h.get("_source", {})
|
|
acc, _, doc = h.get("_id", "").partition(":")
|
|
if acc in seen or not doc.lower().endswith((".htm", ".html")):
|
|
continue
|
|
seen.add(acc)
|
|
out.append({"accession": acc, "doc": doc,
|
|
"period_end": src.get("period_ending") or "",
|
|
"display": src.get("display_names", ["?"])[0],
|
|
"ciks": src.get("ciks", []),
|
|
"score": h.get("_score", 0)})
|
|
out.sort(key=lambda f: f["period_end"], reverse=True)
|
|
return out
|
|
|
|
|
|
def find_filings(ticker: str, name: str, limit: int = 6) -> list[dict]:
|
|
"""Candidate filings, best first. A bare ticker matches hundreds of
|
|
unrelated documents, so: (1) the full fund name as one phrase (it only
|
|
appears in the fund's own filings), (2) ticker + distinctive name
|
|
words, (3) the bare ticker."""
|
|
def merge(f: list[dict]) -> None:
|
|
for x in f:
|
|
if x["accession"] not in {y["accession"] for y in out}:
|
|
out.append(x)
|
|
out: list[dict] = []
|
|
words = [w for w in re.findall(r"[A-Za-z][A-Za-z\-'.]{2,}", name)
|
|
if w.lower() not in _STOP]
|
|
if len(words) >= 4:
|
|
merge(_efits('"' + name + '"'))
|
|
if len(out) < limit and words:
|
|
merge(_efits('"' + ticker + '" AND ' +
|
|
" AND ".join('"' + w + '"' for w in words[:3])))
|
|
if len(out) < limit:
|
|
merge(_efits('"' + ticker + '"'))
|
|
return out[:limit]
|
|
|
|
|
|
def _title_runs(seg: str) -> list[str]:
|
|
return re.findall(r"[A-Z][A-Za-z0-9'\u2019\-]*"
|
|
r"(?:\s+[A-Z][A-Za-z0-9'\u2019\-]*)+", seg)
|
|
|
|
|
|
def fund_region(text: str, ticker: str, name: str) -> Optional[str]:
|
|
"""The slice of the document holding this fund's highlights tables.
|
|
|
|
The ticker appears only in the TOC/cover, not in the financial
|
|
statements. The TOC cell "<Class> Shares - TICKER" is preceded by the
|
|
official fund name cell; section headers in the statements drop
|
|
prefixes like "Vanguard ", so anchor on the name or a tail of it.
|
|
"""
|
|
occs = [m.start() for m in re.finditer(rf"(?<![A-Z0-9]){ticker}(?![A-Z0-9])",
|
|
text, re.I)]
|
|
if not occs:
|
|
return None
|
|
cands: list[str] = []
|
|
for occ in occs[:3]:
|
|
cells = [re.sub(r"&#\w+;|\s+", " ", c).strip()
|
|
for c in text[occ - 800:occ + 200].split("|")]
|
|
for i, c in enumerate(cells):
|
|
# the ticker cell: bare "PMAIX", "Shares - VTSAX", "Class A / PMAIX"...
|
|
if not re.search(rf"(?<![A-Z0-9]){re.escape(ticker)}(?![A-Z0-9])",
|
|
c, re.I):
|
|
continue
|
|
for prev in reversed(cells[:i]):
|
|
if prev and "shares" not in prev.lower() \
|
|
and "class" not in prev.lower() \
|
|
and len(prev.split()) >= 3:
|
|
cands.append(prev)
|
|
break
|
|
cands.append(name)
|
|
for cand in dict.fromkeys(cands):
|
|
words = cand.split()
|
|
if len(words) < 4:
|
|
continue
|
|
for tail_n in range(len(words), 3, -1):
|
|
anchor = " ".join(words[-tail_n:])
|
|
for m in re.finditer(re.escape(anchor), text, re.I):
|
|
fh = text.find("Financial Highlights", m.start(), m.start() + 4000)
|
|
if fh == -1:
|
|
continue
|
|
reg = text[fh:fh + 150000]
|
|
if parse_highlights(reg):
|
|
return reg
|
|
return None
|
|
|
|
|
|
def _wordset(s: str) -> set:
|
|
return set(re.findall(r"[a-z0-9]{3,}", s.lower())) - _STOP
|
|
|
|
|
|
def region_by_name(text: str, name: str, min_shared: int = 3):
|
|
"""Fallback when the ticker is absent from the document (families that
|
|
don't publish distribution tickers): match the fund by name — the cell
|
|
immediately before each Financial Highlights heading — and keep the
|
|
best word-overlap candidate with a parseable region."""
|
|
target = _wordset(name)
|
|
best = None
|
|
for m in re.finditer(r"Financial Highlights", text):
|
|
pre = re.sub(r"&#\w+;", " ", text[max(0, m.start() - 400):m.start()])
|
|
pre = re.sub(r"[\s|]+", " ", pre).strip()
|
|
runs = re.findall(r"[A-Z][A-Za-z0-9'\u2019\-]*"
|
|
r"(?: [A-Z][A-Za-z0-9'\u2019\-]*){2,11}", pre)
|
|
if not runs:
|
|
continue
|
|
for cand in dict.fromkeys(runs): # any run, best overlap wins
|
|
shared = len(_wordset(cand) & target)
|
|
if shared < min_shared:
|
|
continue
|
|
reg = text[m.start():m.start() + 150000]
|
|
if not parse_highlights(reg):
|
|
continue
|
|
if best is None or shared > best[0]:
|
|
best = (shared, reg, cand)
|
|
return best
|
|
|
|
|
|
# ---------------------------------------------------------------- compare
|
|
def local_series(sym: str, root: Path, frozen: Path) -> dict:
|
|
"""Local (date, amount) distributions and close series; frozen copies
|
|
shadow the data root."""
|
|
def read(sfx: str):
|
|
p = frozen / f"{sym}-{sfx}.csv"
|
|
if not p.exists():
|
|
p = root / f"{sym}-{sfx}.csv"
|
|
if not p.exists():
|
|
return []
|
|
out = []
|
|
for line in p.read_text(errors="replace").splitlines()[1:]:
|
|
parts = line.split(",")
|
|
if len(parts) >= 2:
|
|
out.append((parts[0][:10], float(parts[1])))
|
|
return out
|
|
dist = sorted(read("dividend") + read("capitalGain"))
|
|
p = frozen / f"{sym}-history.csv"
|
|
if not p.exists():
|
|
p = root / f"{sym}-history.csv"
|
|
closes = []
|
|
if p.exists():
|
|
for line in p.read_text(errors="replace").splitlines()[1:]:
|
|
parts = line.split(",")
|
|
if len(parts) >= 5:
|
|
closes.append((parts[0][:10], float(parts[4])))
|
|
return {"dist": dist, "close": closes}
|
|
|
|
|
|
def _window_sum(dist, lo, hi):
|
|
"""Sum of distributions with lo < date <= hi (ISO date strings)."""
|
|
return round(sum(v for d, v in dist if (lo is None or d > lo) and d <= hi),
|
|
4)
|
|
|
|
|
|
def compare(sym: str, blocks: list[dict], loc: dict) -> dict:
|
|
"""Pick the best-matching class and compare period by period.
|
|
|
|
Each official column covers (previous period end, this period end];
|
|
local distributions are summed over the same window, so non-calendar
|
|
fiscal years compare correctly.
|
|
"""
|
|
def period_errs(b):
|
|
errs = []
|
|
for i, p in enumerate(b["periods"]):
|
|
if p["kind"] != "fy":
|
|
continue
|
|
if i + 1 >= len(b["periods"]):
|
|
continue # oldest column: unbounded window, unverifiable
|
|
off = abs(b["total"][i])
|
|
lo = b["periods"][i + 1]["end"]
|
|
lc = _window_sum(loc["dist"], lo, p["end"])
|
|
errs.append(abs(off - lc) / max(off, lc, 1e-9))
|
|
return errs
|
|
|
|
best = None
|
|
for b in blocks:
|
|
errs = period_errs(b)
|
|
if not errs:
|
|
continue
|
|
me = sum(errs) / len(errs)
|
|
if best is None or me < best["err"]:
|
|
best = {"err": me, "block": b}
|
|
if best is None:
|
|
return {"verdict": "not-found", "classes": len(blocks), "detail":
|
|
"no class block with overlapping periods"}
|
|
b = best["block"]
|
|
rows = []
|
|
mismatch = False
|
|
for i, p in enumerate(b["periods"]):
|
|
off = abs(b["total"][i])
|
|
# columns are newest-first: the window lower bound is the NEXT
|
|
# (older) column's end; the oldest column is unbounded
|
|
lo = b["periods"][i + 1]["end"] if i + 1 < len(b["periods"]) else None
|
|
lc = _window_sum(loc["dist"], lo, p["end"])
|
|
d = off - lc
|
|
ok = abs(d) <= TOL_ABS or abs(d) / max(off, 1e-9) <= TOL_REL
|
|
mismatch |= p["kind"] == "fy" and lo is not None and not ok
|
|
rows.append({"period": p["kind"], "end": p["end"],
|
|
"official": round(off, 4), "local": lc,
|
|
"diff": round(d, 4), "ok": ok,
|
|
"unbounded": lo is None})
|
|
# NAV check: most recent period with an official ending NAV
|
|
nav = None
|
|
if b.get("nav_end") and loc["close"]:
|
|
closes = dict(loc["close"])
|
|
for i in range(len(b["periods"]) - 1, -1, -1):
|
|
p = b["periods"][i]
|
|
prior = [d for d in closes if d <= p["end"]]
|
|
if not prior:
|
|
continue
|
|
o, l = abs(b["nav_end"][i]), closes[prior[-1]]
|
|
nav = {"date": prior[-1], "official": o, "local": l,
|
|
"rel": round(abs(o - l) / o, 4),
|
|
"ok": abs(o - l) / o <= NAV_TOL}
|
|
break
|
|
# verdict: a good class fit with a bad year is a real mismatch; a bad
|
|
# class fit overall means the document format (or fund) isn't matched
|
|
if best["err"] >= 0.30:
|
|
verdict = "weak-match"
|
|
else:
|
|
verdict = "mismatch" if mismatch else "ok"
|
|
return {"verdict": verdict,
|
|
"class": b["class"], "mean_rel_err": round(best["err"], 4),
|
|
"nav_check": nav, "periods": rows,
|
|
"other_classes": [x["class"] for x in blocks if x is not b]}
|
|
|
|
|
|
# ---------------------------------------------------------------- driver
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("syms", nargs="*")
|
|
ap.add_argument("--stale", action="store_true",
|
|
help="also verify all frozen (stale) tickers")
|
|
ap.add_argument("--json-only", action="store_true")
|
|
ap.add_argument("--limit", type=int, default=0)
|
|
args = ap.parse_args()
|
|
|
|
syms = [s.lower() for s in args.syms]
|
|
if not syms and not args.stale:
|
|
syms = list(json.loads(FUNDS.read_text()).keys())
|
|
if args.stale:
|
|
syms += sorted(f.stem for f in (D.FROZEN_DIR.glob("*.json")))
|
|
syms = list(dict.fromkeys(syms))
|
|
if args.limit:
|
|
syms = syms[:args.limit]
|
|
|
|
REPORT_DIR.mkdir(parents=True, exist_ok=True)
|
|
summary = []
|
|
t0 = time.monotonic()
|
|
for n, sym in enumerate(syms, 1):
|
|
out = REPORT_DIR / f"{sym}.json"
|
|
cached = None
|
|
if out.exists() and not args.json_only:
|
|
cached = json.loads(out.read_text())
|
|
if not (cached.get("filing") or cached.get("period_end")):
|
|
cached = None # not-found from a prior run: retry
|
|
if cached:
|
|
summary.append(cached)
|
|
print(f"[{n}/{len(syms)}] {sym} cached {cached['verdict']}", flush=True)
|
|
continue
|
|
name = ""
|
|
meta_f = ROOT / f"{sym}.json"
|
|
if meta_f.exists():
|
|
try:
|
|
name = json.loads(meta_f.read_text())["chart"]["result"][0] \
|
|
.get("meta", {}).get("longName", "")
|
|
except Exception:
|
|
pass
|
|
if not name:
|
|
mf = D.FROZEN_DIR / f"{sym}.json"
|
|
if mf.exists():
|
|
name = json.loads(mf.read_text()).get("longName", "")
|
|
try:
|
|
if args.json_only:
|
|
raise RuntimeError("json-only")
|
|
cands = find_filings(sym.upper(), name)
|
|
rec = {"symbol": sym, "name": name}
|
|
for filing in cands:
|
|
try:
|
|
doc = edgar.sec_get(
|
|
f"https://www.sec.gov/Archives/edgar/data/"
|
|
f"{int(filing['ciks'][0]):010d}/{filing['accession'].replace('-', '')}/"
|
|
f"{filing['doc']}")
|
|
except Exception:
|
|
continue
|
|
text = html_to_cells_text(doc)
|
|
region, how = None, None
|
|
if re.search(rf"(?<![A-Z0-9]){re.escape(sym.upper())}(?![A-Z0-9])",
|
|
text):
|
|
region = fund_region(text, sym.upper(), name)
|
|
how = "ticker"
|
|
if region is None:
|
|
rb = region_by_name(text, name)
|
|
if rb:
|
|
region, how = rb[1], f"name:{rb[2]} (shared={rb[0]})"
|
|
if region is None:
|
|
continue # wrong document; try next candidate
|
|
rec["filing"] = filing
|
|
rec["how"] = how
|
|
rec["n_bytes"] = len(doc)
|
|
blocks = parse_highlights(region)
|
|
loc = local_series(sym, ROOT, D.FROZEN_DIR)
|
|
res = compare(sym, blocks, loc)
|
|
rec.update(res)
|
|
rec["verdict"] = res["verdict"]
|
|
break
|
|
else:
|
|
rec["verdict"] = "not-found"
|
|
rec["detail"] = ("no candidate filing contained this fund "
|
|
f"({len(cands)} tried)")
|
|
except Exception as e:
|
|
rec = {"symbol": sym, name: name, "verdict": "error",
|
|
"detail": str(e)[:300]}
|
|
out.write_text(json.dumps(rec, indent=1))
|
|
summary.append(rec)
|
|
print(f"[{n}/{len(syms)}] {sym} {rec.get('verdict')} "
|
|
f"({time.monotonic() - t0:.0f}s)", flush=True)
|
|
|
|
# summary covers EVERY record in the report dir, not just this batch
|
|
recs = []
|
|
for f in sorted(REPORT_DIR.glob("*.json")):
|
|
try:
|
|
recs.append(json.loads(f.read_text()))
|
|
except Exception:
|
|
pass
|
|
lines = ["# Official-filing cross-check (SEC N-CSR/N-CSRS/10-K/10-Q)\n",
|
|
"Verdicts: ok = all bounded fiscal periods agree; mismatch = class matched "
|
|
"but some period differs (review the year, then correct via overrides/); "
|
|
"weak-match = best class fit too poor to trust (format/fund mismatch); "
|
|
"not-found = fund not located in any candidate filing.\n",
|
|
"| Symbol | Verdict | Class | Periods (fy end, ~ = unbounded) | Detail |",
|
|
"|---|---|---|---|---|"]
|
|
for rec in recs:
|
|
yrs = ""
|
|
if rec.get("periods"):
|
|
yrs = " ".join(
|
|
((f"{y['end'][:7]}~") if y.get("unbounded")
|
|
else f"{y['end'][:7]}{'✓' if y['ok'] else '✗'}"
|
|
for y in rec["periods"] if y["period"] == "fy"))
|
|
lines.append(f"| {rec['symbol'].upper()} | {rec.get('verdict')} | "
|
|
f"{rec.get('class', '')} | {yrs} | "
|
|
f"{rec.get('detail', '')[:80]} |")
|
|
(REPORT_DIR / "SUMMARY.md").write_text("\n".join(lines) + "\n")
|
|
from collections import Counter
|
|
c = Counter(r.get("verdict", "?") for r in summary)
|
|
print(f"\n{dict(c)} ({time.monotonic() - t0:.0f}s) -> {REPORT_DIR / 'SUMMARY.md'}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|