#!/usr/bin/env python3
"""Tier-3 verification: cross-check local Yahoo distributions against
official SEC filings (N-CSR / N-CSRS / 10-K / 10-Q financial statements).

For each symbol:
  1. EDGAR full-text search for the ticker restricted to annual/semi-annual
     report forms; take the filing with the latest period end;
  2. fetch the document and locate the fund's Financial Highlights tables;
  3. parse every share-class block (per-share dividends from net investment
     income, distributions from realized capital gains, NAV begin/end,
     total return, for the current period + prior years);
  4. match the class to the ticker by the per-share distribution series
     and by NAV magnitude vs the local year-end closes;
  5. compare official vs local (~/prog/fin/stocks CSVs, frozen copies for
     stale tickers) per year and write a verdict.

Verdicts:
  ok          best-matching class agrees on every bounded fiscal period
  mismatch    the class clearly matches but one or more periods differ
              beyond tolerance -> real data finding; review the period
              (per-date local rows vs the filing) and correct via
              overrides/ (see reports/stale-funds.md conventions)
  weak-match  best class fit too poor to trust (uncovered document
              format or wrong fund) -> manual review
  not-found   fund not located in any candidate filing
  error       fetch/parse failure (see the json for details)

Notes:
  - The oldest column has an unbounded local window (all earlier history)
    and is displayed as ~ but never drives the verdict.
  - Distributions are compared ex-date based, in the period window
    (prev period end, period end]; this handles non-calendar fiscal years.
  - ETF/annual-report formats are covered only partially (several ETF
    families come back not-found); open-end N-CSR/N-CSRS formats of the
    Vanguard/Victory style are the well-covered case.

Usage:
  python3 scripts/verify_official.py [SYM ...]   # default: funds.json
  python3 scripts/verify_official.py --stale     # + all frozen tickers
  python3 scripts/verify_official.py --json-only # skip SEC, re-emit report

Output: reports/xcheck_official/{SYM}.json + SUMMARY.md
Politely rate-limited (~1 doc per second). Cached results are re-used
when the requested period is already covered (no re-fetch).
"""

from __future__ import annotations

import argparse
import json
import re
import sys
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

import fundlab.edgar as edgar  # noqa: E402  (sec_get with UA + delay)

import data as D  # noqa: E402

ROOT = D.DEFAULT_ROOT
REPORT_DIR = Path(__file__).resolve().parent.parent / "reports" / "xcheck_official"
FUNDS = Path(__file__).resolve().parent.parent / "funds.json"

EFTS_URL = "https://efts.sec.gov/LATEST/search-index?q={q}&forms=N-CSR,N-CSRS,10-K,10-Q,497,497K"
FORMS = ("N-CSR", "N-CSRS", "10-K", "10-Q", "497", "497K")
_STOP = {"fund", "funds", "inc", "ltd", "llc", "trust", "company", "the",
         "and", "of", "for", "class", "series", "shares", "share", "in",
         "on", "at", "to", "a", "an", "with", "portfolio", "portfolios",
         "investment", "investments", "admiral", "investor", "investors",
         "institutional", "etf", "plus", "select", "retail", "wholesale",
         "group", "capital", "global", "us", "u.s."}

# 'Financial Highlights' often sits in separate table cells
# ("Financial |Highlights"); tolerate pipes/whitespace between the words
FH = r"Financial[\s|]+Highlights"
TOL_REL = 0.015     # 1.5% per year
TOL_ABS = 0.015     # or $0.015/share (official figures are 3-dp)
NAV_TOL = 0.25      # class match: |local_close - official_nav| / nav


# ---------------------------------------------------------------- parsing
MONTHS = ("January|February|March|April|May|June|July|August|September|"
          "October|November|December")


_MON = {m: f"{i + 1:02d}" for i, m in enumerate(
    [x for x in MONTHS.split("|")])}


def _num(window: str) -> Optional[float]:
    c = window.strip().replace("$", "").replace(",", "")
    neg = c.startswith("(") and c.endswith(")")
    c = c.strip("() ")
    try:
        v = float(c)
    except ValueError:
        return None
    return -v if neg else v


# row labels found in fund Financial Highlights tables; used to bound
# each row's value window so one row's numbers never bleed into the next
_ROW_LBL = (r"Net Asset Value,\s*(?:Beginning|End) of Period"
            r"|Net Investment Income"
            r"|Capital Gain Distributions Received"
            r"|Net Realized(?:\s+and\s+Unrealized)?"
            r"|Total from Investment Operations"
            r"|Total Distributions?"
            r"|Dividends?\s+from\s+(?:Net\s+)?"
            r"|Distributions?\s+from\s+(?:Net\s+)?"
            r"|Total Return"
            r"|Ratios/Supplemental")


def _decimals(text: str, start: int, end: int, n: int) -> Optional[list]:
    """The n decimal numbers in `text[start:end]`, in order.

    Footnote markers (bare integers) and dashes are skipped because only
    values with a decimal point are accepted; negative values may span
    multiple cells ("(.961 | )"), which the pipe-stripping handles.
    """
    w = re.sub(r"[|\s]+", "", text[start:end])
    w = w.replace("—", "-").replace(" ", "")
    vals = []
    for m in re.finditer(r"\((\d+\.\d+|\.\d+)\)|(\d+\.\d+|\.\d+)", w):
        v = float(m.group(1) or m.group(2))
        vals.append(-v if m.group(1) else v)
        if len(vals) == n:
            return vals
    return None


_MON3 = {"jan": "01", "feb": "02", "mar": "03", "apr": "04", "may": "05",
         "jun": "06", "jul": "07", "aug": "08", "sep": "09", "oct": "10",
         "nov": "11", "dec": "12"}


def _parse_date(s: str) -> Optional[str]:
    """'June 30, 2026' | '1/31/26' | 'Dec-31-25' -> ISO date."""
    s = re.sub(r"&#\w+;", " ", s).strip()
    m = re.match(rf"((?:{MONTHS}))\.?\s+(\d{{1,2}}),?\s+(\d{{2,4}})", s, re.I)
    if m:
        y = m.group(3) if len(m.group(3)) == 4 else "20" + m.group(3)
        return f"{y}-{_MON[m.group(1).title()]}-{int(m.group(2)):02d}"
    m = re.match(r"(\d{1,2})[/-](\d{1,2})[/-](\d{2,4})", s)
    if m:
        y = m.group(3) if len(m.group(3)) == 4 else "20" + m.group(3)
        if 1 <= int(m.group(1)) <= 12:
            return f"{y}-{int(m.group(1)):02d}-{int(m.group(2)):02d}"
    m = re.match(r"((?:" + "|".join(_MON3) + r"))\.?\s*[-/]\s*(\d{1,2})"
                 r"[-/]\s*(\d{2,4})", s, re.I)
    if m:
        y = m.group(3) if len(m.group(3)) == 4 else "20" + m.group(3)
        return f"{y}-{_MON3[m.group(1).lower()]}-{int(m.group(2)):02d}"
    return None


def _header_periods(text: str, i0: int, i_nav: int) -> list[dict]:
    """Period labels from the header between the class label and the first
    'Net asset value, beginning' row.

    Handles both styles:
      Six Months Ended | June 30, | 2026 | Year Ended December 31, | 2025
      | 2024 | ...          (month-name dates + bare 4-digit years)
      Six Months Ended 1/31/26 | Year Ended 7/31/25 | 7/31/24 | ...
      (M/D/YY dates)
    Bare years inherit the month-day of the preceding dated column.
    """
    hdr = re.sub(r"&#\w+;", " ", text[i0:i_nav])
    hdr = re.sub(r"[|\s]+", " ", hdr).strip()
    # columns are newest-first and dated/bare-year columns may interleave;
    # scan left to right in a single pass, never sort
    periods = []
    state = {"md": None}

    def _eat(m):
        g = m.group(0)
        iso = _parse_date(g)
        if iso:
            state["md"] = iso[5:]
            periods.append({"end": iso, "year": iso[:4], "kind": "fy"})
            return " "
        y = g.strip()
        if not (2 <= len(y) <= 4) or not y.isdigit():
            return g
        if len(y) == 2:
            y = "20" + y
        if not (1950 <= int(y) <= 2040):
            return g
        if not state["md"]:
            state["md"] = "12-31"
        periods.append({"end": f"{y}-{state['md']}", "year": y, "kind": "fy"})
        return " "

    hdr = re.sub(rf"(?:{MONTHS})\.?\s+\d{{1,2}},?\s+\d{{2,4}}"
                 rf"|\d{{1,2}}[/-]\d{{1,2}}[/-]\d{{2,4}}"
                 r"|\b(19\d{2}|20\d{2}|\d{2})\b",
                 _eat, hdr, flags=re.I)
    # drop duplicate ends, keep first
    seen, out = set(), []
    for p in periods:
        if p["end"] in seen:
            continue
        seen.add(p["end"])
        out.append(p)
    if len(out) >= 2 and out[0]["end"] >= out[1]["end"]:
        out[0]["kind"] = "current"
    return out


def parse_highlights(text: str) -> list[dict]:
    """All share-class Financial Highlights blocks in `text` (pipe-delimited,
    tag-stripped). A block starts at a 'Net asset value, beginning' row
    (both Vanguard's 'Net Asset Value, Beginning of Period' and Victory's
    'Net asset value, beginning of period' style)."""
    # Row labels are often split across table cells ("Net | asset | value,
    # beginning"); pattern searches run on a pipe-flattened copy (1:1
    # offsets). Cell-based label extraction below uses the original.
    flat = text.replace("|", " ")
    starts = [m.start() for m in
              re.finditer(r"net\s+asset\s+value,\s*beginning", flat, re.I)]
    blocks = []
    for i, s in enumerate(starts):
        end = starts[i + 1] if i + 1 < len(starts) else s + 25000
        pre = re.sub(r"&#\w+;", " ", text[max(0, s - 900):s])
        cells = [c.strip() for c in pre.split("|") if c.strip()]
        # class label: nearest "... Shares" / "Class X" cell; the cells
        # right before the row are often column years, not the label
        label = "?"
        for c in reversed(cells):
            if re.fullmatch(r"[A-Za-z][A-Za-z0-9 \-]*\s+Shares", c) or \
                    re.fullmatch(r"Class\s+[A-Z0-9]+", c):
                label = re.sub(r"\s+", " ", c).strip()
                break
        # header window: back at most 2000 chars, but never across a
        # "Table of Contents" page footer (its page numbers would be read
        # as bare year columns)
        h0 = max(0, s - 2000)
        toc = flat.rfind("Table of Contents", h0, s)
        fh = max([m.end() for m in re.finditer(FH, flat[h0:s])] or [h0]) + h0
        h0 = max(h0, toc, fh)
        periods = _header_periods(flat, h0, s)
        if len(periods) < 2:
            continue
        n = len(periods)
        i_end = min(end, flat.find("See accompanying", s) if
                    flat.find("See accompanying", s) != -1 else end)

        def row(pattern: str) -> Optional[list]:
            mm = re.search(pattern, flat[s:i_end], re.I)
            if not mm:
                return None
            i0, i1 = s + mm.end(), i_end
            for b in [x.start() for x in re.finditer(_ROW_LBL,
                                                     flat[s:i_end], re.I)]:
                if b > mm.end():
                    i1 = s + b
                    break
            return _decimals(flat, i0, i1, n)

        # distribution rows: prefer explicit "from net investment income" /
        # "from realized capital gains" labels (Vanguard); otherwise the
        # "Distributions to shareholders:" sub-table (Victory), where the
        # NII row is the first "Net investment income" after that header
        ni = row(r"dividends?\s+from\s+(?:net\s+)?investment\s+income")
        cg = row(r"(?:distributions?|dividends?)\s+from\s+(?:net\s+)?realized\s+(?:capital\s+)?gains?")
        if ni is None:
            dm = re.search(r"distributions?\s+to\s+shareholders", flat[s:i_end], re.I)
            if dm:
                mm = re.search(r"net\s+investment\s+income", flat[s + dm.end():i_end], re.I)
                if mm:
                    i0 = s + dm.end() + mm.end()
                    ni = _decimals(flat, i0, i_end, n)
        tot = row(r"total\s+distributions?\b")
        if ni is None and tot is None:
            continue
        if tot is None:
            if ni is None or cg is None:
                continue
            tot = [round(abs(a) + abs(b), 4) for a, b in zip(ni, cg)]
        # block values: this block's NAV rows
        navb = row(r"net\s+asset\s+value,\s*beginning")
        nave = row(r"net\s+asset\s+value,\s*end")
        blocks.append({"class": label, "periods": periods,
                       "nii": [abs(v) for v in ni] if ni else None,
                       "capg": [abs(v) for v in cg] if cg else None,
                       "total": [abs(v) for v in tot],
                       "nav_begin": navb, "nav_end": nave})
    return blocks


def _cellnum(c: str) -> Optional[float]:
    """One table cell -> number. Dashes (a zero with a footnote) count as
    0.0; footnote-only and junk cells are None."""
    c = c.replace("$", "").replace(",", "").replace("&#160;", " ").strip()
    if not c:
        return None
    if c.startswith("—") or c in ("-", "–"):
        return 0.0
    m = re.fullmatch(r"(\()?(\d+\.\d+|\.\d+)(\))?(\s*\([a-z]\))?", c)
    if m:
        return -float(m.group(2)) if m.group(1) else float(m.group(2))
    return None


def parse_per_share_blocks(text: str) -> list[dict]:
    """JPMorgan-style shareholder-report layout ('Per share operating
    performance'): each class block lists, per 'Year Ended <date>' column,
    NAV beginning, NII, realized/unrealized gain, total from operations,
    distribution NII, distribution capital gain, total distributions.
    Cell-based (not _decimals): dash columns are real zeros, and the value
    window must not run into the next year's column. Returns blocks
    compatible with parse_highlights()."""
    cells = [re.sub(r"\s+", " ", c).strip().replace("&#8212;", "—")
             .replace("&#8195;", "").replace("&#8201;", "")
             for c in text.split("|")]
    events: list[tuple] = []
    for i, c in enumerate(cells):
        m = re.fullmatch(r"Year[s]?\s+Ended\s+(.+)", c)
        if m and _parse_date(m.group(1)):
            events.append((i, "year", m.group(1)))
        elif re.fullmatch(r"Class\s+[A-Z0-9]{1,3}", c):
            events.append((i, "class", c))
    blocks: list[dict] = []
    cur = None
    for j, (i, kind, val) in enumerate(events):
        if kind == "class":
            cur = {"class": val, "periods": [], "nii": [], "capg": [],
                   "total": [], "nav_begin": [], "nav_end": None}
            blocks.append(cur)
            continue
        if cur is None:
            continue
        d = _parse_date(val)
        stop = events[j + 1][0] if j + 1 < len(events) else len(cells)
        vals: list[float] = []
        for c in cells[i + 1:stop]:
            if re.fullmatch(r"Year[s]?\s+Ended\s+.+", c):
                break
            v = _cellnum(c)
            if v is None:
                continue
            vals.append(v)
            if len(vals) == 7:
                break
        if len(vals) != 7:
            continue
        cur["periods"].append({"end": d, "year": d[:4], "kind": "fy"})
        cur["nav_begin"].append(abs(vals[0]))
        cur["nii"].append(abs(vals[4]))
        cur["capg"].append(abs(vals[5]))
        cur["total"].append(abs(vals[6]))
    return [b for b in blocks if len(b["periods"]) >= 2]


def html_to_cells_text(html: bytes) -> str:
    t = re.sub(rb"<\s*(br|/p|/div|/tr|/h[1-6])[^>]*>", b"|\n", html, flags=re.I)
    t = re.sub(rb"<[^>]+>", b"|", t)
    t = t.decode("utf-8", "replace")
    # numeric entities (&#8239; narrow nbsp, &#160; nbsp, ...) break literal
    # searches like "Financial Highlights"; keep dashes (zero markers),
    # collapse the rest to plain spaces
    t = (t.replace("&#8212;", "—").replace("&#8211;", "–")
          .replace("&mdash;", "—").replace("&ndash;", "–"))
    t = re.sub(r"&#\d+;", " ", t)
    t = re.sub(r"&[a-z]+;", " ", t)
    return t


# ---------------------------------------------------------------- EFTS
def _efits(q: str) -> list[dict]:
    """EFTS hits -> [{accession, doc, period_end, display, ciks, score}]."""
    import urllib.parse
    d = json.loads(edgar.sec_get(EFTS_URL.format(q=urllib.parse.quote(q)))
                  .decode("utf-8", "replace"))
    out, seen = [], set()
    # EFTS returns up to 100; small filings (497K notices) rank above the
    # large combined reports, so keep the whole page and let the
    # find_filings re-ranking pick the real thing
    for h in d.get("hits", {}).get("hits", [])[:100]:
        src = h.get("_source", {})
        acc, _, doc = h.get("_id", "").partition(":")
        if acc in seen or not doc.lower().endswith((".htm", ".html")):
            continue
        seen.add(acc)
        out.append({"accession": acc, "doc": doc,
                    # 497s carry no period_ending; file_date keeps them
                    # orderable
                    "period_end": src.get("period_ending")
                    or src.get("file_date") or "",
                    "display": src.get("display_names", ["?"])[0],
                    "ciks": src.get("ciks", []),
                    "score": h.get("_score", 0)})
    out.sort(key=lambda f: f["period_end"], reverse=True)
    return out


def _brand(name: str) -> Optional[str]:
    """First distinctive word of the fund name, usually the family brand
    (iShares, Fidelity, JPMorgan, SPDR, Invesco...)."""
    for w in re.findall(r"[A-Za-z][A-Za-z\-'.]{2,}", name):
        if w.lower() not in _STOP:
            return w
    return None


def find_filings(ticker: str, name: str, limit: int = 6) -> list[dict]:
    """Candidate filings, best first. A bare ticker matches hundreds of
    unrelated documents, so: (1) the full fund name as one phrase, (2)
    ticker + family brand, (3) ticker + distinctive name words, (4) the
    bare ticker. Results are then ranked: a registrant whose display name
    contains the ticker (single-fund trusts like 'SPDR GOLD TRUST (GLD)')
    or the brand words (families like iShares Trust) ranks first, newest
    filing first within a registrant. EFTS itself scores bag-of-words,
    so unrelated prospectuses often outrank the fund's own 10-K — this
    re-ranking is what makes ETF verification possible."""
    def merge(f: list[dict]) -> None:
        for x in f:
            if x["accession"] not in {y["accession"] for y in out}:
                out.append(x)
    out: list[dict] = []
    words = [w for w in re.findall(r"[A-Za-z][A-Za-z\-'.]{2,}", name)
             if w.lower() not in _STOP]
    brand = _brand(name)
    if len(words) >= 4:
        merge(_efits('"' + name + '"'))
    if brand:
        merge(_efits('"' + ticker + '" AND "' + brand + '"'))
    if len(out) < limit and words:
        merge(_efits('"' + ticker + '" AND ' +
                     " AND ".join('"' + w + '"' for w in words[:3])))
    if len(out) < limit:
        merge(_efits('"' + ticker + '"'))
    name_words = {w.lower() for w in re.findall(r"[A-Za-z][A-Za-z\-'.]{3,}",
                                                 name)
                  if w.lower() not in _STOP}
    scored = []
    for f in out:
        disp = f["display"].lower()
        s = 4 * (1 if ticker.lower() in disp else 0) \
            + sum(1 for w in name_words if w in disp)
        scored.append((s, f["period_end"], f))
    scored.sort(key=lambda t: (t[0], t[1]), reverse=True)
    per_cik: dict = {}
    res = []
    for s, pe, f in scored:
        key = tuple(f["ciks"]) or (f["display"],)
        if per_cik.get(key, 0) >= 2:      # at most 2 filings per registrant
            continue
        per_cik[key] = per_cik.get(key, 0) + 1
        res.append(f)
        if len(res) >= limit:
            break
    return res


def _flex(s: str) -> str:
    """A word pattern that tolerates pipes/whitespace between words
    (fund names often sit in one cell per word)."""
    return r"[\s|]+".join(re.escape(w) for w in s.split())


def _title_runs(seg: str) -> list[str]:
    return re.findall(r"[A-Z][A-Za-z0-9'\u2019\-]*"
                      r"(?:\s+[A-Z][A-Za-z0-9'\u2019\-]*)+", seg)


def fund_region(text: str, ticker: str, name: str) -> Optional[str]:
    """The slice of the document holding this fund's highlights tables.

    The ticker appears only in the TOC/cover, not in the financial
    statements. The TOC cell "<Class> Shares - TICKER" is preceded by the
    official fund name cell; section headers in the statements drop
    prefixes like "Vanguard ", so anchor on the name or a tail of it.
    """
    occs = [m.start() for m in re.finditer(rf"(?<![A-Z0-9]){ticker}(?![A-Z0-9])",
                                           text, re.I)]
    if not occs:
        return None
    cands: list[str] = []
    for occ in occs[:3]:
        cells = [re.sub(r"&#\w+;|\s+", " ", c).strip()
                 for c in text[occ - 800:occ + 200].split("|")]
        for i, c in enumerate(cells):
            # the ticker cell: bare "PMAIX", "Shares - VTSAX", "Class A / PMAIX"...
            if not re.search(rf"(?<![A-Z0-9]){re.escape(ticker)}(?![A-Z0-9])",
                             c, re.I):
                continue
            for prev in reversed(cells[:i]):
                if prev and "shares" not in prev.lower() \
                        and "class" not in prev.lower() \
                        and len(prev.split()) >= 3:
                    cands.append(prev)
                    break
    cands.append(name)
    for cand in dict.fromkeys(cands):
        words = cand.split()
        if len(words) < 4:
            continue
        for tail_n in range(len(words), 3, -1):
            anchor = _flex(" ".join(words[-tail_n:]))
            for m in re.finditer(anchor, text, re.I):
                fhm = re.search(FH, text[m.start():m.start() + 4000])
                if not fhm:
                    continue
                fh = m.start() + fhm.start()
                reg = text[fh:fh + 150000]
                if parse_highlights(reg) or parse_per_share_blocks(reg):
                    return reg
    return None


def _name_pats(name: str) -> list[str]:
    """Pipe-tolerant name patterns. Yahoo longNames and filing names differ
    ("JPMorgan US Large Cap Core Plus I" vs "JPMorgan U.S. Large Cap Core
    Plus Fund"), so match on runs of 3-4 distinctive words, dots optional.
    Trailing single-letter / RC# class designators are dropped."""
    if not name:
        return []
    words = [re.sub(r"\.", "", w) for w in name.split()]
    while words and (len(words[-1]) <= 2 and words[-1].isalpha()):
        words.pop()
    pats = []
    def mk(ws: list[str]) -> str:
        # optional dots between letters: "US" matches "U.S." and vice versa
        return r"[\s|]+".join(r"(?:" + r"\.?".join(re.escape(ch) for ch in w)
                               + r"\.?)" for w in ws)
    # tail runs (the distinctive part) and the head (brand + next words)
    for n in (4, 3):
        if len(words) >= n:
            pats.append(mk(words[-n:]))
        if len(words) > n:
            pats.append(mk(words[:n]))
    pats.append(mk(words))
    return pats


def region_per_share(text: str, ticker: str, name: str,
                     loc: Optional[dict] = None) -> Optional[str]:
    """Region finder for the 'Per share operating performance' layout
    (JPMorgan-style shareholder reports). Anchors on the fund name or
    ticker; when `loc` (local series) is given, a candidate region is only
    accepted if one of its class blocks actually matches the local
    distributions — name mentions in notes/TOC of a 58 MB combined report
    would otherwise attribute another fund's tables."""
    pats = _name_pats(name)
    pats.append(rf"(?<![A-Z0-9]){re.escape(ticker)}(?![A-Z0-9])")
    for p in pats:
        for m in re.finditer(p, text, re.I):
            reg = text[m.start():m.start() + 400000]
            blocks = parse_per_share_blocks(reg)
            if not blocks:
                continue
            if loc is None:
                return reg
            for b in blocks:
                errs = []
                for i in range(len(b["periods"]) - 1):
                    off = abs(b["total"][i])
                    lc = _window_sum(loc["dist"], b["periods"][i + 1]["end"],
                                     b["periods"][i]["end"])
                    if off < 0.01 and lc < 0.01:
                        continue
                    errs.append(abs(off - lc) / max(off, lc, 1e-9))
                if errs and sum(errs) / len(errs) < 0.15:
                    return reg
    return None


def _wordset(s: str) -> set:
    return set(re.findall(r"[a-z0-9]{3,}", s.lower())) - _STOP


def region_by_name(text: str, name: str, min_shared: int = 3):
    """Fallback when the ticker is absent from the document (families that
    don't publish distribution tickers): match the fund by name — the cell
    immediately before each Financial Highlights heading — and keep the
    best word-overlap candidate with a parseable region."""
    target = _wordset(name)
    best = None
    for m in re.finditer(FH, text):
        pre = re.sub(r"&#\w+;", " ", text[max(0, m.start() - 400):m.start()])
        pre = re.sub(r"[\s|]+", " ", pre).strip()
        runs = re.findall(r"[A-Z][A-Za-z0-9'\u2019\-]*"
                          r"(?: [A-Z][A-Za-z0-9'\u2019\-]*){2,11}", pre)
        if not runs:
            continue
        for cand in dict.fromkeys(runs):        # any run, best overlap wins
            shared = len(_wordset(cand) & target)
            if shared < min_shared:
                continue
            reg = text[m.start():m.start() + 150000]
            if not (parse_highlights(reg) or parse_per_share_blocks(reg)):
                continue
            if best is None or shared > best[0]:
                best = (shared, reg, cand)
    return best


# ---------------------------------------------------------------- compare
def local_series(sym: str, root: Path, frozen: Path) -> dict:
    """Local (date, amount) distributions and close series; frozen copies
    shadow the data root, and the overrides/corrections overlay (confirmed
    official fixes, see overrides/corrections/) is applied on top so a
    corrected fund verifies against its filing rather than the raw dump."""
    import pandas as pd
    def read(sfx: str):
        p = frozen / f"{sym}-{sfx}.csv"
        if not p.exists():
            p = root / f"{sym}-{sfx}.csv"
        if not p.exists():
            return []
        out = []
        for line in p.read_text(errors="replace").splitlines()[1:]:
            parts = line.split(",")
            if len(parts) >= 2:
                out.append((parts[0][:10], float(parts[1])))
        ser = pd.Series({pd.Timestamp(d): v for d, v in out}, dtype=float)
        try:
            ser = D._apply_corrections(sym, sfx, ser)
        except Exception:
            pass
        return [(ts.strftime("%Y-%m-%d"), float(v)) for ts, v in ser.items()]
    div, capg = read("dividend"), read("capitalGain")
    try:
        div, capg = D.dedupe_event_rows(D._load_corrections(sym), div, capg)
    except Exception:
        pass
    dist = sorted(div + capg)
    p = frozen / f"{sym}-history.csv"
    if not p.exists():
        p = root / f"{sym}-history.csv"
    closes = []
    if p.exists():
        for line in p.read_text(errors="replace").splitlines()[1:]:
            parts = line.split(",")
            if len(parts) >= 5:
                closes.append((parts[0][:10], float(parts[4])))
    return {"dist": dist, "close": closes}


def _window_sum(dist, lo, hi):
    """Sum of distributions with lo < date <= hi (ISO date strings)."""
    return round(sum(v for d, v in dist if (lo is None or d > lo) and d <= hi),
                 4)


def compare(sym: str, blocks: list[dict], loc: dict) -> dict:
    """Pick the best-matching class and compare period by period.

    Each official column covers (previous period end, this period end];
    local distributions are summed over the same window, so non-calendar
    fiscal years compare correctly.
    """
    def period_errs(b):
        errs = []
        for i, p in enumerate(b["periods"]):
            if p["kind"] != "fy":
                continue
            if i + 1 >= len(b["periods"]):
                continue          # oldest column: unbounded window, unverifiable
            off = abs(b["total"][i])
            lo = b["periods"][i + 1]["end"]
            lc = _window_sum(loc["dist"], lo, p["end"])
            errs.append(abs(off - lc) / max(off, lc, 1e-9))
        return errs

    best = None
    for b in blocks:
        errs = period_errs(b)
        if not errs:
            continue
        me = sum(errs) / len(errs)
        if best is None or me < best["err"]:
            best = {"err": me, "block": b}
    if best is None:
        return {"verdict": "not-found", "classes": len(blocks), "detail":
                "no class block with overlapping periods"}
    b = best["block"]
    rows = []
    mismatch = False
    for i, p in enumerate(b["periods"]):
        off = abs(b["total"][i])
        # columns are newest-first: the window lower bound is the NEXT
        # (older) column's end; the oldest column is unbounded
        lo = b["periods"][i + 1]["end"] if i + 1 < len(b["periods"]) else None
        lc = _window_sum(loc["dist"], lo, p["end"])
        d = off - lc
        ok = abs(d) <= TOL_ABS or abs(d) / max(off, 1e-9) <= TOL_REL
        mismatch |= p["kind"] == "fy" and lo is not None and not ok
        rows.append({"period": p["kind"], "end": p["end"],
                     "official": round(off, 4), "local": lc,
                     "diff": round(d, 4), "ok": ok,
                     "unbounded": lo is None})
    # NAV check: most recent period with an official ending NAV
    nav = None
    if b.get("nav_end") and loc["close"]:
        closes = dict(loc["close"])
        for i in range(len(b["periods"]) - 1, -1, -1):
            p = b["periods"][i]
            prior = [d for d in closes if d <= p["end"]]
            if not prior:
                continue
            o, l = abs(b["nav_end"][i]), closes[prior[-1]]
            nav = {"date": prior[-1], "official": o, "local": l,
                   "rel": round(abs(o - l) / o, 4),
                   "ok": abs(o - l) / o <= NAV_TOL}
            break
    # verdict: a good class fit with a bad year is a real mismatch; a bad
    # class fit overall means the document format (or fund) isn't matched
    if best["err"] >= 0.30:
        verdict = "weak-match"
    else:
        verdict = "mismatch" if mismatch else "ok"
    return {"verdict": verdict,
            "class": b["class"], "mean_rel_err": round(best["err"], 4),
            "nav_check": nav, "periods": rows,
            "other_classes": [x["class"] for x in blocks if x is not b]}


# ---------------------------------------------------------------- driver
def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("syms", nargs="*")
    ap.add_argument("--stale", action="store_true",
                    help="also verify all frozen (stale) tickers")
    ap.add_argument("--json-only", action="store_true")
    ap.add_argument("--limit", type=int, default=0)
    args = ap.parse_args()

    syms = [s.lower() for s in args.syms]
    if not syms and not args.stale:
        syms = list(json.loads(FUNDS.read_text()).keys())
    if args.stale:
        syms += sorted(f.stem for f in (D.FROZEN_DIR.glob("*.json")))
    syms = list(dict.fromkeys(syms))
    if args.limit:
        syms = syms[:args.limit]

    REPORT_DIR.mkdir(parents=True, exist_ok=True)
    summary = []
    t0 = time.monotonic()
    for n, sym in enumerate(syms, 1):
        out = REPORT_DIR / f"{sym}.json"
        cached = None
        if out.exists() and not args.json_only:
            cached = json.loads(out.read_text())
            if not (cached.get("filing") or cached.get("period_end")):
                cached = None          # not-found from a prior run: retry
        if cached:
            summary.append(cached)
            print(f"[{n}/{len(syms)}] {sym} cached {cached['verdict']}", flush=True)
            continue
        name = ""
        meta_f = ROOT / f"{sym}.json"
        if meta_f.exists():
            try:
                name = json.loads(meta_f.read_text())["chart"]["result"][0] \
                                             .get("meta", {}).get("longName", "")
            except Exception:
                pass
        if not name:
            mf = D.FROZEN_DIR / f"{sym}.json"
            if mf.exists():
                name = json.loads(mf.read_text()).get("longName", "")
        try:
            if args.json_only:
                raise RuntimeError("json-only")
            cands = find_filings(sym.upper(), name)
            rec = {"symbol": sym, "name": name}
            loc = local_series(sym, ROOT, D.FROZEN_DIR)
            best = None
            for filing in cands:
                try:
                    doc = edgar.sec_get(
                        f"https://www.sec.gov/Archives/edgar/data/"
                        f"{int(filing['ciks'][0]):010d}/{filing['accession'].replace('-', '')}/"
                        f"{filing['doc']}")
                except Exception:
                    continue
                text = html_to_cells_text(doc)
                region, how = None, None
                if re.search(rf"(?<![A-Z0-9]){re.escape(sym.upper())}(?![A-Z0-9])",
                             text):
                    region = fund_region(text, sym.upper(), name)
                    how = "ticker"
                if region is None:
                    rb = region_by_name(text, name)
                    if rb:
                        region, how = rb[1], f"name:{rb[2]} (shared={rb[0]})"
                if region is None:
                    rp = region_per_share(text, sym.upper(), name, loc)
                    if rp:
                        region, how = rp, "per-share-data"
                if region is None:
                    continue          # wrong document; try next candidate
                blocks = (parse_highlights(region)
                          or parse_per_share_blocks(region))
                res = compare(sym, blocks, loc)
                res.update({"symbol": sym, "name": name, "filing": filing,
                            "how": how, "n_bytes": len(doc)})
                # different official docs can disagree by a cent or two
                # (N-CSRS vs 497 rounding); keep the best match but stop
                # early on a clean "ok"
                key = (res["verdict"] != "ok", res.get("mean_rel_err", 1.0))
                if best is None or key < (best["verdict"] != "ok",
                                          best.get("mean_rel_err", 1.0)):
                    best = res
                if res["verdict"] == "ok":
                    break
            if best is not None:
                rec.update(best)
            else:
                rec["verdict"] = "not-found"
                rec["detail"] = ("no candidate filing contained this fund "
                                 f"({len(cands)} tried)")
        except Exception as e:
            rec = {"symbol": sym, "name": name, "verdict": "error",
                   "detail": str(e)[:300]}
        out.write_text(json.dumps(rec, indent=1))
        summary.append(rec)
        print(f"[{n}/{len(syms)}] {sym} {rec.get('verdict')} "
              f"({time.monotonic() - t0:.0f}s)", flush=True)

    # summary covers EVERY record in the report dir, not just this batch
    recs = []
    for f in sorted(REPORT_DIR.glob("*.json")):
        try:
            recs.append(json.loads(f.read_text()))
        except Exception:
            pass
    lines = ["# Official-filing cross-check (SEC N-CSR/N-CSRS/10-K/10-Q)\n",
             "Verdicts: ok = all bounded fiscal periods agree; mismatch = class matched "
             "but some period differs (review the year, then correct via overrides/); "
             "weak-match = best class fit too poor to trust (format/fund mismatch); "
             "not-found = fund not located in any candidate filing.\n",
             "| Symbol | Verdict | Class | Periods (fy end, ~ = unbounded) | Detail |",
             "|---|---|---|---|---|"]
    for rec in recs:
        yrs = ""
        if rec.get("periods"):
            yrs = " ".join(
                ((f"{y['end'][:7]}~") if y.get("unbounded")
                 else f"{y['end'][:7]}{'✓' if y['ok'] else '✗'}"
                 for y in rec["periods"] if y["period"] == "fy"))
        lines.append(f"| {rec['symbol'].upper()} | {rec.get('verdict')} | "
                     f"{rec.get('class', '')} | {yrs} | "
                     f"{rec.get('detail', '')[:80]} |")
    (REPORT_DIR / "SUMMARY.md").write_text("\n".join(lines) + "\n")
    from collections import Counter
    c = Counter(r.get("verdict", "?") for r in summary)
    print(f"\n{dict(c)}  ({time.monotonic() - t0:.0f}s) -> {REPORT_DIR / 'SUMMARY.md'}")
    return 0


if __name__ == "__main__":
    sys.exit(main())