Greg Pomerantz
e20b2e30cb
Yahoo changed the shape of its event feed between downloads: a 2026-08
re-download of CVIX/JLPSX shows it no longer returns capitalGain events at
all (the dividend stream still carries the year-end rows), while stale
tickers now return no events at all. File-specific remove ops therefore
break silently on the next re-download, so the double-listing fix is now
expressed as layout-agnostic invariants applied at bundle assembly
(idempotent, hold for full and incremental builds):
dedup: [[date, amount]] keep at most one copy of a same-date/
same-amount cross-file pair (the capitalGain copy when both present)
drop_capg_copy: [date] the capitalGain row on that date is the
spurious copy of the dividend row
- data.py: apply_invariants() at bundle assembly + pure dedupe_event_rows()
shared with verify_official and tests (8 new test cases, 22 passing)
- JLPSX/CVSIX corrections rewritten with the invariants (2019-08-08 now
keeps the dividend amount per the verified same-date pattern)
- scripts/scan_double_listing.py: whole-dump sweep -> reports/double_listing/
6,421 symbols scanned: 1,218 with same-amount pairs (6,589), 1,631 with
differing-amount pairs (12,366, reported only - not distinguishable from
legitimate same-day div+capg without per-fund official data), 83 with
repeated within-file rows (ingest keep-last already collapses them)
- scripts/apply_dedup_corrections.py: bulk 'dedup' corrections for the
1,218 same-amount symbols (1,212 new files; the 6 verified funds keep
their explicit, official-verified corrections)
- scripts/check_corrections.py: integrity check for every correction op
against the actual (frozen) files - caught a mis-filed CVSIX entry
82 lines
3.0 KiB
Python
82 lines
3.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Integrity check for overrides/corrections/*.json.
|
|
|
|
Every remove/replace op must reference a (date, amount) row that actually
|
|
exists in the file it targets (frozen copy if present, else the data root) —
|
|
_apply_corrections silently no-ops a remove that matches nothing, so a row
|
|
filed under the wrong section (dividends vs capitalGains) or with a stale
|
|
amount would be applied as a do-nothing correction. add ops must not
|
|
reference an existing row (double-add risk).
|
|
|
|
Usage: python3 scripts/check_corrections.py
|
|
"""
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
DATA = Path.home() / "prog/fin/stocks"
|
|
FROZEN = ROOT / "overrides" / "frozen"
|
|
CORR = ROOT / "overrides" / "corrections"
|
|
|
|
|
|
def load(sfx: str, sym: str) -> dict:
|
|
p = FROZEN / f"{sym}-{sfx}.csv"
|
|
if not p.exists():
|
|
p = DATA / f"{sym}-{sfx}.csv"
|
|
if not p.exists():
|
|
return None
|
|
out = {}
|
|
for line in p.read_text(errors="replace").splitlines()[1:]:
|
|
a = line.split(",")
|
|
if len(a) >= 2:
|
|
out.setdefault(a[0][:10], []).append(float(a[1]))
|
|
return out
|
|
|
|
|
|
def main() -> int:
|
|
bad = 0
|
|
for f in sorted(CORR.glob("*.json")):
|
|
sym = f.stem.lower()
|
|
d = json.loads(f.read_text())
|
|
files = {"dividends": load("dividend", sym),
|
|
"capitalGains": load("capitalGain", sym)}
|
|
divf = files["dividends"] or {}
|
|
capgf = files["capitalGains"] or {}
|
|
for date, amt in d.get("dedup", []):
|
|
if amt not in divf.get(date, []) and amt not in capgf.get(date, []):
|
|
print(f"BAD {sym} dedup {date} {amt}: row in neither file")
|
|
bad += 1
|
|
for date in d.get("drop_capg_copy", []):
|
|
if not capgf.get(date):
|
|
print(f"BAD {sym} drop_capg_copy {date}: no capitalGain row")
|
|
bad += 1
|
|
for key, rows_f in files.items():
|
|
ops = d.get(key)
|
|
if not ops or rows_f is None:
|
|
if rows_f is None:
|
|
print(f"?? {sym}: no data file for {key} correction")
|
|
continue
|
|
for date, amt in ops.get("remove", []):
|
|
if amt not in rows_f.get(date, []):
|
|
print(f"BAD {sym} {key}.remove {date} {amt}: "
|
|
f"row not present (values: {rows_f.get(date, 'missing')})")
|
|
bad += 1
|
|
for date, amt in ops.get("replace", {}).items():
|
|
if date not in rows_f:
|
|
print(f"BAD {sym} {key}.replace {date}: date not present")
|
|
bad += 1
|
|
for date, amt in ops.get("add", []):
|
|
if amt in rows_f.get(date, []):
|
|
print(f"BAD {sym} {key}.add {date} {amt}: row already present")
|
|
bad += 1
|
|
if bad:
|
|
print(f"\n{bad} problem(s) — fix the corrections above")
|
|
return 1
|
|
print(f"all {len(list(CORR.glob('*.json')))} correction files consistent")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|