Greg Pomerantz
43ddd3ec7f
JLPSX showed a bogus +29.3%/-22.9% 3-day wiggle in Dec 2020: Yahoo dated
the 6.824 year-end cap-gain distribution on the 12-11 record date but the
market went ex-div on 12-14 (close 30.10 -> 23.22), so the raw Adj Close
column pre-applied the adjustment 3 days before the price actually fell.
New 'history' correction op ({date: {col: value}}) patches individual OHLC
cells at bundle assembly (full build, incremental, and correction-changed
recompute paths); check_corrections validates dates/columns; 4 new tests.
scripts/scan_adj_misalign.py finds the artifact set-wide: 481 hits on 278
symbols, overwhelmingly December year-end distributions of value funds
(JLPSX's class of fund). scripts/fix_adj_misalign.py repairs it
arithmetic-only (rescale adj in [event, ex-div) by (1-f); cumulative
returns unchanged, cross-checked implied dist vs the price drop). Applied
to the only curated-fund hit (JLPSX) and its sister class JLPYX (implied
dist 6.824 both = official amount; ex-div 2020-12-14). The remaining ~276
symbols are reported in reports/adj_misalign/scan.md for a bulk run.
94 lines
3.7 KiB
Python
94 lines
3.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Integrity check for overrides/corrections/*.json.
|
|
|
|
Every remove/replace op must reference a (date, amount) row that actually
|
|
exists in the file it targets (frozen copy if present, else the data root) —
|
|
_apply_corrections silently no-ops a remove that matches nothing, so a row
|
|
filed under the wrong section (dividends vs capitalGains) or with a stale
|
|
amount would be applied as a do-nothing correction. add ops must not
|
|
reference an existing row (double-add risk).
|
|
|
|
Usage: python3 scripts/check_corrections.py
|
|
"""
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
DATA = Path.home() / "prog/fin/stocks"
|
|
FROZEN = ROOT / "overrides" / "frozen"
|
|
CORR = ROOT / "overrides" / "corrections"
|
|
|
|
|
|
def load(sfx: str, sym: str) -> dict:
|
|
p = FROZEN / f"{sym}-{sfx}.csv"
|
|
if not p.exists():
|
|
p = DATA / f"{sym}-{sfx}.csv"
|
|
if not p.exists():
|
|
return None
|
|
out = {}
|
|
for line in p.read_text(errors="replace").splitlines()[1:]:
|
|
a = line.split(",")
|
|
if len(a) >= 2:
|
|
out.setdefault(a[0][:10], []).append(float(a[1]))
|
|
return out
|
|
|
|
|
|
def main() -> int:
|
|
bad = 0
|
|
for f in sorted(CORR.glob("*.json")):
|
|
sym = f.stem.lower()
|
|
d = json.loads(f.read_text())
|
|
files = {"dividends": load("dividend", sym),
|
|
"capitalGains": load("capitalGain", sym)}
|
|
divf = files["dividends"] or {}
|
|
capgf = files["capitalGains"] or {}
|
|
for date, amt in d.get("dedup", []):
|
|
if amt not in divf.get(date, []) and amt not in capgf.get(date, []):
|
|
print(f"BAD {sym} dedup {date} {amt}: row in neither file")
|
|
bad += 1
|
|
for date in d.get("drop_capg_copy", []):
|
|
if not capgf.get(date):
|
|
print(f"BAD {sym} drop_capg_copy {date}: no capitalGain row")
|
|
bad += 1
|
|
for date, cells in (d.get("history") or {}).items():
|
|
p = FROZEN / f"{sym}-history.csv"
|
|
if not p.exists():
|
|
p = DATA / f"{sym}-history.csv"
|
|
dates = {l.split(",")[0][:10] for l in p.read_text(errors="replace").splitlines()[1:]} if p.exists() else set()
|
|
if date not in dates:
|
|
print(f"BAD {sym} history {date}: date not in history file")
|
|
bad += 1
|
|
for col in cells:
|
|
if col not in ("Open", "High", "Low", "Close", "Adj Close", "Volume"):
|
|
print(f"BAD {sym} history {date}: unknown column {col!r}")
|
|
bad += 1
|
|
for key, rows_f in files.items():
|
|
ops = d.get(key)
|
|
if not ops or rows_f is None:
|
|
if rows_f is None:
|
|
print(f"?? {sym}: no data file for {key} correction")
|
|
continue
|
|
for date, amt in ops.get("remove", []):
|
|
if amt not in rows_f.get(date, []):
|
|
print(f"BAD {sym} {key}.remove {date} {amt}: "
|
|
f"row not present (values: {rows_f.get(date, 'missing')})")
|
|
bad += 1
|
|
for date, amt in ops.get("replace", {}).items():
|
|
if date not in rows_f:
|
|
print(f"BAD {sym} {key}.replace {date}: date not present")
|
|
bad += 1
|
|
for date, amt in ops.get("add", []):
|
|
if amt in rows_f.get(date, []):
|
|
print(f"BAD {sym} {key}.add {date} {amt}: row already present")
|
|
bad += 1
|
|
if bad:
|
|
print(f"\n{bad} problem(s) — fix the corrections above")
|
|
return 1
|
|
print(f"all {len(list(CORR.glob('*.json')))} correction files consistent")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|