Repair Yahoo adj-close/event-date misalignment (JLPSX, JLPYX)

JLPSX showed a bogus +29.3%/-22.9% 3-day wiggle in Dec 2020: Yahoo dated
the 6.824 year-end cap-gain distribution on the 12-11 record date but the
market went ex-div on 12-14 (close 30.10 -> 23.22), so the raw Adj Close
column pre-applied the adjustment 3 days before the price actually fell.

New 'history' correction op ({date: {col: value}}) patches individual OHLC
cells at bundle assembly (full build, incremental, and correction-changed
recompute paths); check_corrections validates dates/columns; 4 new tests.

scripts/scan_adj_misalign.py finds the artifact set-wide: 481 hits on 278
symbols, overwhelmingly December year-end distributions of value funds
(JLPSX's class of fund). scripts/fix_adj_misalign.py repairs it
arithmetic-only (rescale adj in [event, ex-div) by (1-f); cumulative
returns unchanged, cross-checked implied dist vs the price drop). Applied
to the only curated-fund hit (JLPSX) and its sister class JLPYX (implied
dist 6.824 both = official amount; ex-div 2020-12-14). The remaining ~276
symbols are reported in reports/adj_misalign/scan.md for a bulk run.
This commit is contained in:
Greg Pomerantz 2026-08-31 23:08:42 -04:00
parent 3c5bee96ce
commit 43ddd3ec7f
8 changed files with 826 additions and 10 deletions

31
data.py
View File

@ -128,7 +128,20 @@ def apply_invariants(div: pd.DataFrame, capg: pd.DataFrame) -> None:
div.at[ts, sym] = 0.0
def _apply_corrections(sym: str, suffix: str, ser: pd.Series) -> pd.Series:
def _apply_corrections(sym: str, suffix: str, ser: pd.Series,
col: str | None = None) -> pd.Series:
if suffix == "history":
# {date: {col: value}} — patch individual OHLC cells (e.g. a
# misaligned Yahoo adj-close spike). Only touches the requested col.
ops = _load_corrections(sym).get("history")
if not ops or col is None:
return ser
d = {ts: float(v) for ts, v in ser.items()}
for dstr, cells in ops.items():
ts = pd.Timestamp(dstr)
if ts in d and col in cells:
d[ts] = float(cells[col])
return pd.Series(d).sort_index()
key = {"dividend": "dividends", "capitalGain": "capitalGains"}.get(suffix)
ops = _load_corrections(sym).get(key) if key else None
if not ops:
@ -238,7 +251,7 @@ def _read_panel(root: Path, suffix: str, col: str) -> pd.DataFrame:
except Exception:
continue
if col in df.columns:
cols.append(_apply_corrections(sym, suffix, df[col]))
cols.append(_apply_corrections(sym, suffix, df[col], col))
syms.append(sym)
if not cols:
return pd.DataFrame()
@ -362,7 +375,9 @@ def _apply_changes(root: Path, cache: Path, old: dict, new: dict) -> None:
def _recompute_corrected(symfile: str) -> None:
"""corrections/{SYM}.json changed: re-derive the symbol's
dividend/capital-gain series from the base file + corrections."""
for sfx, colname, key in (("dividend", "Dividends", "div"),
for sfx, colname, key in (("history", "Adj Close", "adj"),
("history", "Close", "close"),
("dividend", "Dividends", "div"),
("capitalGain", "Capital Gains", "capg")):
sym = None
p = None
@ -382,8 +397,8 @@ def _apply_changes(root: Path, cache: Path, old: dict, new: dict) -> None:
continue
if colname not in df.columns:
continue
updates[key][sym] = _apply_corrections(sym, sfx, df[colname]) \
.fillna(0.0)
ser = _apply_corrections(sym, sfx, df[colname], colname)
updates[key][sym] = ser if sfx == "history" else ser.fillna(0.0)
authoritative.add(sym)
for fname in changed:
@ -401,9 +416,11 @@ def _apply_changes(root: Path, cache: Path, old: dict, new: dict) -> None:
continue # e.g. a file mid-download
if sfx == "history":
if "Adj Close" in df.columns:
updates["adj"][sym] = df["Adj Close"]
updates["adj"][sym] = _apply_corrections(
sym, "history", df["Adj Close"], "Adj Close")
if "Close" in df.columns:
updates["close"][sym] = df["Close"]
updates["close"][sym] = _apply_corrections(
sym, "history", df["Close"], "Close")
elif sfx == "dividend" and "Dividends" in df.columns:
updates["div"][sym] = _apply_corrections(sym, sfx, df["Dividends"]) \
.fillna(0.0)

View File

@ -1,7 +1,7 @@
{
"symbol": "jlpsx",
"name": "JPMorgan U.S. Large Cap Core Plus Fund (I shares)",
"note": "Yahoo listed the year-end capital-gain distribution in BOTH the dividend and capitalGains event files. Expressed as layout-agnostic invariants because a 2026-08 re-download shows Yahoo no longer returns capitalGain events for this fund at all (the dividend stream still carries the year-end rows): 'dedup' keeps at most one copy of each (date, amount) (the capitalGain copy when both files carry it; the lone dividend copy otherwise); 'drop_capg_copy' drops the spurious capitalGain row on a date where the dividend row is the true distribution (any amount). Official per-share data (JPMorgan 497, filed 2025-11-03, Class I, FYE June 30): FY2025 1.37, FY2024 2.03, FY2023 4.86, FY2022 6.78, FY2021 6.89 - local sums after dedup match every one (1.368, 2.029, 4.859, 6.785, 6.889). 2006-12-15 (div 0.207 vs capg 0.141) and 2019-08-08 (div 4.138 vs capg 4.077) keep the dividend amount per the pattern confirmed on all other verified same-date pairs; pre-2012 years have no official totals (the 2021 497 is delisted from SEC Archives) - mechanism-inferred.",
"note": "Yahoo listed the year-end capital-gain distribution in BOTH the dividend and capitalGains event files. Expressed as layout-agnostic invariants because a 2026-08 re-download shows Yahoo no longer returns capitalGain events for this fund at all (the dividend stream still carries the year-end rows): 'dedup' keeps at most one copy of each (date, amount) (the capitalGain copy when both files carry it; the lone dividend copy otherwise); 'drop_capg_copy' drops the spurious capitalGain row on a date where the dividend row is the true distribution (any amount). Official per-share data (JPMorgan 497, filed 2025-11-03, Class I, FYE June 30): FY2025 1.37, FY2024 2.03, FY2023 4.86, FY2022 6.78, FY2021 6.89 - local sums after dedup match every one (1.368, 2.029, 4.859, 6.785, 6.889). 2006-12-15 (div 0.207 vs capg 0.141) and 2019-08-08 (div 4.138 vs capg 4.077) keep the dividend amount per the pattern confirmed on all other verified same-date pairs; pre-2012 years have no official totals (the 2021 497 is delisted from SEC Archives) - mechanism-inferred. 'history' patches the 2020-12-11 Adj Close cell: Yahoo dated the 6.824 distribution on the 12-11 record date but the market went ex-div on 2020-12-14 (close 30.10 -> 23.22, i.e. -6.88), so the raw Adj Close column spiked +29.3% on 12-11 and snapped back -22.9% on 12-14; setting 12-11 to the 12-10 value (11.247889) removes the bogus 3-day wiggle (cumulative return unchanged). adj-close repair: Yahoo dated a distribution on 2020-12-11 (record date) but the price went ex-div on 2020-12-14 (close 30.10 -> 23.22, drop ~6.824); the raw Adj Close column spiked +29.3% on 2020-12-11 and snapped back later. Cells in [2020-12-11, 2020-12-14) rescaled by (1-f) so the adjusted path is smooth; cumulative returns are unchanged. Mechanism: scripts/fix_adj_misalign.py, scan: reports/adj_misalign/scan.md.",
"verified": [
"FY2021-FY2025 exact match against 497 filed 2025-11-03 (accession 0001193125-25-261054, CIK 0000763852)"
],
@ -66,5 +66,10 @@
"drop_capg_copy": [
"2006-12-15",
"2019-08-08"
]
],
"history": {
"2020-12-11": {
"Adj Close": 11.247889
}
}
}

View File

@ -38,5 +38,10 @@
0.656
]
],
"note": "Bulk correction from the double-listing scan (scripts/scan_double_listing.py): Yahoo listed this distribution in both the dividend and capitalGains event files (same date, same amount); one copy is spurious. The 'dedup' invariant keeps at most one copy, layout-agnostically. Mechanism verified against official filings for JLPSX/GDEUX/GSOUX/FAEVX/CVSIX/FZAGX; mechanism-inferred for this symbol (no per-fund official check). Revert the entry if a future official cross-check shows both rows were real."
"note": "Bulk correction from the double-listing scan (scripts/scan_double_listing.py): Yahoo listed this distribution in both the dividend and capitalGains event files (same date, same amount); one copy is spurious. The 'dedup' invariant keeps at most one copy, layout-agnostically. Mechanism verified against official filings for JLPSX/GDEUX/GSOUX/FAEVX/CVSIX/FZAGX; mechanism-inferred for this symbol (no per-fund official check). Revert the entry if a future official cross-check shows both rows were real. adj-close repair: Yahoo dated a distribution on 2020-12-11 (record date) but the price went ex-div on 2020-12-14 (close 30.29 -> 23.40, drop ~6.824); the raw Adj Close column spiked +29.1% on 2020-12-11 and snapped back later. Cells in [2020-12-11, 2020-12-14) rescaled by (1-f) so the adjusted path is smooth; cumulative returns are unchanged. Mechanism: scripts/fix_adj_misalign.py, scan: reports/adj_misalign/scan.md.",
"history": {
"2020-12-11": {
"Adj Close": 11.400895
}
}
}

View File

@ -0,0 +1,487 @@
# Adj-close / event-date misalignment scan
params: min_adj=0.05 max_close=0.02 horizon=3; 481 hits on 278 symbols
| symbol | date | adj ret | close ret | later drop |
|---|---|---|---|---|
| wbqnl | 2023-06-13 | +436.00% | +0.00% | +83.21% |
| frccx | 2024-04-17 | +308.92% | +0.00% | +75.53% |
| almrx | 2021-12-15 | +82.25% | +0.00% | +44.76% |
| ivinx | 2022-12-15 | +73.70% | +0.00% | +45.47% |
| itgrx | 2022-12-15 | +70.65% | +0.00% | +44.46% |
| amgox | 2021-12-15 | +58.02% | +0.00% | +36.49% |
| aasox | 2021-12-15 | +52.48% | +0.00% | +33.80% |
| krft | 27-Mar-15 | +44.00% | -1.67% | +32.33% |
| amczx | 2021-12-15 | +42.22% | +0.00% | +29.75% |
| aasox | 2015-12-16 | +41.49% | +0.00% | +29.07% |
| mxjvx | 2018-12-27 | +37.16% | +0.00% | +26.59% |
| rmbhx | 2016-12-15 | +36.53% | +0.00% | +26.43% |
| specx | 2021-12-15 | +35.44% | +0.00% | +27.52% |
| mveix | 2020-12-29 | +34.33% | -1.26% | +26.12% |
| seekx | 2021-12-17 | +32.69% | +0.00% | +24.43% |
| irsex | 2022-12-15 | +31.68% | +0.00% | +28.42% |
| mvtrx | 2021-12-21 | +31.40% | +1.62% | +19.23% |
| irsyx | 2022-12-15 | +31.04% | +0.00% | +28.03% |
| mgqsx | 2024-12-16 | +30.30% | +0.14% | +25.18% |
| bgatx | 2018-12-27 | +30.07% | +0.00% | +22.33% |
| jlpsx | 2020-12-11 | +29.32% | +0.00% | +21.43% |
| jlpyx | 2020-12-11 | +29.08% | +0.00% | +21.29% |
| mgfzx | 2022-12-13 | +28.38% | +1.79% | +22.58% |
| ahszx | 2021-12-15 | +28.30% | +0.00% | +20.38% |
| vscax | 2018-12-14 | +26.66% | +0.00% | +25.20% |
| line | 2016-04-07 | -26.30% | +0.00% | +15.62% |
| jevrx | 2024-12-20 | +26.19% | +0.00% | +20.40% |
| iysyx | 2022-12-15 | +26.05% | +0.00% | +24.14% |
| mxmsx | 2018-12-27 | +25.90% | +0.00% | +18.42% |
| scjax | 2021-12-17 | +25.48% | +0.00% | +19.81% |
| ibarx | 2022-12-15 | +25.08% | +0.00% | +22.77% |
| ibnax | 2022-12-15 | +25.03% | +0.00% | +22.73% |
| mgrex | 2022-12-16 | +25.00% | -1.83% | +25.46% |
| alsrx | 2021-12-15 | +24.78% | +0.00% | +19.44% |
| ispvx | 2022-12-15 | +24.31% | +0.00% | +23.10% |
| spgsx | 2024-12-27 | +24.02% | +0.00% | +22.07% |
| ienax | 2007-12-14 | +23.72% | +0.00% | +20.90% |
| mxykx | 2024-12-27 | +23.34% | +0.00% | +20.66% |
| mxgsx | 2018-12-27 | +23.26% | +0.00% | +17.13% |
| timvx | 2019-12-06 | +23.16% | +1.00% | +17.07% |
| line | 2011-08-11 | +23.11% | -0.40% | +15.72% |
| mxkwx | 2018-12-27 | +23.06% | +0.00% | +17.19% |
| acazx | 2021-12-15 | +22.93% | +0.00% | +20.14% |
| mxykx | 2018-12-27 | +22.01% | +0.00% | +16.05% |
| irsyx | 2025-12-12 | +21.60% | +0.00% | +17.84% |
| line | 2015-09-01 | +21.45% | -1.72% | +16.91% |
| wsgrx | 2025-12-12 | +21.39% | +0.00% | +22.21% |
| wmicx | 2005-12-28 | +21.16% | +0.00% | +15.69% |
| mxvhx | 2024-12-27 | +21.01% | +0.13% | +18.47% |
| mpivx | 2021-12-15 | +20.69% | +1.11% | +16.64% |
| cacsx | 2018-12-14 | +20.33% | +0.00% | +20.74% |
| alarx | 2021-12-15 | +19.99% | +0.00% | +18.21% |
| line | 2015-10-08 | +19.98% | +1.49% | +10.71% |
| te | 2002-10-21 | +19.83% | +0.46% | +14.20% |
| vallx | 1991-12-17 | +19.70% | -0.70% | +18.77% |
| mxkwx | 2023-12-27 | +19.47% | +0.00% | +16.33% |
| line | 2013-07-18 | +19.14% | +0.57% | +9.97% |
| chtrx | 2019-12-13 | +18.74% | +0.00% | +15.27% |
| mxerx | 2018-12-27 | +18.59% | +0.00% | +14.25% |
| greux | 2017-12-11 | +18.51% | +0.00% | +15.14% |
| chftx | 2019-12-13 | +18.13% | +0.00% | +14.82% |
| mxhtx | 2018-12-27 | +18.09% | +0.00% | +14.79% |
| prnhx | 1986-12-15 | +17.68% | +0.00% | +16.04% |
| ace | 1998-12-24 | +17.62% | +1.74% | +14.70% |
| smeax | 2018-12-14 | +17.59% | +0.00% | +19.60% |
| ace | 2000-07-28 | +17.27% | -1.22% | +13.22% |
| oalgx | 2025-12-10 | +16.73% | +0.00% | +16.03% |
| nwcfx | 2016-12-15 | +16.70% | +0.00% | +13.89% |
| mxzhx | 2019-09-10 | +16.69% | +0.00% | +11.10% |
| wasax | 2022-12-15 | +16.29% | +0.00% | +16.67% |
| mxvhx | 2018-12-27 | +16.13% | +0.00% | +12.53% |
| gegax | 2006-12-27 | +16.12% | +0.00% | +11.26% |
| iastx | 2022-12-15 | +16.02% | +0.00% | +16.47% |
| vscvx | 2024-12-13 | +15.83% | +0.00% | +20.39% |
| tiscx | 2024-12-06 | +15.80% | +0.00% | +14.57% |
| sscgx | 2018-12-13 | +15.74% | +0.00% | +18.93% |
| line | 2015-01-26 | +15.71% | -1.27% | +9.25% |
| wmcvx | 2005-12-28 | +15.69% | +0.00% | +12.39% |
| wamcx | 2005-12-28 | +15.67% | +0.00% | +13.05% |
| swobx | 2000-12-28 | +15.63% | +0.93% | +13.00% |
| smefx | 2018-12-14 | +15.56% | +0.00% | +18.10% |
| algzx | 2017-12-15 | +15.47% | +0.00% | +12.60% |
| line | 2011-08-12 | +15.44% | -1.87% | +7.78% |
| acgix | 1999-12-13 | +15.43% | +0.00% | +14.34% |
| muxyx | 2024-12-13 | +15.37% | +0.00% | +16.04% |
| parwx | 2021-11-18 | +15.09% | +0.00% | +12.89% |
| sllax | 2018-12-13 | +14.91% | +0.00% | +18.12% |
| mrimx | 2024-12-13 | +14.77% | +0.00% | +18.78% |
| mxkjx | 2024-12-27 | +14.75% | +0.10% | +14.30% |
| mvssx | 2024-12-13 | +14.63% | +0.00% | +19.78% |
| jmvsx | 2021-12-13 | +14.62% | +0.00% | +12.92% |
| te | 2001-03-30 | +14.59% | -0.03% | +8.27% |
| vvoax | 2018-12-14 | +14.57% | +0.00% | +17.06% |
| mxwex | 2019-09-10 | +14.53% | +0.00% | +9.58% |
| wceax | 2022-12-15 | +14.53% | +0.00% | +16.62% |
| line | 2015-11-04 | -14.49% | +1.43% | +8.96% |
| trdfx | 2021-12-17 | +14.44% | +0.00% | +10.94% |
| vvosx | 2018-12-14 | +14.44% | +0.00% | +17.00% |
| ace | 1996-02-09 | -14.42% | -1.03% | +8.21% |
| algrx | 2021-12-15 | +14.41% | +0.00% | +14.30% |
| mxkjx | 2017-12-27 | +14.29% | +0.00% | +11.92% |
| ipoyx | 2022-12-15 | +14.22% | +0.00% | +14.76% |
| omsix | 2015-12-03 | +14.18% | +0.00% | +13.29% |
| pom | 1998-09-30 | +14.17% | -1.85% | +7.18% |
| imegx | 2022-12-15 | +14.13% | +0.00% | +14.64% |
| mxykx | 2023-12-27 | +14.11% | +0.00% | +12.48% |
| alzfx | 2021-12-15 | +14.05% | +0.00% | +14.03% |
| slgax | 2018-12-13 | +13.98% | +0.00% | +16.02% |
| te | 2009-05-05 | -13.86% | -1.88% | +9.41% |
| jgemx | 2024-12-20 | +13.83% | +0.00% | +10.73% |
| hlgzx | 2017-12-15 | +13.78% | -0.36% | +11.93% |
| mgfzx | 2021-12-15 | +13.72% | +1.62% | +12.70% |
| selcx | 2018-12-13 | +13.72% | +0.00% | +15.98% |
| mgosx | 2024-12-13 | +13.70% | +0.00% | +16.37% |
| rpxfx | 2019-12-16 | +13.70% | +1.17% | +8.94% |
| pom | 2009-09-17 | -13.66% | -0.13% | +7.18% |
| pvfix | 2024-12-13 | +13.63% | +0.00% | +14.65% |
| jvmrx | 2024-12-20 | +13.59% | +1.14% | +8.89% |
| waiox | 2011-12-28 | +13.54% | +0.00% | +10.76% |
| bbhlx | 2021-12-15 | +13.39% | +0.00% | +12.97% |
| line | 2012-06-28 | +13.38% | -1.05% | +7.56% |
| ace | 2001-04-30 | -13.37% | -1.30% | +6.83% |
| prnex | 1986-12-15 | +13.33% | +0.00% | +12.11% |
| ace | 2007-08-20 | +13.32% | -0.82% | +7.83% |
| swobx | 1999-12-30 | +13.26% | +0.99% | +9.60% |
| mxnzx | 2018-12-27 | +13.23% | +0.00% | +10.28% |
| rpfix | 2016-12-15 | +13.22% | +0.00% | +10.80% |
| waaex | 2005-12-28 | +13.09% | +0.00% | +10.72% |
| ace | 2003-04-22 | -13.03% | -1.82% | +11.22% |
| line | 2008-02-27 | -12.99% | -1.66% | +7.12% |
| mxtbx | 2019-09-10 | +12.87% | +0.00% | +8.77% |
| mxigx | 2018-12-27 | +12.85% | +0.00% | +10.78% |
| line | 2009-05-29 | +12.67% | -1.15% | +6.46% |
| oglix | 2018-12-06 | +12.67% | +0.00% | +14.21% |
| te | 2002-07-30 | +12.64% | -0.13% | +8.83% |
| dvzrx | 2025-12-15 | +12.57% | +0.00% | +11.94% |
| ace | 1999-10-25 | -12.56% | -0.36% | +7.83% |
| dbef | 2012-12-19 | +12.54% | +0.00% | +8.86% |
| amrgx | 2021-12-09 | +12.52% | +0.00% | +11.73% |
| te | 2009-07-15 | +12.52% | -1.24% | +6.78% |
| jlcwx | 2024-12-20 | +12.50% | +0.00% | +8.77% |
| rlsfx | 2019-12-16 | +12.34% | +0.95% | +8.27% |
| pom | 2011-08-11 | +12.24% | +0.00% | +6.50% |
| semcx | 2018-12-13 | +12.14% | +0.00% | +14.61% |
| iceqx | 2022-12-15 | +12.12% | +0.00% | +14.80% |
| wagox | 2011-12-28 | +12.12% | +0.00% | +10.15% |
| line | 2008-04-07 | -12.11% | -0.04% | +9.53% |
| jdeux | 2021-12-13 | +12.08% | +0.00% | +11.71% |
| te | 2002-10-28 | -12.08% | +0.34% | +7.75% |
| tirex | 2004-12-29 | +12.07% | +0.00% | +11.03% |
| kauax | 2019-12-05 | +12.04% | +0.00% | +10.67% |
| jdiux | 2024-12-20 | +12.01% | +0.00% | +9.97% |
| lprex | 2018-12-31 | +11.92% | +0.00% | +9.00% |
| te | 2003-09-25 | +11.89% | -1.95% | +7.72% |
| ace | 2000-07-12 | +11.87% | +0.40% | +6.43% |
| mxtfx | 2023-12-27 | +11.86% | +0.00% | +11.75% |
| lpsgx | 2018-12-31 | +11.80% | +0.00% | +8.82% |
| te | 2008-03-31 | +11.80% | -1.97% | +5.90% |
| auuyx | 2018-12-12 | +11.74% | +0.00% | +13.51% |
| msoox | 2021-12-15 | +11.73% | +1.29% | +10.85% |
| lprfx | 2018-12-31 | +11.72% | +0.00% | +8.78% |
| allfx | 2019-12-13 | +11.61% | +0.00% | +9.33% |
| sequx | 1986-02-07 | +11.58% | +0.25% | +9.07% |
| frccx | 2022-12-15 | +11.52% | +0.00% | +12.88% |
| jppex | 2020-12-11 | +11.44% | +0.00% | +9.37% |
| mpivx | 2022-12-13 | +11.44% | +0.71% | +12.09% |
| opmix | 2018-12-07 | +11.41% | +0.00% | +13.47% |
| tisex | 2024-12-06 | +11.41% | +0.00% | +10.78% |
| smisx | 2018-12-14 | +11.38% | +0.00% | +15.81% |
| mgisx | 2024-12-16 | +11.37% | -0.23% | +12.57% |
| pdinx | 2024-12-18 | +11.34% | +0.00% | +10.89% |
| svoax | 2018-12-13 | +11.34% | +0.00% | +13.50% |
| smeax | 2019-12-13 | +11.33% | +0.00% | +9.40% |
| igfrx | 2019-12-13 | +11.26% | +0.00% | +8.72% |
| auuyx | 2017-12-13 | +11.22% | +0.00% | +9.23% |
| mgosx | 2014-12-29 | +11.21% | +0.00% | +11.11% |
| line | 2013-10-21 | -11.16% | -1.81% | +8.97% |
| abrcx | 2019-12-13 | +11.07% | +0.00% | +8.92% |
| chtrx | 2018-12-14 | +11.03% | +0.00% | +15.05% |
| aiiex | 2019-12-13 | +11.02% | +0.00% | +8.54% |
| ace | 1997-02-21 | -11.01% | -0.96% | +6.51% |
| ace | 1995-09-01 | -10.99% | -1.59% | +5.58% |
| chftx | 2018-12-14 | +10.98% | +0.00% | +14.97% |
| lpsfx | 2018-12-31 | +10.84% | +0.00% | +8.04% |
| jsgtx | 2024-12-20 | +10.81% | +1.16% | +5.50% |
| ace | 2010-06-14 | +10.79% | -1.78% | +5.52% |
| ace | 2009-07-15 | -10.73% | -1.78% | +7.89% |
| ace | 2001-02-26 | +10.72% | -1.57% | +5.96% |
| mefzx | 2013-12-16 | +10.63% | +0.00% | +8.13% |
| sesvx | 2018-12-13 | +10.57% | +0.00% | +14.10% |
| evglx | 2017-12-29 | +10.52% | +0.00% | +8.05% |
| tigrx | 2024-12-06 | +10.51% | +0.00% | +9.57% |
| jdeux | 2019-10-08 | +10.47% | +0.00% | +8.59% |
| wamvx | 2005-12-28 | +10.41% | +0.00% | +7.53% |
| line | 2006-06-23 | +10.40% | -1.57% | +6.03% |
| lprdx | 2018-12-31 | +10.31% | +0.00% | +7.86% |
| glvix | 2019-12-13 | +10.29% | +0.38% | +7.75% |
| pom | 2010-07-07 | -10.26% | -1.56% | +6.30% |
| otcfx | 1993-12-21 | +10.03% | -0.42% | +9.80% |
| igfrx | 2018-12-14 | +10.02% | +0.00% | +12.36% |
| lpsdx | 2018-12-31 | +10.01% | +0.00% | +7.63% |
| cnsfx | 2018-12-14 | +9.96% | +0.00% | +11.66% |
| cnsax | 2018-12-14 | +9.87% | +0.00% | +11.59% |
| evagx | 2017-12-29 | +9.86% | +0.00% | +7.33% |
| esgs | 2017-12-26 | +9.85% | +0.00% | +8.56% |
| ityax | 2019-12-13 | +9.84% | +0.00% | +7.62% |
| scurx | 2018-12-17 | +9.83% | +0.00% | +13.49% |
| smefx | 2019-12-13 | +9.83% | +0.00% | +8.14% |
| mstqx | 2024-12-20 | +9.82% | +0.00% | +6.82% |
| ncbgx | 2015-12-11 | +9.81% | +0.00% | +8.79% |
| line | 2008-05-07 | -9.74% | -0.45% | +4.98% |
| mxsfx | 2024-12-27 | +9.69% | +0.22% | +9.90% |
| pabax | 2024-12-18 | +9.69% | +0.06% | +10.23% |
| ace | 2005-04-29 | +9.67% | +0.02% | +7.75% |
| te | 2002-10-23 | +9.58% | -1.32% | +12.84% |
| aiiex | 2018-12-14 | +9.56% | +0.00% | +12.02% |
| ace | 2000-07-10 | +9.53% | -1.87% | +8.32% |
| ace | 2008-12-16 | +9.48% | +0.90% | +7.18% |
| hcp | 1987-01-27 | +9.43% | -0.43% | +5.56% |
| ufs | 1986-05-09 | +9.36% | -1.10% | +4.97% |
| dnldx | 1991-12-31 | +9.31% | +0.00% | +5.77% |
| line | 2011-10-10 | +9.24% | -0.44% | +5.10% |
| gtsax | 2018-12-14 | +9.23% | +0.00% | +13.87% |
| mgrpx | 2020-12-11 | +9.21% | +0.10% | +7.05% |
| irbax | 2022-12-15 | +9.14% | +0.00% | +9.44% |
| acstx | 1999-12-13 | +9.13% | +0.00% | +9.45% |
| te | 2009-08-21 | -9.13% | -0.66% | +5.12% |
| gtsax | 2019-12-13 | +9.11% | +0.00% | +7.62% |
| fgskx | 2020-12-04 | +9.10% | +0.99% | +7.01% |
| pequx | 1990-11-30 | +9.08% | +0.42% | +6.23% |
| mirsx | 2024-12-13 | +9.04% | +0.00% | +14.88% |
| qalgx | 2020-12-04 | +9.04% | +0.69% | +7.68% |
| te | 2007-08-20 | +9.01% | -0.76% | +4.72% |
| line | 2012-06-29 | +9.00% | -1.04% | +6.52% |
| ace | 1993-08-10 | +8.99% | -0.39% | +9.65% |
| omsix | 2019-12-13 | +8.92% | +0.06% | +7.60% |
| bflax | 1998-12-10 | +8.87% | -0.78% | +10.11% |
| asmmx | 2019-12-13 | +8.86% | +0.00% | +6.98% |
| te | 2014-10-21 | +8.78% | -0.73% | +4.56% |
| inutx | 1999-12-22 | +8.76% | +0.00% | +6.45% |
| pwlix | 2024-12-26 | +8.75% | +0.00% | +7.64% |
| mfllx | 2020-12-11 | +8.71% | -1.11% | +7.45% |
| polyx | 2024-12-18 | +8.66% | +0.00% | +8.99% |
| smisx | 2019-12-13 | +8.66% | +0.00% | +6.79% |
| ryaix | 2008-12-04 | +8.63% | +1.54% | +6.28% |
| line | 2006-10-12 | +8.61% | -1.51% | +5.09% |
| jrsnx | 2018-12-20 | +8.57% | +0.00% | +13.52% |
| ace | 2009-09-28 | +8.56% | -0.82% | +6.68% |
| fgsax | 2020-12-04 | +8.53% | +0.98% | +6.53% |
| gmrix | 2006-12-27 | +8.52% | +0.00% | +7.80% |
| lpjkx | 2018-12-31 | +8.49% | +0.00% | +6.24% |
| pgsix | 2024-12-18 | +8.45% | +0.00% | +8.95% |
| lpjrx | 2018-12-31 | +8.43% | +0.00% | +6.19% |
| te | 2004-07-29 | -8.43% | +0.00% | +5.58% |
| lvitx | 2024-12-06 | +8.42% | +0.00% | +7.97% |
| ityax | 2018-12-14 | +8.40% | +0.00% | +13.80% |
| mxerx | 2024-12-27 | +8.38% | +0.57% | +8.69% |
| ace | 1995-01-24 | +8.37% | -1.59% | +6.35% |
| bdokx | 2014-12-31 | +8.36% | +0.00% | +11.26% |
| ace | 2003-03-17 | +8.27% | -0.58% | +14.23% |
| pom | 1998-04-30 | -8.27% | +0.51% | +4.87% |
| mxjvx | 2024-12-27 | +8.26% | +0.23% | +7.96% |
| ace | 2003-03-19 | -8.25% | -0.25% | +6.57% |
| tilvx | 2019-12-06 | +8.25% | +0.90% | +6.07% |
| gghcx | 2018-12-14 | +8.24% | +0.00% | +13.85% |
| pcvax | 2005-12-13 | +8.22% | +0.00% | +9.44% |
| te | 2008-01-31 | -8.21% | -1.48% | +4.79% |
| muxyx | 2014-12-29 | +8.20% | +0.00% | +8.92% |
| te | 2002-09-27 | -8.20% | -0.38% | +9.07% |
| ifutx | 2018-12-14 | +8.17% | +0.00% | +11.51% |
| te | 2003-01-06 | +8.17% | +0.95% | +7.81% |
| awsax | 2018-12-14 | +8.16% | +0.00% | +11.55% |
| iautx | 2018-12-14 | +8.15% | +0.00% | +11.48% |
| ftpsx | 2019-12-13 | +8.11% | +0.00% | +6.12% |
| gghsx | 2018-12-14 | +8.05% | +0.00% | +13.71% |
| pwtax | 2024-12-17 | +8.05% | +0.00% | +9.65% |
| dlqax | 1996-12-30 | +8.04% | +0.33% | +7.76% |
| iarax | 2019-12-13 | +8.03% | +0.00% | +6.54% |
| mxmsx | 2024-12-27 | +8.03% | +0.67% | +8.05% |
| lprax | 2023-12-22 | +7.97% | +0.00% | +6.70% |
| te | 2009-01-02 | +7.97% | -0.16% | +8.80% |
| te | 1986-04-17 | -7.97% | -0.87% | +4.07% |
| pom | 2011-04-27 | +7.96% | -0.88% | +4.06% |
| lphkx | 2018-12-31 | +7.94% | +0.00% | +5.66% |
| rseix | 2016-12-15 | +7.93% | +0.00% | +5.47% |
| lphax | 2018-12-31 | +7.89% | +0.00% | +5.58% |
| kauax | 2020-12-04 | +7.86% | +0.67% | +6.67% |
| line | 2007-03-20 | -7.78% | -1.07% | +4.62% |
| ryshx | 2023-12-12 | +7.78% | +0.00% | +11.87% |
| msoox | 2020-12-11 | +7.77% | -0.40% | +6.51% |
| ssgfx | 1997-11-28 | +7.77% | +0.00% | +4.39% |
| giffx | 2019-12-13 | +7.65% | +0.00% | +6.45% |
| tln | 2016-06-07 | +7.65% | +0.58% | +13.04% |
| ace | 1995-03-29 | +7.60% | -0.49% | +5.83% |
| evttx | 2017-12-29 | +7.58% | +0.00% | +5.29% |
| bbhlx | 2017-12-15 | +7.57% | +0.00% | +6.64% |
| line | 2006-04-17 | +7.57% | +1.46% | +3.84% |
| mphzx | 2012-12-19 | +7.57% | +0.00% | +7.01% |
| te | 2005-03-01 | -7.57% | -0.74% | +4.19% |
| esmsx | 2019-12-13 | +7.56% | +0.00% | +5.74% |
| acgix | 2019-12-13 | +7.55% | +0.00% | +6.35% |
| asisx | 2018-12-14 | +7.55% | +0.00% | +9.38% |
| tnvdx | 2017-12-14 | +7.54% | +0.00% | +6.79% |
| mipzx | 2021-12-15 | +7.52% | +0.00% | +6.66% |
| pom | 1998-10-02 | -7.52% | -0.23% | +3.88% |
| evmlx | 2017-12-29 | +7.51% | +0.00% | +5.64% |
| pom | 1987-08-11 | +7.51% | -1.08% | +4.30% |
| rymhx | 2023-12-12 | +7.49% | +0.00% | +10.37% |
| lprax | 2018-12-31 | +7.47% | +0.00% | +5.86% |
| mxgsx | 2024-12-27 | +7.47% | -0.15% | +10.02% |
| lpbkx | 2018-12-31 | +7.46% | +0.00% | +5.53% |
| lpvkx | 2018-12-31 | +7.46% | +0.00% | +5.20% |
| ace | 2003-10-01 | +7.45% | -1.25% | +5.23% |
| ace | 1998-07-07 | +7.44% | +0.30% | +4.27% |
| lpvax | 2018-12-31 | +7.44% | +0.00% | +5.15% |
| asisx | 2019-12-13 | +7.38% | +0.00% | +4.80% |
| mdbzx | 2012-12-19 | +7.38% | +0.00% | +6.61% |
| ace | 2011-12-21 | +7.37% | -1.00% | +4.60% |
| tedmx | 1997-12-15 | +7.33% | +0.00% | +3.90% |
| tnxax | 2017-12-14 | +7.29% | +0.00% | +6.55% |
| ssbox | 2024-12-27 | +7.28% | +0.00% | +7.30% |
| ace | 2009-09-29 | -7.26% | -0.09% | +6.70% |
| pxtix | 2015-12-16 | +7.21% | -1.17% | +3.67% |
| esmax | 2019-12-13 | +7.19% | +0.00% | +5.44% |
| fiusx | 1997-11-28 | +7.19% | +0.33% | +5.05% |
| mxzhx | 2018-09-11 | +7.19% | +0.00% | +6.00% |
| specx | 2000-12-15 | +7.15% | +0.00% | +15.03% |
| mxnzx | 2024-12-27 | +7.10% | +0.34% | +7.66% |
| ace | 1994-04-07 | +7.09% | -0.89% | +10.18% |
| line | 2007-03-22 | +7.07% | -0.29% | +4.57% |
| ftpsx | 2018-12-14 | +7.06% | +0.00% | +12.68% |
| vafax | 2019-12-13 | +7.05% | +0.00% | +5.16% |
| ace | 2001-05-29 | +7.03% | -1.06% | +3.64% |
| dfgex | 2010-12-09 | +7.03% | +0.00% | +6.47% |
| tnyrx | 2017-12-14 | +7.02% | +0.00% | +6.39% |
| asiax | 2018-12-14 | +7.01% | +0.00% | +8.92% |
| pom | 2002-07-29 | -7.00% | -1.69% | +13.40% |
| mxtfx | 2024-12-27 | +6.94% | +0.55% | +6.90% |
| ace | 2010-06-11 | -6.92% | -0.25% | +3.93% |
| asiax | 2019-12-13 | +6.90% | +0.00% | +4.38% |
| line | 2007-03-21 | -6.88% | -0.80% | +5.44% |
| vadfx | 2018-12-14 | +6.86% | +0.00% | +11.40% |
| te | 2001-10-02 | +6.85% | -0.39% | +9.20% |
| te | 1994-04-18 | -6.80% | -0.64% | +3.82% |
| scijx | 2018-12-17 | +6.79% | +0.00% | +8.88% |
| pom | 2000-03-01 | -6.78% | -0.93% | +4.33% |
| rofix | 2016-12-15 | +6.77% | +0.00% | +4.15% |
| vaffx | 2019-12-13 | +6.77% | +0.00% | +4.92% |
| jacdx | 2024-12-20 | +6.74% | +0.00% | +4.89% |
| te | 2004-02-09 | +6.73% | +0.20% | +5.35% |
| pom | 2000-03-14 | -6.72% | -0.93% | +4.66% |
| dfitx | 2016-12-14 | +6.71% | -1.76% | +10.59% |
| te | 2009-12-11 | -6.71% | -0.81% | +3.37% |
| ssbsx | 2024-12-27 | +6.70% | +0.00% | +7.08% |
| ace | 2005-09-14 | +6.69% | -1.59% | +3.91% |
| lpsax | 2023-12-22 | +6.68% | +0.00% | +5.58% |
| rvt | 2009-03-04 | +6.68% | +0.70% | +13.49% |
| lpsax | 2018-12-31 | +6.67% | +0.00% | +5.08% |
| ace | 1995-09-13 | -6.65% | +0.36% | +3.65% |
| mxugx | 2024-12-27 | +6.64% | +0.12% | +5.81% |
| timvx | 2024-12-06 | +6.64% | +0.00% | +7.66% |
| line | 2006-07-31 | +6.63% | -0.30% | +3.67% |
| ace | 2012-04-19 | +6.58% | -0.61% | +3.80% |
| esmsx | 2018-12-14 | +6.57% | +0.00% | +8.26% |
| pom | 2009-12-01 | -6.57% | -1.06% | +4.55% |
| tammx | 2021-12-21 | +6.57% | +0.00% | +3.31% |
| ufs | 1987-05-08 | +6.56% | -1.21% | +9.31% |
| line | 2006-10-11 | +6.54% | -1.26% | +4.63% |
| etgix | 2008-03-11 | +6.52% | +1.69% | +5.53% |
| pom | 2014-02-12 | +6.50% | -1.33% | +4.42% |
| dnldx | 1990-12-31 | +6.47% | +0.00% | +6.59% |
| pom | 2015-09-14 | +6.45% | -1.53% | +3.28% |
| auuyx | 2022-12-07 | +6.43% | +0.00% | +5.09% |
| gtddx | 1995-12-20 | +6.42% | +0.00% | +3.68% |
| mxzhx | 2017-12-28 | +6.42% | +0.19% | +4.83% |
| line | 2009-05-26 | +6.40% | -0.95% | +11.39% |
| sequx | 1988-02-05 | +6.37% | +0.28% | +5.47% |
| tisbx | 2024-12-06 | +6.36% | +0.00% | +6.03% |
| ryjsx | 2023-12-12 | +6.34% | +0.00% | +3.26% |
| te | 2005-09-30 | -6.33% | -0.93% | +3.46% |
| icsfx | 2019-12-13 | +6.32% | +0.00% | +5.43% |
| tilgx | 2024-12-06 | +6.32% | +0.00% | +4.16% |
| pom | 2009-07-13 | -6.30% | +0.15% | +3.62% |
| te | 2013-01-30 | +6.30% | -0.73% | +3.26% |
| sivix | 2019-12-20 | +6.29% | +0.00% | +5.37% |
| ace | 2013-07-08 | -6.28% | -0.99% | +3.68% |
| mpw | 2009-03-17 | +6.27% | +0.99% | +11.91% |
| te | 2001-04-10 | +6.26% | +0.26% | +4.20% |
| ace | 2013-05-06 | +6.25% | -0.78% | +3.38% |
| ace | 2011-08-30 | -6.24% | -0.65% | +4.94% |
| drcvx | 1988-12-13 | +6.24% | +0.00% | +7.12% |
| mxtbx | 2017-12-28 | +6.22% | +0.19% | +4.80% |
| mxzhx | 2023-09-08 | +6.22% | +0.00% | +5.89% |
| ssbyx | 2024-12-27 | +6.21% | +0.00% | +6.91% |
| acstx | 2019-12-13 | +6.20% | +0.00% | +5.35% |
| line | 2012-02-29 | +6.19% | -1.45% | +3.15% |
| ace | 2011-04-04 | +6.17% | +0.80% | +3.40% |
| te | 2008-05-06 | +6.16% | +1.36% | +3.48% |
| esmax | 2018-12-14 | +6.15% | +0.00% | +7.97% |
| mczzx | 2020-12-11 | +6.14% | +0.00% | +5.64% |
| te | 2007-10-25 | +6.12% | -1.67% | +4.48% |
| jgrsx | 2024-12-20 | +6.11% | +0.00% | +3.52% |
| te | 2005-06-09 | +6.10% | -1.37% | +3.22% |
| pom | 2005-05-31 | -6.09% | -1.10% | +3.07% |
| te | 2009-06-29 | -6.08% | +0.92% | +3.27% |
| mxwex | 2017-12-28 | +6.07% | +0.28% | +4.40% |
| ace | 2004-10-28 | +6.06% | -0.21% | +8.59% |
| vadcx | 2018-12-14 | +6.01% | +0.00% | +10.69% |
| te | 1993-02-22 | -5.97% | -0.56% | +3.08% |
| pom | 2003-05-16 | +5.96% | -0.60% | +4.01% |
| tln | 2016-02-24 | -5.96% | +1.52% | +8.50% |
| tqccx | 1999-12-14 | +5.96% | +0.00% | +4.00% |
| ace | 2005-02-07 | +5.95% | +0.74% | +3.61% |
| hnmvx | 2024-12-20 | +5.94% | +0.00% | +3.30% |
| ace | 1998-02-05 | +5.91% | +0.19% | +4.33% |
| ace | 2013-07-09 | +5.89% | +0.69% | +3.00% |
| ace | 2010-07-12 | +5.89% | -1.54% | +7.14% |
| lcefx | 2018-12-14 | +5.89% | +0.00% | +9.34% |
| line | 2014-08-08 | -5.89% | -1.14% | +3.40% |
| pom | 2016-03-04 | -5.89% | +0.99% | +5.67% |
| tnvdx | 2024-12-13 | +5.89% | +0.00% | +6.81% |
| wgrox | 1991-12-30 | +5.89% | +1.69% | +3.38% |
| te | 1999-03-24 | -5.88% | +1.18% | +4.72% |
| tnxax | 2024-12-13 | +5.88% | +0.00% | +6.81% |
| line | 2013-04-01 | +5.87% | -0.49% | +4.46% |
| ssfox | 2024-12-27 | +5.85% | +0.00% | +6.03% |
| tnyrx | 2024-12-13 | +5.85% | +0.00% | +6.82% |
| line | 2006-06-26 | -5.83% | -0.49% | +3.90% |
| lpdkx | 2018-12-31 | +5.83% | +0.00% | +3.77% |
| ace | 1993-11-11 | -5.79% | -0.40% | +3.95% |
| pom | 2002-06-18 | +5.78% | +0.73% | +4.70% |
| lceax | 2018-12-14 | +5.77% | +0.00% | +9.30% |
| te | 2014-12-24 | -5.77% | -1.35% | +5.00% |
| secex | 1985-04-29 | +5.76% | -1.83% | +4.40% |
| rvt | 1999-03-04 | +5.75% | -1.96% | +5.39% |
| te | 2009-07-16 | -5.75% | +0.00% | +5.62% |
| prrsx | 2022-09-08 | +5.73% | -1.96% | +3.64% |
| ace | 2004-01-13 | +5.70% | -0.78% | +3.39% |
| ldfvx | 1997-11-12 | +5.67% | -1.35% | +4.73% |
| ivinx | 1990-12-28 | +5.65% | +1.43% | +3.27% |
| wsgrx | 2022-12-15 | +5.64% | +0.00% | +10.65% |
| hctpf | 2016-02-11 | +5.63% | +0.00% | +8.89% |
| mxqbx | 2017-12-28 | +5.59% | +0.19% | +4.46% |
| line | 2012-04-30 | -5.57% | -0.20% | +2.85% |
| ace | 2013-04-08 | +5.56% | -0.54% | +2.78% |
| pom | 2011-05-10 | +5.56% | +0.25% | +4.12% |
| asrax | 2019-12-13 | +5.54% | +0.00% | +4.63% |
| te | 1987-09-30 | +5.54% | -1.56% | +3.12% |
| pom | 2010-07-22 | +5.51% | -1.35% | +3.29% |
| ace | 1996-10-07 | +5.50% | +1.78% | +3.56% |
| vecrx | 2019-07-31 | +5.49% | -1.16% | +5.73% |
| grf | 2024-11-29 | +5.48% | -1.33% | +11.29% |
| ssckx | 2024-12-27 | +5.47% | +0.00% | +6.43% |
| tunix | 2021-12-21 | +5.47% | +0.00% | +4.92% |
| ace | 2005-09-26 | -5.44% | +0.48% | +3.57% |
| amrgx | 2022-12-05 | +5.44% | -1.72% | +8.05% |
| teg | 25-Jun-14 | +5.44% | -1.28% | +13.21% |
| lpsax | 2013-12-31 | +5.43% | +0.00% | +5.36% |
| line | 2010-05-28 | +5.40% | +1.96% | +5.46% |
| mxwex | 2023-09-08 | +5.40% | +0.00% | +5.12% |
| oscix | 2019-12-13 | +5.40% | +0.34% | +4.38% |
| pom | 1991-10-02 | -5.40% | +0.00% | +4.21% |
| pom | 2000-12-26 | +5.37% | -0.73% | +4.06% |
| gapux | 2024-12-06 | +5.32% | +0.00% | +5.04% |
| ieifx | 2019-12-13 | +5.31% | +0.00% | +4.58% |
| pom | 1995-06-14 | +5.27% | -0.57% | +4.60% |
| teg | 10-Oct-14 | +5.25% | -0.89% | +3.21% |
| mac | 2008-11-13 | +5.24% | +1.20% | +35.70% |
| iegfx | 2019-12-13 | +5.23% | +0.00% | +3.30% |
| nie | 2008-12-24 | +5.23% | +0.08% | +7.99% |
| ace | 2004-01-14 | -5.22% | +0.21% | +2.79% |
| line | 2012-04-26 | +5.22% | -0.83% | +2.66% |
| aceix | 2019-12-13 | +5.21% | +0.00% | +4.49% |
| hacsx | 2014-12-18 | +5.20% | +0.00% | +3.57% |
| mxxjx | 2024-12-27 | +5.20% | +0.00% | +4.72% |
| origx | 2008-11-26 | +5.15% | +1.80% | +3.23% |
| ace | 1995-06-14 | -5.14% | -0.44% | +4.00% |
| pom | 2001-05-15 | -5.08% | -0.88% | +2.92% |
| eaasx | 2008-12-23 | +5.07% | +1.81% | +3.25% |
| fiusx | 1996-12-30 | +5.04% | +0.31% | +4.07% |
| line | 2012-10-16 | +5.01% | -0.63% | +2.94% |

View File

@ -51,6 +51,18 @@ def main() -> int:
if not capgf.get(date):
print(f"BAD {sym} drop_capg_copy {date}: no capitalGain row")
bad += 1
for date, cells in (d.get("history") or {}).items():
p = FROZEN / f"{sym}-history.csv"
if not p.exists():
p = DATA / f"{sym}-history.csv"
dates = {l.split(",")[0][:10] for l in p.read_text(errors="replace").splitlines()[1:]} if p.exists() else set()
if date not in dates:
print(f"BAD {sym} history {date}: date not in history file")
bad += 1
for col in cells:
if col not in ("Open", "High", "Low", "Close", "Adj Close", "Volume"):
print(f"BAD {sym} history {date}: unknown column {col!r}")
bad += 1
for key, rows_f in files.items():
ops = d.get(key)
if not ops or rows_f is None:

144
scripts/fix_adj_misalign.py Normal file
View File

@ -0,0 +1,144 @@
#!/usr/bin/env python3
"""Repair Yahoo adj-close / event-date misalignment (see
scripts/scan_adj_misalign.py).
For a hit date D (adj spikes, close flat, price drops later) the repair is
purely arithmetic and leaves cumulative returns unchanged:
r(t) = adj(t)/close(t); k = r(D)/r(D-1) # the bogus spike
f = 1 - 1/k # dist fraction Yahoo applied
d = f * close(D-1) # implied per-share dist
E = first t > D with close(t) < close(t-1) - 0.5*d # true ex-div date
for t in [D, E): adj(t) *= (1 - f) # restore pre-event ratio
For t before D and from E on the series is already consistent, so only the
[D, E) window is patched. The patch is written as a 'history' correction
(overrides/corrections/{SYM}.json, merged, with a note), so it survives
re-downloads that rewrite the raw CSVs with the same Yahoo values.
Usage:
python3 scripts/fix_adj_misalign.py [--only SYM[,SYM...]] [--dry-run]
"""
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from scripts.scan_adj_misalign import scan # noqa: E402
ROOT_DEFAULT = Path.home() / "prog/fin/stocks"
CORR = Path(__file__).resolve().parent.parent / "overrides" / "corrections"
NOTE = ("adj-close repair: Yahoo dated a distribution on {d} (record date) "
"but the price went ex-div on {e} (close {c0:.2f} -> {c1:.2f}, drop "
"~{d_amt:.3f}); the raw Adj Close column spiked {spike:+.1%} on {d} "
"and snapped back later. Cells in [{d}, {e}) rescaled by (1-f) so "
"the adjusted path is smooth; cumulative returns are unchanged. "
"Mechanism: scripts/fix_adj_misalign.py, "
"scan: reports/adj_misalign/scan.md.")
def load_rows(sym: str, root: Path):
import data as D
fr = D.FROZEN_DIR / f"{sym}-history.csv"
f = fr if (D.FROZEN_DIR.is_dir() and fr.exists()) else root / f"{sym}-history.csv"
rows = []
for l in f.read_text(errors="replace").splitlines()[1:]:
if not l.strip():
continue
a = l.split(",")
if len(a) < 6:
continue
try:
rows.append((a[0][:10], float(a[4]), float(a[5])))
except ValueError:
continue
return rows
def repair(sym: str, date: str, rows) -> dict:
"""Return {date: {"Adj Close": value}} patch, or {} if not repairable."""
i = next((k for k, r in enumerate(rows) if r[0] == date), None)
if i is None or i == 0:
return {}
dates, closes, adjs = (list(x) for x in zip(*rows))
r = [a / c for _, c, a in rows]
if r[i - 1] <= 0 or r[i] <= 0 or r[i] <= r[i - 1]:
return {}
f = 1.0 - r[i - 1] / r[i]
if not (0.005 < f < 0.95):
return {}
d_amt = f * closes[i - 1]
E = None
for t in range(i + 1, min(i + 11, len(rows))):
if closes[t] < closes[t - 1] - 0.5 * d_amt:
E = t
break
if E is None:
return {}
k = 1.0 - f
patch = {}
for t in range(i, E):
patch[dates[t]] = {"Adj Close": round(adjs[t] * k, 6)}
return patch, f, d_amt, E
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--data", type=Path, default=ROOT_DEFAULT)
ap.add_argument("--only", default="", help="comma-separated symbols")
ap.add_argument("--dry-run", action="store_true")
args = ap.parse_args()
only = {s.strip().lower() for s in args.only.split(",") if s.strip()}
hits = scan(args.data, 0.05, 0.02, 3)
if only:
hits = [h for h in hits if h["symbol"] in only]
by_sym: dict[str, list] = {}
for h in hits:
by_sym.setdefault(h["symbol"], []).append(h)
applied = 0
for sym, hs in sorted(by_sym.items()):
rows = load_rows(sym, args.data)
cells, notes = {}, []
for h in hs:
try:
patch, f, d_amt, E = repair(sym, h["date"], rows)
except Exception as ex:
print(f" !! {sym} {h['date']}: {ex}")
continue
if not patch:
print(f" -- {sym} {h['date']}: not repairable (skipped)")
continue
cells.update(patch)
dates = [x[0] for x in rows]
di = next(k for k, r in enumerate(rows) if r[0] == h["date"])
notes.append(NOTE.format(d=h["date"], e=dates[E], c0=rows[di - 1][1],
c1=rows[E][1], d_amt=d_amt,
spike=h["adj_ret"]))
print(f" {sym} {h['date']}: patch {len(patch)} cell(s), "
f"implied dist {d_amt:.3f}, ex-div {dates[E]}")
if not cells:
continue
pf = CORR / f"{sym.upper()}.json"
d = json.loads(pf.read_text()) if pf.exists() else {"symbol": sym}
d.setdefault("symbol", sym)
d.setdefault("note", "")
if notes:
d["note"] = (d["note"] + " " if d["note"] else "") + \
" ".join(notes).strip()
d.setdefault("history", {})
for dt, cells_ in cells.items():
d["history"].setdefault(dt, {}).update(cells_)
if not args.dry_run:
pf.write_text(json.dumps(d, indent=2) + "\n")
applied += 1
print(f"{'(dry) ' if args.dry_run else ''}wrote {pf.name}: "
f"{sum(len(v) for v in d['history'].values())} cell(s)")
print(f"{'dry-run: ' if args.dry_run else ''}{applied} symbol(s) repaired "
f"of {len(by_sym)} candidate(s)")
return 0
if __name__ == "__main__":
main()

View File

@ -0,0 +1,115 @@
#!/usr/bin/env python3
"""Scan for Yahoo adj-close/event-date misalignment.
Yahoo's Adj Close column applies a distribution's adjustment on the EVENT
date from its event feed. When that date precedes the actual ex-div date
(the market price only drops later), the adjusted series shows a bogus
spike on the event date and a snap-back a day or two later.
Signature per symbol: a date D with |adj return| >= --min-adj (default 5%)
while |close return| on D is < --max-close (default 2%), AND the close
drops by more than half the implied adjustment within --horizon following
sessions (the price finally going ex-div).
Usage: python3 scripts/scan_adj_misalign.py [--data DIR] [--out PATH]
Prints a summary and a table; writes markdown to --out (default
reports/adj_misalign/scan.md).
"""
import argparse
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
ROOT_DEFAULT = Path.home() / "prog/fin/stocks"
def scan(root: Path, min_adj: float, max_close: float, horizon: int) -> list[dict]:
import data as D
hits = []
syms = sorted({f.name[:-len("-history.csv")] for f in root.glob("*-history.csv")})
for i, sym in enumerate(syms):
fr = D.FROZEN_DIR / f"{sym}-history.csv"
f = fr if (D.FROZEN_DIR.is_dir() and fr.exists()) else root / f"{sym}-history.csv"
if not f.exists():
continue
rows = []
for l in f.read_text(errors="replace").splitlines()[1:]:
if not l.strip():
continue
r = l.split(",")
if len(r) < 6:
continue
try:
cl, aj = float(r[4]), float(r[5])
except ValueError: # e.g. '%!f(<nil>)' from an old goget bug
continue
rows.append((r[0], cl, aj))
if len(rows) < 3:
continue
closes = [x[1] for x in rows]
adjs = [x[2] for x in rows]
for i in range(1, len(rows)):
if closes[i - 1] <= 0 or adjs[i - 1] <= 0 or closes[i] <= 0:
continue
adj_ret = adjs[i] / adjs[i - 1] - 1
cl_ret = closes[i] / closes[i - 1] - 1
if abs(adj_ret) < min_adj or abs(cl_ret) >= max_close:
continue
# the drop has to land within the next `horizon` sessions
j = i + horizon
if j >= len(rows) or closes[j] <= 0:
continue
later_drop = (closes[i - 1] - closes[j]) / closes[i - 1]
implied = min(abs(adj_ret), 0.6) # cap: events rarely exceed 60%
if later_drop < 0.5 * implied:
continue
hits.append({
"symbol": sym,
"date": rows[i][0],
"adj_ret": round(adj_ret, 4),
"close_ret": round(cl_ret, 4),
"close_before": closes[i - 1],
"close_after_horizon": closes[j],
"drop_later": round(later_drop, 4),
})
if i % 500 == 0:
print(f"{i}/{len(syms)} ...", file=sys.stderr)
return hits
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--data", type=Path, default=ROOT_DEFAULT)
ap.add_argument("--min-adj", type=float, default=0.05)
ap.add_argument("--max-close", type=float, default=0.02)
ap.add_argument("--horizon", type=int, default=3)
ap.add_argument("--out", type=Path, default=None)
args = ap.parse_args()
hits = scan(args.data, args.min_adj, args.max_close, args.horizon)
hits.sort(key=lambda h: -abs(h["adj_ret"]))
print(json.dumps({"hits": len(hits),
"symbols": len({h["symbol"] for h in hits})}, indent=1))
for h in hits[:30]:
print(f"{h['symbol']:8s} {h['date']} adj {h['adj_ret']:+.1%} "
f"(close {h['close_ret']:+.2%}), drop later {h['drop_later']:.1%}")
if args.out:
args.out.parent.mkdir(parents=True, exist_ok=True)
lines = ["# Adj-close / event-date misalignment scan",
"",
f"params: min_adj={args.min_adj} max_close={args.max_close} "
f"horizon={args.horizon}; {len(hits)} hits on "
f"{len({h['symbol'] for h in hits})} symbols",
"",
"| symbol | date | adj ret | close ret | later drop |",
"|---|---|---|---|---|"]
lines += [f"| {h['symbol']} | {h['date']} | {h['adj_ret']:+.2%} | "
f"{h['close_ret']:+.2%} | {h['drop_later']:+.2%} |" for h in hits]
args.out.write_text("\n".join(lines) + "\n")
print(f"-> {args.out}")
return 0
if __name__ == "__main__":
main()

View File

@ -189,6 +189,37 @@ def main() -> int:
D.CORRECTIONS_DIR = real_corr
finally:
shutil.rmtree(tmp, ignore_errors=True)
# --- history cell corrections (misaligned Yahoo adj-close spike) ---
tmp = pathlib.Path(tempfile.mkdtemp(prefix="fd-"))
try:
root = tmp / "data"; cache = tmp / "cache"; corrdir = tmp / "corr"
root.mkdir(parents=True); cache.mkdir(); corrdir.mkdir()
(root / "ghh-history.csv").write_text(
"Date,Open,High,Low,Close,Adj Close,Volume\n"
"2020-12-10,30.10,30.10,30.10,30.10,11.247889,1000\n"
"2020-12-11,30.10,30.10,30.10,30.10,14.545517,1000\n"
"2020-12-14,23.22,23.22,23.22,23.22,11.220826,1000\n")
(corrdir / "GHH.json").write_text(json.dumps(
{"history": {"2020-12-11": {"Adj Close": 11.247889}}}))
real_corr = D.CORRECTIONS_DIR
D.CORRECTIONS_DIR = corrdir
try:
b = D.load_bundle(root=root, cache=cache)
t10, t11, t14 = (pd.Timestamp(x) for x in
("2020-12-10", "2020-12-11", "2020-12-14"))
check("history: adj cell patched",
b.adj["ghh"][t11] == 11.247889)
check("history: other adj cells untouched",
b.adj["ghh"][t10] == 11.247889 and b.adj["ghh"][t14] == 11.220826)
check("history: raw close untouched",
b.close["ghh"][t11] == 30.10 and b.close["ghh"][t14] == 23.22)
b2 = D.load_bundle(root=root, cache=cache)
check("history: idempotent on reload", b2.adj["ghh"][t11] == 11.247889)
finally:
D.CORRECTIONS_DIR = real_corr
finally:
shutil.rmtree(tmp, ignore_errors=True)
print(f"\n{PASS} passed, {FAIL} failed")
return 1 if FAIL else 0