diff --git a/data.py b/data.py index 1615b08..3869a61 100644 --- a/data.py +++ b/data.py @@ -128,7 +128,20 @@ def apply_invariants(div: pd.DataFrame, capg: pd.DataFrame) -> None: div.at[ts, sym] = 0.0 -def _apply_corrections(sym: str, suffix: str, ser: pd.Series) -> pd.Series: +def _apply_corrections(sym: str, suffix: str, ser: pd.Series, + col: str | None = None) -> pd.Series: + if suffix == "history": + # {date: {col: value}} — patch individual OHLC cells (e.g. a + # misaligned Yahoo adj-close spike). Only touches the requested col. + ops = _load_corrections(sym).get("history") + if not ops or col is None: + return ser + d = {ts: float(v) for ts, v in ser.items()} + for dstr, cells in ops.items(): + ts = pd.Timestamp(dstr) + if ts in d and col in cells: + d[ts] = float(cells[col]) + return pd.Series(d).sort_index() key = {"dividend": "dividends", "capitalGain": "capitalGains"}.get(suffix) ops = _load_corrections(sym).get(key) if key else None if not ops: @@ -238,7 +251,7 @@ def _read_panel(root: Path, suffix: str, col: str) -> pd.DataFrame: except Exception: continue if col in df.columns: - cols.append(_apply_corrections(sym, suffix, df[col])) + cols.append(_apply_corrections(sym, suffix, df[col], col)) syms.append(sym) if not cols: return pd.DataFrame() @@ -362,7 +375,9 @@ def _apply_changes(root: Path, cache: Path, old: dict, new: dict) -> None: def _recompute_corrected(symfile: str) -> None: """corrections/{SYM}.json changed: re-derive the symbol's dividend/capital-gain series from the base file + corrections.""" - for sfx, colname, key in (("dividend", "Dividends", "div"), + for sfx, colname, key in (("history", "Adj Close", "adj"), + ("history", "Close", "close"), + ("dividend", "Dividends", "div"), ("capitalGain", "Capital Gains", "capg")): sym = None p = None @@ -382,8 +397,8 @@ def _apply_changes(root: Path, cache: Path, old: dict, new: dict) -> None: continue if colname not in df.columns: continue - updates[key][sym] = _apply_corrections(sym, sfx, df[colname]) \ - .fillna(0.0) + ser = _apply_corrections(sym, sfx, df[colname], colname) + updates[key][sym] = ser if sfx == "history" else ser.fillna(0.0) authoritative.add(sym) for fname in changed: @@ -401,9 +416,11 @@ def _apply_changes(root: Path, cache: Path, old: dict, new: dict) -> None: continue # e.g. a file mid-download if sfx == "history": if "Adj Close" in df.columns: - updates["adj"][sym] = df["Adj Close"] + updates["adj"][sym] = _apply_corrections( + sym, "history", df["Adj Close"], "Adj Close") if "Close" in df.columns: - updates["close"][sym] = df["Close"] + updates["close"][sym] = _apply_corrections( + sym, "history", df["Close"], "Close") elif sfx == "dividend" and "Dividends" in df.columns: updates["div"][sym] = _apply_corrections(sym, sfx, df["Dividends"]) \ .fillna(0.0) diff --git a/overrides/corrections/JLPSX.json b/overrides/corrections/JLPSX.json index 7a183ce..3f5c26d 100644 --- a/overrides/corrections/JLPSX.json +++ b/overrides/corrections/JLPSX.json @@ -1,7 +1,7 @@ { "symbol": "jlpsx", "name": "JPMorgan U.S. Large Cap Core Plus Fund (I shares)", - "note": "Yahoo listed the year-end capital-gain distribution in BOTH the dividend and capitalGains event files. Expressed as layout-agnostic invariants because a 2026-08 re-download shows Yahoo no longer returns capitalGain events for this fund at all (the dividend stream still carries the year-end rows): 'dedup' keeps at most one copy of each (date, amount) (the capitalGain copy when both files carry it; the lone dividend copy otherwise); 'drop_capg_copy' drops the spurious capitalGain row on a date where the dividend row is the true distribution (any amount). Official per-share data (JPMorgan 497, filed 2025-11-03, Class I, FYE June 30): FY2025 1.37, FY2024 2.03, FY2023 4.86, FY2022 6.78, FY2021 6.89 - local sums after dedup match every one (1.368, 2.029, 4.859, 6.785, 6.889). 2006-12-15 (div 0.207 vs capg 0.141) and 2019-08-08 (div 4.138 vs capg 4.077) keep the dividend amount per the pattern confirmed on all other verified same-date pairs; pre-2012 years have no official totals (the 2021 497 is delisted from SEC Archives) - mechanism-inferred.", + "note": "Yahoo listed the year-end capital-gain distribution in BOTH the dividend and capitalGains event files. Expressed as layout-agnostic invariants because a 2026-08 re-download shows Yahoo no longer returns capitalGain events for this fund at all (the dividend stream still carries the year-end rows): 'dedup' keeps at most one copy of each (date, amount) (the capitalGain copy when both files carry it; the lone dividend copy otherwise); 'drop_capg_copy' drops the spurious capitalGain row on a date where the dividend row is the true distribution (any amount). Official per-share data (JPMorgan 497, filed 2025-11-03, Class I, FYE June 30): FY2025 1.37, FY2024 2.03, FY2023 4.86, FY2022 6.78, FY2021 6.89 - local sums after dedup match every one (1.368, 2.029, 4.859, 6.785, 6.889). 2006-12-15 (div 0.207 vs capg 0.141) and 2019-08-08 (div 4.138 vs capg 4.077) keep the dividend amount per the pattern confirmed on all other verified same-date pairs; pre-2012 years have no official totals (the 2021 497 is delisted from SEC Archives) - mechanism-inferred. 'history' patches the 2020-12-11 Adj Close cell: Yahoo dated the 6.824 distribution on the 12-11 record date but the market went ex-div on 2020-12-14 (close 30.10 -> 23.22, i.e. -6.88), so the raw Adj Close column spiked +29.3% on 12-11 and snapped back -22.9% on 12-14; setting 12-11 to the 12-10 value (11.247889) removes the bogus 3-day wiggle (cumulative return unchanged). adj-close repair: Yahoo dated a distribution on 2020-12-11 (record date) but the price went ex-div on 2020-12-14 (close 30.10 -> 23.22, drop ~6.824); the raw Adj Close column spiked +29.3% on 2020-12-11 and snapped back later. Cells in [2020-12-11, 2020-12-14) rescaled by (1-f) so the adjusted path is smooth; cumulative returns are unchanged. Mechanism: scripts/fix_adj_misalign.py, scan: reports/adj_misalign/scan.md.", "verified": [ "FY2021-FY2025 exact match against 497 filed 2025-11-03 (accession 0001193125-25-261054, CIK 0000763852)" ], @@ -66,5 +66,10 @@ "drop_capg_copy": [ "2006-12-15", "2019-08-08" - ] + ], + "history": { + "2020-12-11": { + "Adj Close": 11.247889 + } + } } diff --git a/overrides/corrections/JLPYX.json b/overrides/corrections/JLPYX.json index cc1a87d..24b875c 100644 --- a/overrides/corrections/JLPYX.json +++ b/overrides/corrections/JLPYX.json @@ -38,5 +38,10 @@ 0.656 ] ], - "note": "Bulk correction from the double-listing scan (scripts/scan_double_listing.py): Yahoo listed this distribution in both the dividend and capitalGains event files (same date, same amount); one copy is spurious. The 'dedup' invariant keeps at most one copy, layout-agnostically. Mechanism verified against official filings for JLPSX/GDEUX/GSOUX/FAEVX/CVSIX/FZAGX; mechanism-inferred for this symbol (no per-fund official check). Revert the entry if a future official cross-check shows both rows were real." + "note": "Bulk correction from the double-listing scan (scripts/scan_double_listing.py): Yahoo listed this distribution in both the dividend and capitalGains event files (same date, same amount); one copy is spurious. The 'dedup' invariant keeps at most one copy, layout-agnostically. Mechanism verified against official filings for JLPSX/GDEUX/GSOUX/FAEVX/CVSIX/FZAGX; mechanism-inferred for this symbol (no per-fund official check). Revert the entry if a future official cross-check shows both rows were real. adj-close repair: Yahoo dated a distribution on 2020-12-11 (record date) but the price went ex-div on 2020-12-14 (close 30.29 -> 23.40, drop ~6.824); the raw Adj Close column spiked +29.1% on 2020-12-11 and snapped back later. Cells in [2020-12-11, 2020-12-14) rescaled by (1-f) so the adjusted path is smooth; cumulative returns are unchanged. Mechanism: scripts/fix_adj_misalign.py, scan: reports/adj_misalign/scan.md.", + "history": { + "2020-12-11": { + "Adj Close": 11.400895 + } + } } diff --git a/reports/adj_misalign/scan.md b/reports/adj_misalign/scan.md new file mode 100644 index 0000000..4154335 --- /dev/null +++ b/reports/adj_misalign/scan.md @@ -0,0 +1,487 @@ +# Adj-close / event-date misalignment scan + +params: min_adj=0.05 max_close=0.02 horizon=3; 481 hits on 278 symbols + +| symbol | date | adj ret | close ret | later drop | +|---|---|---|---|---| +| wbqnl | 2023-06-13 | +436.00% | +0.00% | +83.21% | +| frccx | 2024-04-17 | +308.92% | +0.00% | +75.53% | +| almrx | 2021-12-15 | +82.25% | +0.00% | +44.76% | +| ivinx | 2022-12-15 | +73.70% | +0.00% | +45.47% | +| itgrx | 2022-12-15 | +70.65% | +0.00% | +44.46% | +| amgox | 2021-12-15 | +58.02% | +0.00% | +36.49% | +| aasox | 2021-12-15 | +52.48% | +0.00% | +33.80% | +| krft | 27-Mar-15 | +44.00% | -1.67% | +32.33% | +| amczx | 2021-12-15 | +42.22% | +0.00% | +29.75% | +| aasox | 2015-12-16 | +41.49% | +0.00% | +29.07% | +| mxjvx | 2018-12-27 | +37.16% | +0.00% | +26.59% | +| rmbhx | 2016-12-15 | +36.53% | +0.00% | +26.43% | +| specx | 2021-12-15 | +35.44% | +0.00% | +27.52% | +| mveix | 2020-12-29 | +34.33% | -1.26% | +26.12% | +| seekx | 2021-12-17 | +32.69% | +0.00% | +24.43% | +| irsex | 2022-12-15 | +31.68% | +0.00% | +28.42% | +| mvtrx | 2021-12-21 | +31.40% | +1.62% | +19.23% | +| irsyx | 2022-12-15 | +31.04% | +0.00% | +28.03% | +| mgqsx | 2024-12-16 | +30.30% | +0.14% | +25.18% | +| bgatx | 2018-12-27 | +30.07% | +0.00% | +22.33% | +| jlpsx | 2020-12-11 | +29.32% | +0.00% | +21.43% | +| jlpyx | 2020-12-11 | +29.08% | +0.00% | +21.29% | +| mgfzx | 2022-12-13 | +28.38% | +1.79% | +22.58% | +| ahszx | 2021-12-15 | +28.30% | +0.00% | +20.38% | +| vscax | 2018-12-14 | +26.66% | +0.00% | +25.20% | +| line | 2016-04-07 | -26.30% | +0.00% | +15.62% | +| jevrx | 2024-12-20 | +26.19% | +0.00% | +20.40% | +| iysyx | 2022-12-15 | +26.05% | +0.00% | +24.14% | +| mxmsx | 2018-12-27 | +25.90% | +0.00% | +18.42% | +| scjax | 2021-12-17 | +25.48% | +0.00% | +19.81% | +| ibarx | 2022-12-15 | +25.08% | +0.00% | +22.77% | +| ibnax | 2022-12-15 | +25.03% | +0.00% | +22.73% | +| mgrex | 2022-12-16 | +25.00% | -1.83% | +25.46% | +| alsrx | 2021-12-15 | +24.78% | +0.00% | +19.44% | +| ispvx | 2022-12-15 | +24.31% | +0.00% | +23.10% | +| spgsx | 2024-12-27 | +24.02% | +0.00% | +22.07% | +| ienax | 2007-12-14 | +23.72% | +0.00% | +20.90% | +| mxykx | 2024-12-27 | +23.34% | +0.00% | +20.66% | +| mxgsx | 2018-12-27 | +23.26% | +0.00% | +17.13% | +| timvx | 2019-12-06 | +23.16% | +1.00% | +17.07% | +| line | 2011-08-11 | +23.11% | -0.40% | +15.72% | +| mxkwx | 2018-12-27 | +23.06% | +0.00% | +17.19% | +| acazx | 2021-12-15 | +22.93% | +0.00% | +20.14% | +| mxykx | 2018-12-27 | +22.01% | +0.00% | +16.05% | +| irsyx | 2025-12-12 | +21.60% | +0.00% | +17.84% | +| line | 2015-09-01 | +21.45% | -1.72% | +16.91% | +| wsgrx | 2025-12-12 | +21.39% | +0.00% | +22.21% | +| wmicx | 2005-12-28 | +21.16% | +0.00% | +15.69% | +| mxvhx | 2024-12-27 | +21.01% | +0.13% | +18.47% | +| mpivx | 2021-12-15 | +20.69% | +1.11% | +16.64% | +| cacsx | 2018-12-14 | +20.33% | +0.00% | +20.74% | +| alarx | 2021-12-15 | +19.99% | +0.00% | +18.21% | +| line | 2015-10-08 | +19.98% | +1.49% | +10.71% | +| te | 2002-10-21 | +19.83% | +0.46% | +14.20% | +| vallx | 1991-12-17 | +19.70% | -0.70% | +18.77% | +| mxkwx | 2023-12-27 | +19.47% | +0.00% | +16.33% | +| line | 2013-07-18 | +19.14% | +0.57% | +9.97% | +| chtrx | 2019-12-13 | +18.74% | +0.00% | +15.27% | +| mxerx | 2018-12-27 | +18.59% | +0.00% | +14.25% | +| greux | 2017-12-11 | +18.51% | +0.00% | +15.14% | +| chftx | 2019-12-13 | +18.13% | +0.00% | +14.82% | +| mxhtx | 2018-12-27 | +18.09% | +0.00% | +14.79% | +| prnhx | 1986-12-15 | +17.68% | +0.00% | +16.04% | +| ace | 1998-12-24 | +17.62% | +1.74% | +14.70% | +| smeax | 2018-12-14 | +17.59% | +0.00% | +19.60% | +| ace | 2000-07-28 | +17.27% | -1.22% | +13.22% | +| oalgx | 2025-12-10 | +16.73% | +0.00% | +16.03% | +| nwcfx | 2016-12-15 | +16.70% | +0.00% | +13.89% | +| mxzhx | 2019-09-10 | +16.69% | +0.00% | +11.10% | +| wasax | 2022-12-15 | +16.29% | +0.00% | +16.67% | +| mxvhx | 2018-12-27 | +16.13% | +0.00% | +12.53% | +| gegax | 2006-12-27 | +16.12% | +0.00% | +11.26% | +| iastx | 2022-12-15 | +16.02% | +0.00% | +16.47% | +| vscvx | 2024-12-13 | +15.83% | +0.00% | +20.39% | +| tiscx | 2024-12-06 | +15.80% | +0.00% | +14.57% | +| sscgx | 2018-12-13 | +15.74% | +0.00% | +18.93% | +| line | 2015-01-26 | +15.71% | -1.27% | +9.25% | +| wmcvx | 2005-12-28 | +15.69% | +0.00% | +12.39% | +| wamcx | 2005-12-28 | +15.67% | +0.00% | +13.05% | +| swobx | 2000-12-28 | +15.63% | +0.93% | +13.00% | +| smefx | 2018-12-14 | +15.56% | +0.00% | +18.10% | +| algzx | 2017-12-15 | +15.47% | +0.00% | +12.60% | +| line | 2011-08-12 | +15.44% | -1.87% | +7.78% | +| acgix | 1999-12-13 | +15.43% | +0.00% | +14.34% | +| muxyx | 2024-12-13 | +15.37% | +0.00% | +16.04% | +| parwx | 2021-11-18 | +15.09% | +0.00% | +12.89% | +| sllax | 2018-12-13 | +14.91% | +0.00% | +18.12% | +| mrimx | 2024-12-13 | +14.77% | +0.00% | +18.78% | +| mxkjx | 2024-12-27 | +14.75% | +0.10% | +14.30% | +| mvssx | 2024-12-13 | +14.63% | +0.00% | +19.78% | +| jmvsx | 2021-12-13 | +14.62% | +0.00% | +12.92% | +| te | 2001-03-30 | +14.59% | -0.03% | +8.27% | +| vvoax | 2018-12-14 | +14.57% | +0.00% | +17.06% | +| mxwex | 2019-09-10 | +14.53% | +0.00% | +9.58% | +| wceax | 2022-12-15 | +14.53% | +0.00% | +16.62% | +| line | 2015-11-04 | -14.49% | +1.43% | +8.96% | +| trdfx | 2021-12-17 | +14.44% | +0.00% | +10.94% | +| vvosx | 2018-12-14 | +14.44% | +0.00% | +17.00% | +| ace | 1996-02-09 | -14.42% | -1.03% | +8.21% | +| algrx | 2021-12-15 | +14.41% | +0.00% | +14.30% | +| mxkjx | 2017-12-27 | +14.29% | +0.00% | +11.92% | +| ipoyx | 2022-12-15 | +14.22% | +0.00% | +14.76% | +| omsix | 2015-12-03 | +14.18% | +0.00% | +13.29% | +| pom | 1998-09-30 | +14.17% | -1.85% | +7.18% | +| imegx | 2022-12-15 | +14.13% | +0.00% | +14.64% | +| mxykx | 2023-12-27 | +14.11% | +0.00% | +12.48% | +| alzfx | 2021-12-15 | +14.05% | +0.00% | +14.03% | +| slgax | 2018-12-13 | +13.98% | +0.00% | +16.02% | +| te | 2009-05-05 | -13.86% | -1.88% | +9.41% | +| jgemx | 2024-12-20 | +13.83% | +0.00% | +10.73% | +| hlgzx | 2017-12-15 | +13.78% | -0.36% | +11.93% | +| mgfzx | 2021-12-15 | +13.72% | +1.62% | +12.70% | +| selcx | 2018-12-13 | +13.72% | +0.00% | +15.98% | +| mgosx | 2024-12-13 | +13.70% | +0.00% | +16.37% | +| rpxfx | 2019-12-16 | +13.70% | +1.17% | +8.94% | +| pom | 2009-09-17 | -13.66% | -0.13% | +7.18% | +| pvfix | 2024-12-13 | +13.63% | +0.00% | +14.65% | +| jvmrx | 2024-12-20 | +13.59% | +1.14% | +8.89% | +| waiox | 2011-12-28 | +13.54% | +0.00% | +10.76% | +| bbhlx | 2021-12-15 | +13.39% | +0.00% | +12.97% | +| line | 2012-06-28 | +13.38% | -1.05% | +7.56% | +| ace | 2001-04-30 | -13.37% | -1.30% | +6.83% | +| prnex | 1986-12-15 | +13.33% | +0.00% | +12.11% | +| ace | 2007-08-20 | +13.32% | -0.82% | +7.83% | +| swobx | 1999-12-30 | +13.26% | +0.99% | +9.60% | +| mxnzx | 2018-12-27 | +13.23% | +0.00% | +10.28% | +| rpfix | 2016-12-15 | +13.22% | +0.00% | +10.80% | +| waaex | 2005-12-28 | +13.09% | +0.00% | +10.72% | +| ace | 2003-04-22 | -13.03% | -1.82% | +11.22% | +| line | 2008-02-27 | -12.99% | -1.66% | +7.12% | +| mxtbx | 2019-09-10 | +12.87% | +0.00% | +8.77% | +| mxigx | 2018-12-27 | +12.85% | +0.00% | +10.78% | +| line | 2009-05-29 | +12.67% | -1.15% | +6.46% | +| oglix | 2018-12-06 | +12.67% | +0.00% | +14.21% | +| te | 2002-07-30 | +12.64% | -0.13% | +8.83% | +| dvzrx | 2025-12-15 | +12.57% | +0.00% | +11.94% | +| ace | 1999-10-25 | -12.56% | -0.36% | +7.83% | +| dbef | 2012-12-19 | +12.54% | +0.00% | +8.86% | +| amrgx | 2021-12-09 | +12.52% | +0.00% | +11.73% | +| te | 2009-07-15 | +12.52% | -1.24% | +6.78% | +| jlcwx | 2024-12-20 | +12.50% | +0.00% | +8.77% | +| rlsfx | 2019-12-16 | +12.34% | +0.95% | +8.27% | +| pom | 2011-08-11 | +12.24% | +0.00% | +6.50% | +| semcx | 2018-12-13 | +12.14% | +0.00% | +14.61% | +| iceqx | 2022-12-15 | +12.12% | +0.00% | +14.80% | +| wagox | 2011-12-28 | +12.12% | +0.00% | +10.15% | +| line | 2008-04-07 | -12.11% | -0.04% | +9.53% | +| jdeux | 2021-12-13 | +12.08% | +0.00% | +11.71% | +| te | 2002-10-28 | -12.08% | +0.34% | +7.75% | +| tirex | 2004-12-29 | +12.07% | +0.00% | +11.03% | +| kauax | 2019-12-05 | +12.04% | +0.00% | +10.67% | +| jdiux | 2024-12-20 | +12.01% | +0.00% | +9.97% | +| lprex | 2018-12-31 | +11.92% | +0.00% | +9.00% | +| te | 2003-09-25 | +11.89% | -1.95% | +7.72% | +| ace | 2000-07-12 | +11.87% | +0.40% | +6.43% | +| mxtfx | 2023-12-27 | +11.86% | +0.00% | +11.75% | +| lpsgx | 2018-12-31 | +11.80% | +0.00% | +8.82% | +| te | 2008-03-31 | +11.80% | -1.97% | +5.90% | +| auuyx | 2018-12-12 | +11.74% | +0.00% | +13.51% | +| msoox | 2021-12-15 | +11.73% | +1.29% | +10.85% | +| lprfx | 2018-12-31 | +11.72% | +0.00% | +8.78% | +| allfx | 2019-12-13 | +11.61% | +0.00% | +9.33% | +| sequx | 1986-02-07 | +11.58% | +0.25% | +9.07% | +| frccx | 2022-12-15 | +11.52% | +0.00% | +12.88% | +| jppex | 2020-12-11 | +11.44% | +0.00% | +9.37% | +| mpivx | 2022-12-13 | +11.44% | +0.71% | +12.09% | +| opmix | 2018-12-07 | +11.41% | +0.00% | +13.47% | +| tisex | 2024-12-06 | +11.41% | +0.00% | +10.78% | +| smisx | 2018-12-14 | +11.38% | +0.00% | +15.81% | +| mgisx | 2024-12-16 | +11.37% | -0.23% | +12.57% | +| pdinx | 2024-12-18 | +11.34% | +0.00% | +10.89% | +| svoax | 2018-12-13 | +11.34% | +0.00% | +13.50% | +| smeax | 2019-12-13 | +11.33% | +0.00% | +9.40% | +| igfrx | 2019-12-13 | +11.26% | +0.00% | +8.72% | +| auuyx | 2017-12-13 | +11.22% | +0.00% | +9.23% | +| mgosx | 2014-12-29 | +11.21% | +0.00% | +11.11% | +| line | 2013-10-21 | -11.16% | -1.81% | +8.97% | +| abrcx | 2019-12-13 | +11.07% | +0.00% | +8.92% | +| chtrx | 2018-12-14 | +11.03% | +0.00% | +15.05% | +| aiiex | 2019-12-13 | +11.02% | +0.00% | +8.54% | +| ace | 1997-02-21 | -11.01% | -0.96% | +6.51% | +| ace | 1995-09-01 | -10.99% | -1.59% | +5.58% | +| chftx | 2018-12-14 | +10.98% | +0.00% | +14.97% | +| lpsfx | 2018-12-31 | +10.84% | +0.00% | +8.04% | +| jsgtx | 2024-12-20 | +10.81% | +1.16% | +5.50% | +| ace | 2010-06-14 | +10.79% | -1.78% | +5.52% | +| ace | 2009-07-15 | -10.73% | -1.78% | +7.89% | +| ace | 2001-02-26 | +10.72% | -1.57% | +5.96% | +| mefzx | 2013-12-16 | +10.63% | +0.00% | +8.13% | +| sesvx | 2018-12-13 | +10.57% | +0.00% | +14.10% | +| evglx | 2017-12-29 | +10.52% | +0.00% | +8.05% | +| tigrx | 2024-12-06 | +10.51% | +0.00% | +9.57% | +| jdeux | 2019-10-08 | +10.47% | +0.00% | +8.59% | +| wamvx | 2005-12-28 | +10.41% | +0.00% | +7.53% | +| line | 2006-06-23 | +10.40% | -1.57% | +6.03% | +| lprdx | 2018-12-31 | +10.31% | +0.00% | +7.86% | +| glvix | 2019-12-13 | +10.29% | +0.38% | +7.75% | +| pom | 2010-07-07 | -10.26% | -1.56% | +6.30% | +| otcfx | 1993-12-21 | +10.03% | -0.42% | +9.80% | +| igfrx | 2018-12-14 | +10.02% | +0.00% | +12.36% | +| lpsdx | 2018-12-31 | +10.01% | +0.00% | +7.63% | +| cnsfx | 2018-12-14 | +9.96% | +0.00% | +11.66% | +| cnsax | 2018-12-14 | +9.87% | +0.00% | +11.59% | +| evagx | 2017-12-29 | +9.86% | +0.00% | +7.33% | +| esgs | 2017-12-26 | +9.85% | +0.00% | +8.56% | +| ityax | 2019-12-13 | +9.84% | +0.00% | +7.62% | +| scurx | 2018-12-17 | +9.83% | +0.00% | +13.49% | +| smefx | 2019-12-13 | +9.83% | +0.00% | +8.14% | +| mstqx | 2024-12-20 | +9.82% | +0.00% | +6.82% | +| ncbgx | 2015-12-11 | +9.81% | +0.00% | +8.79% | +| line | 2008-05-07 | -9.74% | -0.45% | +4.98% | +| mxsfx | 2024-12-27 | +9.69% | +0.22% | +9.90% | +| pabax | 2024-12-18 | +9.69% | +0.06% | +10.23% | +| ace | 2005-04-29 | +9.67% | +0.02% | +7.75% | +| te | 2002-10-23 | +9.58% | -1.32% | +12.84% | +| aiiex | 2018-12-14 | +9.56% | +0.00% | +12.02% | +| ace | 2000-07-10 | +9.53% | -1.87% | +8.32% | +| ace | 2008-12-16 | +9.48% | +0.90% | +7.18% | +| hcp | 1987-01-27 | +9.43% | -0.43% | +5.56% | +| ufs | 1986-05-09 | +9.36% | -1.10% | +4.97% | +| dnldx | 1991-12-31 | +9.31% | +0.00% | +5.77% | +| line | 2011-10-10 | +9.24% | -0.44% | +5.10% | +| gtsax | 2018-12-14 | +9.23% | +0.00% | +13.87% | +| mgrpx | 2020-12-11 | +9.21% | +0.10% | +7.05% | +| irbax | 2022-12-15 | +9.14% | +0.00% | +9.44% | +| acstx | 1999-12-13 | +9.13% | +0.00% | +9.45% | +| te | 2009-08-21 | -9.13% | -0.66% | +5.12% | +| gtsax | 2019-12-13 | +9.11% | +0.00% | +7.62% | +| fgskx | 2020-12-04 | +9.10% | +0.99% | +7.01% | +| pequx | 1990-11-30 | +9.08% | +0.42% | +6.23% | +| mirsx | 2024-12-13 | +9.04% | +0.00% | +14.88% | +| qalgx | 2020-12-04 | +9.04% | +0.69% | +7.68% | +| te | 2007-08-20 | +9.01% | -0.76% | +4.72% | +| line | 2012-06-29 | +9.00% | -1.04% | +6.52% | +| ace | 1993-08-10 | +8.99% | -0.39% | +9.65% | +| omsix | 2019-12-13 | +8.92% | +0.06% | +7.60% | +| bflax | 1998-12-10 | +8.87% | -0.78% | +10.11% | +| asmmx | 2019-12-13 | +8.86% | +0.00% | +6.98% | +| te | 2014-10-21 | +8.78% | -0.73% | +4.56% | +| inutx | 1999-12-22 | +8.76% | +0.00% | +6.45% | +| pwlix | 2024-12-26 | +8.75% | +0.00% | +7.64% | +| mfllx | 2020-12-11 | +8.71% | -1.11% | +7.45% | +| polyx | 2024-12-18 | +8.66% | +0.00% | +8.99% | +| smisx | 2019-12-13 | +8.66% | +0.00% | +6.79% | +| ryaix | 2008-12-04 | +8.63% | +1.54% | +6.28% | +| line | 2006-10-12 | +8.61% | -1.51% | +5.09% | +| jrsnx | 2018-12-20 | +8.57% | +0.00% | +13.52% | +| ace | 2009-09-28 | +8.56% | -0.82% | +6.68% | +| fgsax | 2020-12-04 | +8.53% | +0.98% | +6.53% | +| gmrix | 2006-12-27 | +8.52% | +0.00% | +7.80% | +| lpjkx | 2018-12-31 | +8.49% | +0.00% | +6.24% | +| pgsix | 2024-12-18 | +8.45% | +0.00% | +8.95% | +| lpjrx | 2018-12-31 | +8.43% | +0.00% | +6.19% | +| te | 2004-07-29 | -8.43% | +0.00% | +5.58% | +| lvitx | 2024-12-06 | +8.42% | +0.00% | +7.97% | +| ityax | 2018-12-14 | +8.40% | +0.00% | +13.80% | +| mxerx | 2024-12-27 | +8.38% | +0.57% | +8.69% | +| ace | 1995-01-24 | +8.37% | -1.59% | +6.35% | +| bdokx | 2014-12-31 | +8.36% | +0.00% | +11.26% | +| ace | 2003-03-17 | +8.27% | -0.58% | +14.23% | +| pom | 1998-04-30 | -8.27% | +0.51% | +4.87% | +| mxjvx | 2024-12-27 | +8.26% | +0.23% | +7.96% | +| ace | 2003-03-19 | -8.25% | -0.25% | +6.57% | +| tilvx | 2019-12-06 | +8.25% | +0.90% | +6.07% | +| gghcx | 2018-12-14 | +8.24% | +0.00% | +13.85% | +| pcvax | 2005-12-13 | +8.22% | +0.00% | +9.44% | +| te | 2008-01-31 | -8.21% | -1.48% | +4.79% | +| muxyx | 2014-12-29 | +8.20% | +0.00% | +8.92% | +| te | 2002-09-27 | -8.20% | -0.38% | +9.07% | +| ifutx | 2018-12-14 | +8.17% | +0.00% | +11.51% | +| te | 2003-01-06 | +8.17% | +0.95% | +7.81% | +| awsax | 2018-12-14 | +8.16% | +0.00% | +11.55% | +| iautx | 2018-12-14 | +8.15% | +0.00% | +11.48% | +| ftpsx | 2019-12-13 | +8.11% | +0.00% | +6.12% | +| gghsx | 2018-12-14 | +8.05% | +0.00% | +13.71% | +| pwtax | 2024-12-17 | +8.05% | +0.00% | +9.65% | +| dlqax | 1996-12-30 | +8.04% | +0.33% | +7.76% | +| iarax | 2019-12-13 | +8.03% | +0.00% | +6.54% | +| mxmsx | 2024-12-27 | +8.03% | +0.67% | +8.05% | +| lprax | 2023-12-22 | +7.97% | +0.00% | +6.70% | +| te | 2009-01-02 | +7.97% | -0.16% | +8.80% | +| te | 1986-04-17 | -7.97% | -0.87% | +4.07% | +| pom | 2011-04-27 | +7.96% | -0.88% | +4.06% | +| lphkx | 2018-12-31 | +7.94% | +0.00% | +5.66% | +| rseix | 2016-12-15 | +7.93% | +0.00% | +5.47% | +| lphax | 2018-12-31 | +7.89% | +0.00% | +5.58% | +| kauax | 2020-12-04 | +7.86% | +0.67% | +6.67% | +| line | 2007-03-20 | -7.78% | -1.07% | +4.62% | +| ryshx | 2023-12-12 | +7.78% | +0.00% | +11.87% | +| msoox | 2020-12-11 | +7.77% | -0.40% | +6.51% | +| ssgfx | 1997-11-28 | +7.77% | +0.00% | +4.39% | +| giffx | 2019-12-13 | +7.65% | +0.00% | +6.45% | +| tln | 2016-06-07 | +7.65% | +0.58% | +13.04% | +| ace | 1995-03-29 | +7.60% | -0.49% | +5.83% | +| evttx | 2017-12-29 | +7.58% | +0.00% | +5.29% | +| bbhlx | 2017-12-15 | +7.57% | +0.00% | +6.64% | +| line | 2006-04-17 | +7.57% | +1.46% | +3.84% | +| mphzx | 2012-12-19 | +7.57% | +0.00% | +7.01% | +| te | 2005-03-01 | -7.57% | -0.74% | +4.19% | +| esmsx | 2019-12-13 | +7.56% | +0.00% | +5.74% | +| acgix | 2019-12-13 | +7.55% | +0.00% | +6.35% | +| asisx | 2018-12-14 | +7.55% | +0.00% | +9.38% | +| tnvdx | 2017-12-14 | +7.54% | +0.00% | +6.79% | +| mipzx | 2021-12-15 | +7.52% | +0.00% | +6.66% | +| pom | 1998-10-02 | -7.52% | -0.23% | +3.88% | +| evmlx | 2017-12-29 | +7.51% | +0.00% | +5.64% | +| pom | 1987-08-11 | +7.51% | -1.08% | +4.30% | +| rymhx | 2023-12-12 | +7.49% | +0.00% | +10.37% | +| lprax | 2018-12-31 | +7.47% | +0.00% | +5.86% | +| mxgsx | 2024-12-27 | +7.47% | -0.15% | +10.02% | +| lpbkx | 2018-12-31 | +7.46% | +0.00% | +5.53% | +| lpvkx | 2018-12-31 | +7.46% | +0.00% | +5.20% | +| ace | 2003-10-01 | +7.45% | -1.25% | +5.23% | +| ace | 1998-07-07 | +7.44% | +0.30% | +4.27% | +| lpvax | 2018-12-31 | +7.44% | +0.00% | +5.15% | +| asisx | 2019-12-13 | +7.38% | +0.00% | +4.80% | +| mdbzx | 2012-12-19 | +7.38% | +0.00% | +6.61% | +| ace | 2011-12-21 | +7.37% | -1.00% | +4.60% | +| tedmx | 1997-12-15 | +7.33% | +0.00% | +3.90% | +| tnxax | 2017-12-14 | +7.29% | +0.00% | +6.55% | +| ssbox | 2024-12-27 | +7.28% | +0.00% | +7.30% | +| ace | 2009-09-29 | -7.26% | -0.09% | +6.70% | +| pxtix | 2015-12-16 | +7.21% | -1.17% | +3.67% | +| esmax | 2019-12-13 | +7.19% | +0.00% | +5.44% | +| fiusx | 1997-11-28 | +7.19% | +0.33% | +5.05% | +| mxzhx | 2018-09-11 | +7.19% | +0.00% | +6.00% | +| specx | 2000-12-15 | +7.15% | +0.00% | +15.03% | +| mxnzx | 2024-12-27 | +7.10% | +0.34% | +7.66% | +| ace | 1994-04-07 | +7.09% | -0.89% | +10.18% | +| line | 2007-03-22 | +7.07% | -0.29% | +4.57% | +| ftpsx | 2018-12-14 | +7.06% | +0.00% | +12.68% | +| vafax | 2019-12-13 | +7.05% | +0.00% | +5.16% | +| ace | 2001-05-29 | +7.03% | -1.06% | +3.64% | +| dfgex | 2010-12-09 | +7.03% | +0.00% | +6.47% | +| tnyrx | 2017-12-14 | +7.02% | +0.00% | +6.39% | +| asiax | 2018-12-14 | +7.01% | +0.00% | +8.92% | +| pom | 2002-07-29 | -7.00% | -1.69% | +13.40% | +| mxtfx | 2024-12-27 | +6.94% | +0.55% | +6.90% | +| ace | 2010-06-11 | -6.92% | -0.25% | +3.93% | +| asiax | 2019-12-13 | +6.90% | +0.00% | +4.38% | +| line | 2007-03-21 | -6.88% | -0.80% | +5.44% | +| vadfx | 2018-12-14 | +6.86% | +0.00% | +11.40% | +| te | 2001-10-02 | +6.85% | -0.39% | +9.20% | +| te | 1994-04-18 | -6.80% | -0.64% | +3.82% | +| scijx | 2018-12-17 | +6.79% | +0.00% | +8.88% | +| pom | 2000-03-01 | -6.78% | -0.93% | +4.33% | +| rofix | 2016-12-15 | +6.77% | +0.00% | +4.15% | +| vaffx | 2019-12-13 | +6.77% | +0.00% | +4.92% | +| jacdx | 2024-12-20 | +6.74% | +0.00% | +4.89% | +| te | 2004-02-09 | +6.73% | +0.20% | +5.35% | +| pom | 2000-03-14 | -6.72% | -0.93% | +4.66% | +| dfitx | 2016-12-14 | +6.71% | -1.76% | +10.59% | +| te | 2009-12-11 | -6.71% | -0.81% | +3.37% | +| ssbsx | 2024-12-27 | +6.70% | +0.00% | +7.08% | +| ace | 2005-09-14 | +6.69% | -1.59% | +3.91% | +| lpsax | 2023-12-22 | +6.68% | +0.00% | +5.58% | +| rvt | 2009-03-04 | +6.68% | +0.70% | +13.49% | +| lpsax | 2018-12-31 | +6.67% | +0.00% | +5.08% | +| ace | 1995-09-13 | -6.65% | +0.36% | +3.65% | +| mxugx | 2024-12-27 | +6.64% | +0.12% | +5.81% | +| timvx | 2024-12-06 | +6.64% | +0.00% | +7.66% | +| line | 2006-07-31 | +6.63% | -0.30% | +3.67% | +| ace | 2012-04-19 | +6.58% | -0.61% | +3.80% | +| esmsx | 2018-12-14 | +6.57% | +0.00% | +8.26% | +| pom | 2009-12-01 | -6.57% | -1.06% | +4.55% | +| tammx | 2021-12-21 | +6.57% | +0.00% | +3.31% | +| ufs | 1987-05-08 | +6.56% | -1.21% | +9.31% | +| line | 2006-10-11 | +6.54% | -1.26% | +4.63% | +| etgix | 2008-03-11 | +6.52% | +1.69% | +5.53% | +| pom | 2014-02-12 | +6.50% | -1.33% | +4.42% | +| dnldx | 1990-12-31 | +6.47% | +0.00% | +6.59% | +| pom | 2015-09-14 | +6.45% | -1.53% | +3.28% | +| auuyx | 2022-12-07 | +6.43% | +0.00% | +5.09% | +| gtddx | 1995-12-20 | +6.42% | +0.00% | +3.68% | +| mxzhx | 2017-12-28 | +6.42% | +0.19% | +4.83% | +| line | 2009-05-26 | +6.40% | -0.95% | +11.39% | +| sequx | 1988-02-05 | +6.37% | +0.28% | +5.47% | +| tisbx | 2024-12-06 | +6.36% | +0.00% | +6.03% | +| ryjsx | 2023-12-12 | +6.34% | +0.00% | +3.26% | +| te | 2005-09-30 | -6.33% | -0.93% | +3.46% | +| icsfx | 2019-12-13 | +6.32% | +0.00% | +5.43% | +| tilgx | 2024-12-06 | +6.32% | +0.00% | +4.16% | +| pom | 2009-07-13 | -6.30% | +0.15% | +3.62% | +| te | 2013-01-30 | +6.30% | -0.73% | +3.26% | +| sivix | 2019-12-20 | +6.29% | +0.00% | +5.37% | +| ace | 2013-07-08 | -6.28% | -0.99% | +3.68% | +| mpw | 2009-03-17 | +6.27% | +0.99% | +11.91% | +| te | 2001-04-10 | +6.26% | +0.26% | +4.20% | +| ace | 2013-05-06 | +6.25% | -0.78% | +3.38% | +| ace | 2011-08-30 | -6.24% | -0.65% | +4.94% | +| drcvx | 1988-12-13 | +6.24% | +0.00% | +7.12% | +| mxtbx | 2017-12-28 | +6.22% | +0.19% | +4.80% | +| mxzhx | 2023-09-08 | +6.22% | +0.00% | +5.89% | +| ssbyx | 2024-12-27 | +6.21% | +0.00% | +6.91% | +| acstx | 2019-12-13 | +6.20% | +0.00% | +5.35% | +| line | 2012-02-29 | +6.19% | -1.45% | +3.15% | +| ace | 2011-04-04 | +6.17% | +0.80% | +3.40% | +| te | 2008-05-06 | +6.16% | +1.36% | +3.48% | +| esmax | 2018-12-14 | +6.15% | +0.00% | +7.97% | +| mczzx | 2020-12-11 | +6.14% | +0.00% | +5.64% | +| te | 2007-10-25 | +6.12% | -1.67% | +4.48% | +| jgrsx | 2024-12-20 | +6.11% | +0.00% | +3.52% | +| te | 2005-06-09 | +6.10% | -1.37% | +3.22% | +| pom | 2005-05-31 | -6.09% | -1.10% | +3.07% | +| te | 2009-06-29 | -6.08% | +0.92% | +3.27% | +| mxwex | 2017-12-28 | +6.07% | +0.28% | +4.40% | +| ace | 2004-10-28 | +6.06% | -0.21% | +8.59% | +| vadcx | 2018-12-14 | +6.01% | +0.00% | +10.69% | +| te | 1993-02-22 | -5.97% | -0.56% | +3.08% | +| pom | 2003-05-16 | +5.96% | -0.60% | +4.01% | +| tln | 2016-02-24 | -5.96% | +1.52% | +8.50% | +| tqccx | 1999-12-14 | +5.96% | +0.00% | +4.00% | +| ace | 2005-02-07 | +5.95% | +0.74% | +3.61% | +| hnmvx | 2024-12-20 | +5.94% | +0.00% | +3.30% | +| ace | 1998-02-05 | +5.91% | +0.19% | +4.33% | +| ace | 2013-07-09 | +5.89% | +0.69% | +3.00% | +| ace | 2010-07-12 | +5.89% | -1.54% | +7.14% | +| lcefx | 2018-12-14 | +5.89% | +0.00% | +9.34% | +| line | 2014-08-08 | -5.89% | -1.14% | +3.40% | +| pom | 2016-03-04 | -5.89% | +0.99% | +5.67% | +| tnvdx | 2024-12-13 | +5.89% | +0.00% | +6.81% | +| wgrox | 1991-12-30 | +5.89% | +1.69% | +3.38% | +| te | 1999-03-24 | -5.88% | +1.18% | +4.72% | +| tnxax | 2024-12-13 | +5.88% | +0.00% | +6.81% | +| line | 2013-04-01 | +5.87% | -0.49% | +4.46% | +| ssfox | 2024-12-27 | +5.85% | +0.00% | +6.03% | +| tnyrx | 2024-12-13 | +5.85% | +0.00% | +6.82% | +| line | 2006-06-26 | -5.83% | -0.49% | +3.90% | +| lpdkx | 2018-12-31 | +5.83% | +0.00% | +3.77% | +| ace | 1993-11-11 | -5.79% | -0.40% | +3.95% | +| pom | 2002-06-18 | +5.78% | +0.73% | +4.70% | +| lceax | 2018-12-14 | +5.77% | +0.00% | +9.30% | +| te | 2014-12-24 | -5.77% | -1.35% | +5.00% | +| secex | 1985-04-29 | +5.76% | -1.83% | +4.40% | +| rvt | 1999-03-04 | +5.75% | -1.96% | +5.39% | +| te | 2009-07-16 | -5.75% | +0.00% | +5.62% | +| prrsx | 2022-09-08 | +5.73% | -1.96% | +3.64% | +| ace | 2004-01-13 | +5.70% | -0.78% | +3.39% | +| ldfvx | 1997-11-12 | +5.67% | -1.35% | +4.73% | +| ivinx | 1990-12-28 | +5.65% | +1.43% | +3.27% | +| wsgrx | 2022-12-15 | +5.64% | +0.00% | +10.65% | +| hctpf | 2016-02-11 | +5.63% | +0.00% | +8.89% | +| mxqbx | 2017-12-28 | +5.59% | +0.19% | +4.46% | +| line | 2012-04-30 | -5.57% | -0.20% | +2.85% | +| ace | 2013-04-08 | +5.56% | -0.54% | +2.78% | +| pom | 2011-05-10 | +5.56% | +0.25% | +4.12% | +| asrax | 2019-12-13 | +5.54% | +0.00% | +4.63% | +| te | 1987-09-30 | +5.54% | -1.56% | +3.12% | +| pom | 2010-07-22 | +5.51% | -1.35% | +3.29% | +| ace | 1996-10-07 | +5.50% | +1.78% | +3.56% | +| vecrx | 2019-07-31 | +5.49% | -1.16% | +5.73% | +| grf | 2024-11-29 | +5.48% | -1.33% | +11.29% | +| ssckx | 2024-12-27 | +5.47% | +0.00% | +6.43% | +| tunix | 2021-12-21 | +5.47% | +0.00% | +4.92% | +| ace | 2005-09-26 | -5.44% | +0.48% | +3.57% | +| amrgx | 2022-12-05 | +5.44% | -1.72% | +8.05% | +| teg | 25-Jun-14 | +5.44% | -1.28% | +13.21% | +| lpsax | 2013-12-31 | +5.43% | +0.00% | +5.36% | +| line | 2010-05-28 | +5.40% | +1.96% | +5.46% | +| mxwex | 2023-09-08 | +5.40% | +0.00% | +5.12% | +| oscix | 2019-12-13 | +5.40% | +0.34% | +4.38% | +| pom | 1991-10-02 | -5.40% | +0.00% | +4.21% | +| pom | 2000-12-26 | +5.37% | -0.73% | +4.06% | +| gapux | 2024-12-06 | +5.32% | +0.00% | +5.04% | +| ieifx | 2019-12-13 | +5.31% | +0.00% | +4.58% | +| pom | 1995-06-14 | +5.27% | -0.57% | +4.60% | +| teg | 10-Oct-14 | +5.25% | -0.89% | +3.21% | +| mac | 2008-11-13 | +5.24% | +1.20% | +35.70% | +| iegfx | 2019-12-13 | +5.23% | +0.00% | +3.30% | +| nie | 2008-12-24 | +5.23% | +0.08% | +7.99% | +| ace | 2004-01-14 | -5.22% | +0.21% | +2.79% | +| line | 2012-04-26 | +5.22% | -0.83% | +2.66% | +| aceix | 2019-12-13 | +5.21% | +0.00% | +4.49% | +| hacsx | 2014-12-18 | +5.20% | +0.00% | +3.57% | +| mxxjx | 2024-12-27 | +5.20% | +0.00% | +4.72% | +| origx | 2008-11-26 | +5.15% | +1.80% | +3.23% | +| ace | 1995-06-14 | -5.14% | -0.44% | +4.00% | +| pom | 2001-05-15 | -5.08% | -0.88% | +2.92% | +| eaasx | 2008-12-23 | +5.07% | +1.81% | +3.25% | +| fiusx | 1996-12-30 | +5.04% | +0.31% | +4.07% | +| line | 2012-10-16 | +5.01% | -0.63% | +2.94% | diff --git a/scripts/check_corrections.py b/scripts/check_corrections.py index a50ec4d..ff48f68 100644 --- a/scripts/check_corrections.py +++ b/scripts/check_corrections.py @@ -51,6 +51,18 @@ def main() -> int: if not capgf.get(date): print(f"BAD {sym} drop_capg_copy {date}: no capitalGain row") bad += 1 + for date, cells in (d.get("history") or {}).items(): + p = FROZEN / f"{sym}-history.csv" + if not p.exists(): + p = DATA / f"{sym}-history.csv" + dates = {l.split(",")[0][:10] for l in p.read_text(errors="replace").splitlines()[1:]} if p.exists() else set() + if date not in dates: + print(f"BAD {sym} history {date}: date not in history file") + bad += 1 + for col in cells: + if col not in ("Open", "High", "Low", "Close", "Adj Close", "Volume"): + print(f"BAD {sym} history {date}: unknown column {col!r}") + bad += 1 for key, rows_f in files.items(): ops = d.get(key) if not ops or rows_f is None: diff --git a/scripts/fix_adj_misalign.py b/scripts/fix_adj_misalign.py new file mode 100644 index 0000000..608cfc7 --- /dev/null +++ b/scripts/fix_adj_misalign.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Repair Yahoo adj-close / event-date misalignment (see +scripts/scan_adj_misalign.py). + +For a hit date D (adj spikes, close flat, price drops later) the repair is +purely arithmetic and leaves cumulative returns unchanged: + + r(t) = adj(t)/close(t); k = r(D)/r(D-1) # the bogus spike + f = 1 - 1/k # dist fraction Yahoo applied + d = f * close(D-1) # implied per-share dist + E = first t > D with close(t) < close(t-1) - 0.5*d # true ex-div date + for t in [D, E): adj(t) *= (1 - f) # restore pre-event ratio + +For t before D and from E on the series is already consistent, so only the +[D, E) window is patched. The patch is written as a 'history' correction +(overrides/corrections/{SYM}.json, merged, with a note), so it survives +re-downloads that rewrite the raw CSVs with the same Yahoo values. + +Usage: + python3 scripts/fix_adj_misalign.py [--only SYM[,SYM...]] [--dry-run] +""" +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from scripts.scan_adj_misalign import scan # noqa: E402 + +ROOT_DEFAULT = Path.home() / "prog/fin/stocks" +CORR = Path(__file__).resolve().parent.parent / "overrides" / "corrections" +NOTE = ("adj-close repair: Yahoo dated a distribution on {d} (record date) " + "but the price went ex-div on {e} (close {c0:.2f} -> {c1:.2f}, drop " + "~{d_amt:.3f}); the raw Adj Close column spiked {spike:+.1%} on {d} " + "and snapped back later. Cells in [{d}, {e}) rescaled by (1-f) so " + "the adjusted path is smooth; cumulative returns are unchanged. " + "Mechanism: scripts/fix_adj_misalign.py, " + "scan: reports/adj_misalign/scan.md.") + + +def load_rows(sym: str, root: Path): + import data as D + fr = D.FROZEN_DIR / f"{sym}-history.csv" + f = fr if (D.FROZEN_DIR.is_dir() and fr.exists()) else root / f"{sym}-history.csv" + rows = [] + for l in f.read_text(errors="replace").splitlines()[1:]: + if not l.strip(): + continue + a = l.split(",") + if len(a) < 6: + continue + try: + rows.append((a[0][:10], float(a[4]), float(a[5]))) + except ValueError: + continue + return rows + + +def repair(sym: str, date: str, rows) -> dict: + """Return {date: {"Adj Close": value}} patch, or {} if not repairable.""" + i = next((k for k, r in enumerate(rows) if r[0] == date), None) + if i is None or i == 0: + return {} + dates, closes, adjs = (list(x) for x in zip(*rows)) + r = [a / c for _, c, a in rows] + if r[i - 1] <= 0 or r[i] <= 0 or r[i] <= r[i - 1]: + return {} + f = 1.0 - r[i - 1] / r[i] + if not (0.005 < f < 0.95): + return {} + d_amt = f * closes[i - 1] + E = None + for t in range(i + 1, min(i + 11, len(rows))): + if closes[t] < closes[t - 1] - 0.5 * d_amt: + E = t + break + if E is None: + return {} + k = 1.0 - f + patch = {} + for t in range(i, E): + patch[dates[t]] = {"Adj Close": round(adjs[t] * k, 6)} + return patch, f, d_amt, E + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--data", type=Path, default=ROOT_DEFAULT) + ap.add_argument("--only", default="", help="comma-separated symbols") + ap.add_argument("--dry-run", action="store_true") + args = ap.parse_args() + only = {s.strip().lower() for s in args.only.split(",") if s.strip()} + + hits = scan(args.data, 0.05, 0.02, 3) + if only: + hits = [h for h in hits if h["symbol"] in only] + by_sym: dict[str, list] = {} + for h in hits: + by_sym.setdefault(h["symbol"], []).append(h) + applied = 0 + for sym, hs in sorted(by_sym.items()): + rows = load_rows(sym, args.data) + cells, notes = {}, [] + for h in hs: + try: + patch, f, d_amt, E = repair(sym, h["date"], rows) + except Exception as ex: + print(f" !! {sym} {h['date']}: {ex}") + continue + if not patch: + print(f" -- {sym} {h['date']}: not repairable (skipped)") + continue + cells.update(patch) + dates = [x[0] for x in rows] + di = next(k for k, r in enumerate(rows) if r[0] == h["date"]) + notes.append(NOTE.format(d=h["date"], e=dates[E], c0=rows[di - 1][1], + c1=rows[E][1], d_amt=d_amt, + spike=h["adj_ret"])) + print(f" {sym} {h['date']}: patch {len(patch)} cell(s), " + f"implied dist {d_amt:.3f}, ex-div {dates[E]}") + if not cells: + continue + pf = CORR / f"{sym.upper()}.json" + d = json.loads(pf.read_text()) if pf.exists() else {"symbol": sym} + d.setdefault("symbol", sym) + d.setdefault("note", "") + if notes: + d["note"] = (d["note"] + " " if d["note"] else "") + \ + " ".join(notes).strip() + d.setdefault("history", {}) + for dt, cells_ in cells.items(): + d["history"].setdefault(dt, {}).update(cells_) + if not args.dry_run: + pf.write_text(json.dumps(d, indent=2) + "\n") + applied += 1 + print(f"{'(dry) ' if args.dry_run else ''}wrote {pf.name}: " + f"{sum(len(v) for v in d['history'].values())} cell(s)") + print(f"{'dry-run: ' if args.dry_run else ''}{applied} symbol(s) repaired " + f"of {len(by_sym)} candidate(s)") + return 0 + + +if __name__ == "__main__": + main() diff --git a/scripts/scan_adj_misalign.py b/scripts/scan_adj_misalign.py new file mode 100644 index 0000000..94e28ea --- /dev/null +++ b/scripts/scan_adj_misalign.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Scan for Yahoo adj-close/event-date misalignment. + +Yahoo's Adj Close column applies a distribution's adjustment on the EVENT +date from its event feed. When that date precedes the actual ex-div date +(the market price only drops later), the adjusted series shows a bogus +spike on the event date and a snap-back a day or two later. + +Signature per symbol: a date D with |adj return| >= --min-adj (default 5%) +while |close return| on D is < --max-close (default 2%), AND the close +drops by more than half the implied adjustment within --horizon following +sessions (the price finally going ex-div). + +Usage: python3 scripts/scan_adj_misalign.py [--data DIR] [--out PATH] +Prints a summary and a table; writes markdown to --out (default +reports/adj_misalign/scan.md). +""" +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +ROOT_DEFAULT = Path.home() / "prog/fin/stocks" + + +def scan(root: Path, min_adj: float, max_close: float, horizon: int) -> list[dict]: + import data as D + hits = [] + syms = sorted({f.name[:-len("-history.csv")] for f in root.glob("*-history.csv")}) + for i, sym in enumerate(syms): + fr = D.FROZEN_DIR / f"{sym}-history.csv" + f = fr if (D.FROZEN_DIR.is_dir() and fr.exists()) else root / f"{sym}-history.csv" + if not f.exists(): + continue + rows = [] + for l in f.read_text(errors="replace").splitlines()[1:]: + if not l.strip(): + continue + r = l.split(",") + if len(r) < 6: + continue + try: + cl, aj = float(r[4]), float(r[5]) + except ValueError: # e.g. '%!f()' from an old goget bug + continue + rows.append((r[0], cl, aj)) + if len(rows) < 3: + continue + closes = [x[1] for x in rows] + adjs = [x[2] for x in rows] + for i in range(1, len(rows)): + if closes[i - 1] <= 0 or adjs[i - 1] <= 0 or closes[i] <= 0: + continue + adj_ret = adjs[i] / adjs[i - 1] - 1 + cl_ret = closes[i] / closes[i - 1] - 1 + if abs(adj_ret) < min_adj or abs(cl_ret) >= max_close: + continue + # the drop has to land within the next `horizon` sessions + j = i + horizon + if j >= len(rows) or closes[j] <= 0: + continue + later_drop = (closes[i - 1] - closes[j]) / closes[i - 1] + implied = min(abs(adj_ret), 0.6) # cap: events rarely exceed 60% + if later_drop < 0.5 * implied: + continue + hits.append({ + "symbol": sym, + "date": rows[i][0], + "adj_ret": round(adj_ret, 4), + "close_ret": round(cl_ret, 4), + "close_before": closes[i - 1], + "close_after_horizon": closes[j], + "drop_later": round(later_drop, 4), + }) + if i % 500 == 0: + print(f"{i}/{len(syms)} ...", file=sys.stderr) + return hits + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--data", type=Path, default=ROOT_DEFAULT) + ap.add_argument("--min-adj", type=float, default=0.05) + ap.add_argument("--max-close", type=float, default=0.02) + ap.add_argument("--horizon", type=int, default=3) + ap.add_argument("--out", type=Path, default=None) + args = ap.parse_args() + hits = scan(args.data, args.min_adj, args.max_close, args.horizon) + hits.sort(key=lambda h: -abs(h["adj_ret"])) + print(json.dumps({"hits": len(hits), + "symbols": len({h["symbol"] for h in hits})}, indent=1)) + for h in hits[:30]: + print(f"{h['symbol']:8s} {h['date']} adj {h['adj_ret']:+.1%} " + f"(close {h['close_ret']:+.2%}), drop later {h['drop_later']:.1%}") + if args.out: + args.out.parent.mkdir(parents=True, exist_ok=True) + lines = ["# Adj-close / event-date misalignment scan", + "", + f"params: min_adj={args.min_adj} max_close={args.max_close} " + f"horizon={args.horizon}; {len(hits)} hits on " + f"{len({h['symbol'] for h in hits})} symbols", + "", + "| symbol | date | adj ret | close ret | later drop |", + "|---|---|---|---|---|"] + lines += [f"| {h['symbol']} | {h['date']} | {h['adj_ret']:+.2%} | " + f"{h['close_ret']:+.2%} | {h['drop_later']:+.2%} |" for h in hits] + args.out.write_text("\n".join(lines) + "\n") + print(f"-> {args.out}") + return 0 + + +if __name__ == "__main__": + main() diff --git a/tests/test_data.py b/tests/test_data.py index 2d582e7..691060e 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -189,6 +189,37 @@ def main() -> int: D.CORRECTIONS_DIR = real_corr finally: shutil.rmtree(tmp, ignore_errors=True) + + # --- history cell corrections (misaligned Yahoo adj-close spike) --- + tmp = pathlib.Path(tempfile.mkdtemp(prefix="fd-")) + try: + root = tmp / "data"; cache = tmp / "cache"; corrdir = tmp / "corr" + root.mkdir(parents=True); cache.mkdir(); corrdir.mkdir() + (root / "ghh-history.csv").write_text( + "Date,Open,High,Low,Close,Adj Close,Volume\n" + "2020-12-10,30.10,30.10,30.10,30.10,11.247889,1000\n" + "2020-12-11,30.10,30.10,30.10,30.10,14.545517,1000\n" + "2020-12-14,23.22,23.22,23.22,23.22,11.220826,1000\n") + (corrdir / "GHH.json").write_text(json.dumps( + {"history": {"2020-12-11": {"Adj Close": 11.247889}}})) + real_corr = D.CORRECTIONS_DIR + D.CORRECTIONS_DIR = corrdir + try: + b = D.load_bundle(root=root, cache=cache) + t10, t11, t14 = (pd.Timestamp(x) for x in + ("2020-12-10", "2020-12-11", "2020-12-14")) + check("history: adj cell patched", + b.adj["ghh"][t11] == 11.247889) + check("history: other adj cells untouched", + b.adj["ghh"][t10] == 11.247889 and b.adj["ghh"][t14] == 11.220826) + check("history: raw close untouched", + b.close["ghh"][t11] == 30.10 and b.close["ghh"][t14] == 23.22) + b2 = D.load_bundle(root=root, cache=cache) + check("history: idempotent on reload", b2.adj["ghh"][t11] == 11.247889) + finally: + D.CORRECTIONS_DIR = real_corr + finally: + shutil.rmtree(tmp, ignore_errors=True) print(f"\n{PASS} passed, {FAIL} failed") return 1 if FAIL else 0