diff --git a/app.py b/app.py index d6a62b8..b56071f 100644 --- a/app.py +++ b/app.py @@ -289,7 +289,7 @@ def _compute_portfolio(w_key: tuple, scheme: str | None, cost_bps: float, root: str, gen: int): w = dict(w_key) r = portfolio_returns(bundle.adj, w, rebalance=scheme, cost_bps=cost_bps, start=start) - t = after_tax_portfolio(bundle.adj, bundle.div, bundle.capg, w, rebalance=scheme, + t = after_tax_portfolio(bundle.close, bundle.div, bundle.capg, w, rebalance=scheme, cost_bps=cost_bps, lt_rate=lt, st_rate=st_r, div_rate=div, start=start) return r, t diff --git a/overrides/corrections/JLPSX.json b/overrides/corrections/JLPSX.json index 3f5c26d..0f760d1 100644 --- a/overrides/corrections/JLPSX.json +++ b/overrides/corrections/JLPSX.json @@ -1,7 +1,7 @@ { "symbol": "jlpsx", "name": "JPMorgan U.S. Large Cap Core Plus Fund (I shares)", - "note": "Yahoo listed the year-end capital-gain distribution in BOTH the dividend and capitalGains event files. Expressed as layout-agnostic invariants because a 2026-08 re-download shows Yahoo no longer returns capitalGain events for this fund at all (the dividend stream still carries the year-end rows): 'dedup' keeps at most one copy of each (date, amount) (the capitalGain copy when both files carry it; the lone dividend copy otherwise); 'drop_capg_copy' drops the spurious capitalGain row on a date where the dividend row is the true distribution (any amount). Official per-share data (JPMorgan 497, filed 2025-11-03, Class I, FYE June 30): FY2025 1.37, FY2024 2.03, FY2023 4.86, FY2022 6.78, FY2021 6.89 - local sums after dedup match every one (1.368, 2.029, 4.859, 6.785, 6.889). 2006-12-15 (div 0.207 vs capg 0.141) and 2019-08-08 (div 4.138 vs capg 4.077) keep the dividend amount per the pattern confirmed on all other verified same-date pairs; pre-2012 years have no official totals (the 2021 497 is delisted from SEC Archives) - mechanism-inferred. 'history' patches the 2020-12-11 Adj Close cell: Yahoo dated the 6.824 distribution on the 12-11 record date but the market went ex-div on 2020-12-14 (close 30.10 -> 23.22, i.e. -6.88), so the raw Adj Close column spiked +29.3% on 12-11 and snapped back -22.9% on 12-14; setting 12-11 to the 12-10 value (11.247889) removes the bogus 3-day wiggle (cumulative return unchanged). adj-close repair: Yahoo dated a distribution on 2020-12-11 (record date) but the price went ex-div on 2020-12-14 (close 30.10 -> 23.22, drop ~6.824); the raw Adj Close column spiked +29.3% on 2020-12-11 and snapped back later. Cells in [2020-12-11, 2020-12-14) rescaled by (1-f) so the adjusted path is smooth; cumulative returns are unchanged. Mechanism: scripts/fix_adj_misalign.py, scan: reports/adj_misalign/scan.md.", + "note": "Yahoo listed the year-end capital-gain distribution in BOTH the dividend and capitalGains event files. Expressed as layout-agnostic invariants because a 2026-08 re-download shows Yahoo no longer returns capitalGain events for this fund at all (the dividend stream still carries the year-end rows): 'dedup' keeps at most one copy of each (date, amount) (the capitalGain copy when both files carry it; the lone dividend copy otherwise); 'drop_capg_copy' drops the spurious capitalGain row on a date where the dividend row is the true distribution (any amount). Official per-share data (JPMorgan 497, filed 2025-11-03, Class I, FYE June 30): FY2025 1.37, FY2024 2.03, FY2023 4.86, FY2022 6.78, FY2021 6.89 - local sums after dedup match every one (1.368, 2.029, 4.859, 6.785, 6.889). 2006-12-15 (div 0.207 vs capg 0.141) and 2019-08-08 (div 4.138 vs capg 4.077) keep the dividend amount per the pattern confirmed on all other verified same-date pairs; pre-2012 years have no official totals (the 2021 497 is delisted from SEC Archives) - mechanism-inferred. 'history' patches the 2020-12-11 Adj Close cell: Yahoo dated the 6.824 distribution on the 12-11 record date but the market went ex-div on 2020-12-14 (close 30.10 -> 23.22, i.e. -6.88), so the raw Adj Close column spiked +29.3% on 12-11 and snapped back -22.9% on 12-14; setting 12-11 to the 12-10 value (11.247889) removes the bogus 3-day wiggle (cumulative return unchanged). adj-close repair: Yahoo dated a distribution on 2020-12-11 (record date) but the price went ex-div on 2020-12-14 (close 30.10 -> 23.22, drop ~6.824); the raw Adj Close column spiked +29.3% on 2020-12-11 and snapped back later. Cells in [2020-12-11, 2020-12-14) rescaled by (1-f) so the adjusted path is smooth; cumulative returns are unchanged. Mechanism: scripts/fix_adj_misalign.py, scan: reports/adj_misalign/scan.md. Distribution date moved 2020-12-11 -> 2020-12-14 (true ex-div date, close 30.10 -> 23.22 for JLPSX / 30.29 -> 23.40 for JLPYX): Yahoo dated the 6.824 capital-gain distribution on the 12-11 record date, but the after-tax model reinvests at the event date's raw close, so the cash flow must sit on the day the price actually dropped. Ops: remove both file copies on 12-11, add the capitalGains copy on 12-14 (kept in capitalGains so it is taxed at the cap-gain rate, matching the 497's CG 6.82 / NII 0.07 split).", "verified": [ "FY2021-FY2025 exact match against 497 filed 2025-11-03 (accession 0001193125-25-261054, CIK 0000763852)" ], @@ -42,10 +42,6 @@ "2019-12-12", 1.517 ], - [ - "2020-12-11", - 6.824 - ], [ "2021-12-13", 6.785 @@ -71,5 +67,27 @@ "2020-12-11": { "Adj Close": 11.247889 } + }, + "dividends": { + "remove": [ + [ + "2020-12-11", + 6.824 + ] + ] + }, + "capitalGains": { + "remove": [ + [ + "2020-12-11", + 6.824 + ] + ], + "add": [ + [ + "2020-12-14", + 6.824 + ] + ] } } diff --git a/overrides/corrections/JLPYX.json b/overrides/corrections/JLPYX.json index 24b875c..aae8491 100644 --- a/overrides/corrections/JLPYX.json +++ b/overrides/corrections/JLPYX.json @@ -13,10 +13,6 @@ "2019-12-12", 1.517 ], - [ - "2020-12-11", - 6.824 - ], [ "2021-12-13", 6.785 @@ -38,10 +34,32 @@ 0.656 ] ], - "note": "Bulk correction from the double-listing scan (scripts/scan_double_listing.py): Yahoo listed this distribution in both the dividend and capitalGains event files (same date, same amount); one copy is spurious. The 'dedup' invariant keeps at most one copy, layout-agnostically. Mechanism verified against official filings for JLPSX/GDEUX/GSOUX/FAEVX/CVSIX/FZAGX; mechanism-inferred for this symbol (no per-fund official check). Revert the entry if a future official cross-check shows both rows were real. adj-close repair: Yahoo dated a distribution on 2020-12-11 (record date) but the price went ex-div on 2020-12-14 (close 30.29 -> 23.40, drop ~6.824); the raw Adj Close column spiked +29.1% on 2020-12-11 and snapped back later. Cells in [2020-12-11, 2020-12-14) rescaled by (1-f) so the adjusted path is smooth; cumulative returns are unchanged. Mechanism: scripts/fix_adj_misalign.py, scan: reports/adj_misalign/scan.md.", + "note": "Bulk correction from the double-listing scan (scripts/scan_double_listing.py): Yahoo listed this distribution in both the dividend and capitalGains event files (same date, same amount); one copy is spurious. The 'dedup' invariant keeps at most one copy, layout-agnostically. Mechanism verified against official filings for JLPSX/GDEUX/GSOUX/FAEVX/CVSIX/FZAGX; mechanism-inferred for this symbol (no per-fund official check). Revert the entry if a future official cross-check shows both rows were real. adj-close repair: Yahoo dated a distribution on 2020-12-11 (record date) but the price went ex-div on 2020-12-14 (close 30.29 -> 23.40, drop ~6.824); the raw Adj Close column spiked +29.1% on 2020-12-11 and snapped back later. Cells in [2020-12-11, 2020-12-14) rescaled by (1-f) so the adjusted path is smooth; cumulative returns are unchanged. Mechanism: scripts/fix_adj_misalign.py, scan: reports/adj_misalign/scan.md. Distribution date moved 2020-12-11 -> 2020-12-14 (true ex-div date, close 30.10 -> 23.22 for JLPSX / 30.29 -> 23.40 for JLPYX): Yahoo dated the 6.824 capital-gain distribution on the 12-11 record date, but the after-tax model reinvests at the event date's raw close, so the cash flow must sit on the day the price actually dropped. Ops: remove both file copies on 12-11, add the capitalGains copy on 12-14 (kept in capitalGains so it is taxed at the cap-gain rate, matching the 497's CG 6.82 / NII 0.07 split).", "history": { "2020-12-11": { "Adj Close": 11.400895 } + }, + "dividends": { + "remove": [ + [ + "2020-12-11", + 6.824 + ] + ] + }, + "capitalGains": { + "remove": [ + [ + "2020-12-11", + 6.824 + ] + ], + "add": [ + [ + "2020-12-14", + 6.824 + ] + ] } } diff --git a/run_tests.sh b/run_tests.sh index 34a66c2..8175a6a 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -9,15 +9,19 @@ cd "$(dirname "$0")" fail=0 -echo "=== 1/3 data cache tests (incremental refresh) ===" +echo "=== 1/4 data cache tests (incremental refresh) ===" .venv/bin/python tests/test_data.py || fail=1 echo -echo "=== 2/3 app tests (AppTest, no browser) ===" +echo "=== 2/4 after-tax model tests (synthetic) ===" +.venv/bin/python tests/test_tax.py || fail=1 +echo + +echo "=== 3/4 app tests (AppTest, no browser) ===" .venv/bin/python tests/test_app.py || fail=1 echo -echo "=== 3/3 browser e2e (Playwright, needs the server) ===" +echo "=== 4/4 browser e2e (Playwright, needs the server) ===" if curl -s -m 3 -o /dev/null http://localhost:8599/healthz; then .venv/bin/python tests/test_e2e_browser.py || fail=1 else diff --git a/tax.py b/tax.py index c2b4e95..78c29fc 100644 --- a/tax.py +++ b/tax.py @@ -4,11 +4,18 @@ Model (simplified DAS) ---------------------- The account starts at 1.0 (growth-ratio units; no fixed capital). -- Holdings are valued on *adjusted* prices, which already assume - distributions are reinvested. Distributions therefore flow through as: +- Holdings are valued on RAW (close) prices, in raw share units. + Distributions (per-share dollars from the event files) are taxed on + their event date and the after-tax remainder is REINVESTED at the same + day's raw close (the ex-div price): a fund paying d drops by d, the + investor keeps d(1-tau) and buys it back cheaper. The tax's effect is + carried by the (smaller) reinvested units; it is recorded in `taxes` + but not separately deducted from cash (that would double-count it). dividend income -> taxed at `div_rate` capital gain dist-> taxed at `lt_rate` - The after-tax remainder increases each lot's cost basis proportionally. + The after-tax remainder also increases each lot's cost basis + proportionally (standard after-tax-IRR convention, so a liquidation + doesn't re-tax the distribution). - On rebalance, sells are FIFO. A lot is long-term if held more than 365 days at sale, else short-term; realized gains/losses are taxed at `lt_rate` / `st_rate`. @@ -20,6 +27,12 @@ The account starts at 1.0 (growth-ratio units; no fixed capital). rates. This is what you would actually have in your pocket after selling everything and filing your taxes. +NOTE: distribution event dates must be EX-DIV dates for the reinvestment +pricing to be right (the raw close on that day already reflects the +drop). Yahoo usually dates events on the ex-div date; the record-date +misalignments found by scripts/scan_adj_misalign.py are corrected by +remove/add ops in overrides/corrections/ where a symbol is analyzed. + Deliberately NOT modeled: loss carryover, wash sales, state rates, brackets. """ @@ -65,15 +78,17 @@ def _liquidation_tax(lt_gain: float, lt_loss: float, return max(st_net + lt_net, 0.0) * st_rate -def after_tax_portfolio(adj: pd.DataFrame, div: pd.DataFrame, capg: pd.DataFrame, +def after_tax_portfolio(close: pd.DataFrame, div: pd.DataFrame, capg: pd.DataFrame, weights: dict[str, float], rebalance: str | None = None, cost_bps: float = 0.0, lt_rate: float = 0.20, st_rate: float = 0.15, div_rate: float = 0.15, start: str | None = None, end: str | None = None ) -> TaxResult: - syms = [s for s in weights if s in adj.columns] - p = adj[syms] + """`close` must be RAW close prices (bundle.close), not adj: the + per-share distribution dollars are raw, so units must be raw shares.""" + syms = [s for s in weights if s in close.columns] + p = close[syms] d = div[[s for s in syms if s in div.columns]].reindex(index=p.index, columns=syms).fillna(0.0) c = capg[[s for s in syms if s in capg.columns]].reindex(index=p.index, columns=syms).fillna(0.0) if start or end: @@ -112,23 +127,30 @@ def after_tax_portfolio(adj: pd.DataFrame, div: pd.DataFrame, capg: pd.DataFrame for t in range(n): prices = pv[t] - market_value = float(np.dot(units, prices)) - # --- distributions (taxed, net flows back into basis) ----------- + # --- distributions: pay tax, reinvest the after-tax remainder at + # today's close. The tax is recorded in tax_rows but NOT also + # deducted from cash: it is already reflected in the (smaller) + # reinvested units, so a second deduction would double-count it. dinc = units * dv[t] cinc = units * cv[t] d_tax = float(dinc.sum() * div_rate) c_tax = float(cinc.sum() * lt_rate) - cash -= d_tax + c_tax tax_rows[t, 0] = d_tax tax_rows[t, 1] = c_tax - for i in range(len(syms)): # grow cost basis with reinvested net + for i in range(len(syms)): net = (dv[t, i] * (1 - div_rate) + cv[t, i] * (1 - lt_rate)) * units[i] - if net <= 0 or not lots[i]: + if net <= 0: continue - tot = sum(l.units for l in lots[i]) - for l in lots[i]: - l.cost += net * (l.units / tot) + px_i = float(prices[i]) + if px_i > 0: + units[i] += net / px_i # reinvest the after-tax cash + if lots[i]: # grow cost basis + tot = sum(l.units for l in lots[i]) + for l in lots[i]: + l.cost += net * (l.units / tot) + + market_value = float(np.dot(units, prices)) # post-reinvestment # --- rebalance to target weights -------------------------------- if idx[t] in rebal: diff --git a/tests/test_tax.py b/tests/test_tax.py new file mode 100644 index 0000000..564002a --- /dev/null +++ b/tests/test_tax.py @@ -0,0 +1,97 @@ +"""Unit tests for the after-tax model (tax.py) — synthetic, no data bundle. + +Key regression: distributions are RAW per-share dollars, so the account +must hold RAW share units (valued on close prices). The old code valued +units on ADJ (total-return index) prices, which overstates every +distribution — and its tax — by the raw/adj ratio (e.g. a fund that has +paid many distributions: adj 11 vs raw 30 -> 2.7x overcharge). + +Run: .venv/bin/python tests/test_tax.py +""" +from __future__ import annotations + +import sys +from pathlib import Path + +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +import tax # noqa: E402 + +PASS, FAIL = 0, 0 + + +def check(name: str, ok: bool) -> None: + global PASS, FAIL + PASS += bool(ok) + FAIL += not ok + print((" ok " if ok else " FAIL ") + name) + + +def main() -> int: + # One fund, raw prices 30 -> 34, ex-div 5.00 on day 6 (34 -> 29), + # then 29.5. Account starts at 1.0 = 1/30 share. + dates = pd.to_datetime(["2024-01-0%d" % i for i in range(1, 8)]) + close = pd.DataFrame({"f": [30.0, 31.0, 32.0, 33.0, 34.0, 29.0, 29.5]}, + index=dates) + div = pd.DataFrame({"f": [0, 0, 0, 0, 0, 0.0, 0.0]}, index=dates) + capg = pd.DataFrame({"f": [0, 0, 0, 0, 0, 5.0, 0.0]}, index=dates) + + r = tax.after_tax_portfolio(close, div, capg, {"f": 1.0}, lt_rate=0.20, + st_rate=0.15, div_rate=0.15) + t5 = dates[5] + + # --- the regression: tax on the distribution is 20% of 5/30 of the + # account (the distribution as a fraction of the RAW price), i.e. + # 3.333% — not 5/adj (which the old code implied, ~4% with this + # adj path and up to 2.7x for high-history funds). + check("cap-gain tax is 20% of (5/30) of the account", + abs(r.taxes["capg_tax"][t5] - 0.2 * 5.0 / 30.0) < 1e-12) + check("no dividend tax", r.taxes["div_tax"].abs().sum() == 0.0) + + # --- reinvestment: the after-tax remainder (5*0.8 of the 1/30 share) + # is bought at the ex-div close 29, so MV on the event day is + # (1/30)*(29 + 4) = 33/30. The tax is already inside the (smaller) + # units — it must NOT also show up in equity as a second deduction. + mv5 = r.equity[t5] + r.liq_tax[t5] # no cash component on this day + check("market value on ex-div day = 33/30 (after-tax reinvested)", + abs(mv5 - 33.0 / 30.0) < 1e-9) + + # --- liquidation value: price drop (5/30) is exactly offset by the + # distribution kept, so equity falls by the true tax cost only. + # Day-5 equity = MV(33/30) - liq_tax(0: basis 1+4/30 > MV, a "loss" + # under the after-tax basis rule). + check("equity on ex-div day = 33/30 (tax cost lives in the units)", + abs(r.equity[t5] - 33.0 / 30.0) < 1e-9) + + # --- pre-event day: plain ST-marked liquidation value. + t4 = dates[4] + check("equity day before = MV - 15% ST gain tax", + abs(r.equity[t4] - (34.0 / 30.0) * (1 - 0.15 * (4.0 / 34.0))) < 1e-9) + + # --- cross-day consistency: equity(5)/equity(4) from the explicit + # values (day 4 = 34/30 MV less 15% ST gain tax; day 5 = 33/30). + check("ex-div equity step matches explicit values", + abs(r.equity[t5] / r.equity[t4] + - (33.0 / 30.0) / (34.0 / 30.0 - 0.15 * 4.0 / 30.0)) < 1e-9) + + # --- no rebalancing: one lot survives, units grew by the reinvested + # net amount at the ex-div price. + check("single lot outstanding", r.lots_outstanding == 1) + + # --- dividend leg taxed at div_rate, cap-gain leg at lt_rate: + # split a 1.00 distribution 0.4 div / 0.6 capg on day 6. + div2 = pd.DataFrame({"f": [0, 0, 0, 0, 0, 0.4, 0.0]}, index=dates) + capg2 = pd.DataFrame({"f": [0, 0, 0, 0, 0, 0.6, 0.0]}, index=dates) + r2 = tax.after_tax_portfolio(close, div2, capg2, {"f": 1.0}, lt_rate=0.20, + st_rate=0.15, div_rate=0.10) + check("split distribution taxed by component rate", + abs(r2.taxes["capg_tax"][t5] - 0.20 * 0.6 / 30.0) < 1e-12 + and abs(r2.taxes["div_tax"][t5] - 0.10 * 0.4 / 30.0) < 1e-12) + + print(f"\n{PASS} passed, {FAIL} failed") + return 1 if FAIL else 0 + + +if __name__ == "__main__": + sys.exit(main())