Greg Pomerantz
bb8e39605e
Yahoo's 2026 event-feed change (capitalGain events dropped for some funds; no events at all for terminated tickers) can wipe good event history on re-download. Defense in depth: - overrides/event-backup/: last-known-good copy of every dividend/ capitalGain file (8,192 files, 58 MB); refresh with scripts/backup_events.py after each dump update - data.py event_file(): frozen > data-root (while populated) > backup; used by panel reads, verify_official, and the double-listing scanner - README: capital-gain files are mutual-fund-only in this dump; Yahoo has no LT/ST split (tax.py taxes capg at lt_rate; the per-fund split would come from fund-company annual tax statements or commercial feeds) - tests: exact Timestamp .loc keys (pandas 3.x string matching returns a Series on large DatetimeIndex)
47 lines
1.6 KiB
Python
47 lines
1.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Copy the dividend/capitalGain event files from the data root into
|
|
overrides/event-backup/ (the fallback data.py uses when a re-download
|
|
empties them — Yahoo dropped capitalGain events for some funds in 2026).
|
|
|
|
Re-runnable: it refreshes every file from the current data root, so run it
|
|
again whenever the dump is updated (or not — the backup is a floor, new
|
|
data still wins while it is non-empty). Symbols with a frozen copy are
|
|
skipped (overrides/frozen/ already preserves them).
|
|
|
|
Usage: python3 scripts/backup_events.py [--data DIR]
|
|
"""
|
|
import argparse
|
|
import shutil
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
BACKUP = ROOT / "overrides" / "event-backup"
|
|
FROZEN = ROOT / "overrides" / "frozen"
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--data", type=Path, default=Path("~/prog/fin/stocks").expanduser())
|
|
args = ap.parse_args()
|
|
BACKUP.mkdir(parents=True, exist_ok=True)
|
|
n = skipped = 0
|
|
for sfx in ("dividend", "capitalGain"):
|
|
for f in sorted(args.data.glob(f"*-{sfx}.csv")):
|
|
sym = f.name[: -len(f"-{sfx}.csv")]
|
|
if (FROZEN / f"{sym}-history.csv").exists():
|
|
skipped += 1
|
|
continue
|
|
# only files with real rows (a re-download may leave header-only)
|
|
if sum(1 for _ in open(f, errors="replace")) <= 1:
|
|
continue
|
|
shutil.copyfile(f, BACKUP / f.name)
|
|
n += 1
|
|
print(f"backed up {n} event files to {BACKUP}, "
|
|
f"skipped {skipped} frozen symbols")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|