Greg Pomerantz
02aa750a06
230 tickers whose Yahoo chart responses now come back without a timestamp array (terminated/merged funds): goget overwrites the .json on every pass while ohlc.Conv skips the write, leaving the old CSVs as the last known series. Snapshot them into overrides/frozen/ (git-tracked, audited in reports/stale-funds.md) and make data.py prefer the frozen copies and ignore any future data-root rewrite/delete for those symbols, so the final series survives future goget runs. The cache manifest now covers the overrides dir too; incremental refresh skips data-root files of frozen symbols.
118 lines
4.6 KiB
Python
118 lines
4.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Detect stale tickers (Yahoo now returns no price data) and freeze them.
|
|
|
|
A ticker is "stale" when its {SYM}.json in the data root is a Yahoo chart
|
|
response WITHOUT a `timestamp` array. goget still overwrites that .json on
|
|
every download pass, and ohlc.Conv then refuses to (re)write the CSVs — so
|
|
the existing CSVs are the last complete series we will ever get from Yahoo
|
|
(usually because the fund was terminated, merged or liquidated).
|
|
|
|
This script snapshots those CSVs into overrides/frozen/ so the data survives
|
|
any future goget re-download, and writes reports/stale-funds.md for the
|
|
Tier-3 (official source) verification workflow.
|
|
|
|
The data root is never modified. Frozen copies are created/refreshed only
|
|
when the root CSVs are NEWER than the frozen copy (i.e. a download extended
|
|
the series before Yahoo went empty), so a re-run is idempotent.
|
|
|
|
Usage: python3 scripts/audit_stale.py [data_root]
|
|
"""
|
|
|
|
import json
|
|
import shutil
|
|
import sys
|
|
from datetime import date
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(sys.argv[1]).expanduser() if len(sys.argv) > 1 else Path("~/prog/fin/stocks").expanduser()
|
|
FROZEN = Path(__file__).parent.parent / "overrides" / "frozen"
|
|
REPORT = Path(__file__).parent.parent / "reports" / "stale-funds.md"
|
|
SUFFIXES = ("history", "dividend", "capitalGain")
|
|
|
|
|
|
def stale_symbols(root: Path):
|
|
"""{(sym, longName)} for JSONs without a timestamp array."""
|
|
out = []
|
|
for jf in sorted(root.glob("*.json")):
|
|
try:
|
|
d = json.loads(jf.read_text())
|
|
r = d.get("chart", {}).get("result")
|
|
if not isinstance(r, list) or not r:
|
|
continue # not a chart payload (e.g. stray files)
|
|
r0 = r[0]
|
|
if r0.get("timestamp"):
|
|
continue
|
|
out.append((jf.stem, r0.get("meta", {}).get("longName", "?")))
|
|
except Exception:
|
|
continue
|
|
return out
|
|
|
|
|
|
def last_bar(f: Path) -> str:
|
|
if not f.exists():
|
|
return ""
|
|
lines = f.read_text(errors="replace").strip().splitlines()
|
|
if len(lines) < 2:
|
|
return ""
|
|
return lines[-1].split(",")[0]
|
|
|
|
|
|
def main():
|
|
FROZEN.mkdir(parents=True, exist_ok=True)
|
|
stale = stale_symbols(ROOT)
|
|
rows = []
|
|
n_new = n_refresh = n_keep = 0
|
|
for sym, name in stale:
|
|
meta_path = FROZEN / f"{sym}.json"
|
|
meta = {}
|
|
if meta_path.exists():
|
|
meta = json.loads(meta_path.read_text())
|
|
|
|
# refresh frozen CSVs if the root CSVs moved forward
|
|
for sfx in SUFFIXES:
|
|
src = ROOT / f"{sym}-{sfx}.csv"
|
|
dst = FROZEN / f"{sym}-{sfx}.csv"
|
|
if not src.exists():
|
|
continue
|
|
if dst.exists() and dst.stat().st_mtime_ns >= src.stat().st_mtime_ns:
|
|
continue
|
|
if dst.exists():
|
|
# root CSV changed: verify it is a superset (newer last bar),
|
|
# never shrink the frozen series
|
|
if last_bar(src) < last_bar(dst):
|
|
print(f"WARN {sym}: root {sfx} last bar {last_bar(src)!r} is OLDER than "
|
|
f"frozen {last_bar(dst)!r}; keeping frozen copy")
|
|
continue
|
|
n_refresh += 1
|
|
else:
|
|
n_new += 1
|
|
shutil.copy2(src, dst)
|
|
|
|
meta["symbol"] = sym
|
|
meta["longName"] = name
|
|
meta["freeze_date"] = last_bar(FROZEN / f"{sym}-history.csv")
|
|
meta.setdefault("created", date.today().isoformat())
|
|
meta["reason"] = "yahoo-empty"
|
|
meta_path.write_text(json.dumps(meta, indent=1))
|
|
n_keep += 1
|
|
rows.append((sym, name, meta["freeze_date"],
|
|
meta.get("verified", False), meta.get("note", "")))
|
|
|
|
rows.sort(key=lambda r: (r[2] == "", r[2], r[0]))
|
|
REPORT.parent.mkdir(parents=True, exist_ok=True)
|
|
with REPORT.open("w") as f:
|
|
f.write("# Stale (Yahoo-empty) tickers — frozen in `overrides/frozen/`\n\n")
|
|
f.write("Data root copies are snapshots; `data.py` reads the frozen copies for these "
|
|
"symbols. Tier-3 verification: cross-check distributions/termination against "
|
|
"official sources, then set `verified: true` + `note` in `overrides/frozen/{SYM}.json`.\n\n")
|
|
f.write("| Symbol | Name | Last bar | Verified | Note |\n")
|
|
f.write("|---|---|---|---|---|\n")
|
|
for sym, name, fb, ver, note in rows:
|
|
f.write(f"| {sym} | {name} | {fb} | {'yes' if ver else ''} | {note} |\n")
|
|
print(f"{len(stale)} stale tickers; {n_new} new frozen, {n_refresh} refreshed, "
|
|
f"{n_keep} metadata entries; report -> {REPORT}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|