Greg Pomerantz
cba7291676
The cache now tracks every file in the data dir (mtime_ns + size) in
.cache/manifest.json. On load, a directory scan is compared against the
manifest:
- changed/added files are re-read and merged into the parquet panels
(one read + one concat + one write per touched panel; new values
win where present, old values kept where the new file is short)
- removed files drop their symbols (and names)
- an up-to-date cache is a ~30 ms memo hit
Measured on the real 4k-symbol set: full build 54 s, refresh of
5 modified + 1 added + 1 removed files 3.4 s. No scan TTL (a scan is
a few ms); a previous 5 s scan cache masked data updates.
Tests: tests/test_data.py (11 checks) added as step 1 of run_tests.sh.
121 lines
4.1 KiB
Python
121 lines
4.1 KiB
Python
"""Data-cache tests: incremental refresh from changed data files.
|
|
|
|
Uses a small synthetic data root in a temp dir — no real data, no server.
|
|
Run: .venv/bin/python tests/test_data.py
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
|
|
|
|
import data as D # noqa: E402
|
|
|
|
PASS, FAIL = 0, 0
|
|
|
|
|
|
def check(name: str, cond: bool, extra: str = "") -> None:
|
|
global PASS, FAIL
|
|
if cond:
|
|
PASS += 1
|
|
print(f" ok {name}", flush=True)
|
|
else:
|
|
FAIL += 1
|
|
print(f" FAIL {name} {extra}", flush=True)
|
|
|
|
|
|
def write_history(root: pathlib.Path, sym: str, last_price: float | None = None,
|
|
days: int = 10) -> pathlib.Path:
|
|
"""Prices 100.00, 101.00, ..., 109.00 (last day overridable)."""
|
|
f = root / f"{sym}-history.csv"
|
|
lines = ["Date,Open,High,Low,Close,Adj Close,Volume"]
|
|
for i in range(days):
|
|
p = last_price if (last_price is not None and i == days - 1) \
|
|
else 100.0 + i
|
|
lines.append(f"2024-01-{i + 1:02d},{p:.2f},{p:.2f},{p:.2f},{p:.2f},{p:.2f},1000")
|
|
f.write_text("\n".join(lines) + "\n")
|
|
return f
|
|
|
|
|
|
def touch(p: pathlib.Path) -> None:
|
|
"""Bump mtime so the manifest sees the file as changed."""
|
|
st = p.stat()
|
|
os.utime(p, ns=(st.st_atime_ns + 10**9, st.st_mtime_ns + 10**9))
|
|
|
|
|
|
def reload(d: pathlib.Path) -> D.Bundle:
|
|
return D.load_bundle(root=d["root"], cache=d["cache"])
|
|
|
|
|
|
def main() -> int:
|
|
tmp = pathlib.Path(tempfile.mkdtemp(prefix="fdatatest-"))
|
|
root, cache = tmp / "stocks", tmp / "cache"
|
|
root.mkdir()
|
|
d = {"root": root, "cache": cache}
|
|
try:
|
|
write_history(root, "aaa")
|
|
write_history(root, "bbb")
|
|
write_history(root, "ccc")
|
|
(root / "aaa-dividend.csv").write_text("Date,Dividends\n2024-01-05,0.5\n")
|
|
(root / "bbb.json").write_text(
|
|
json.dumps({"chart": {"result": [{"meta": {"longName": "Bee Bee Corp"}}]}}))
|
|
|
|
b = reload(d)
|
|
check("initial full build", list(b.adj.columns) == ["aaa", "bbb", "ccc"]
|
|
and b.adj["aaa"].iloc[-1] == 109.0, str(list(b.adj.columns)))
|
|
check("names from json", b.names.get("bbb") == "Bee Bee Corp")
|
|
check("dividend panel", b.div["aaa"].loc["2024-01-05"] == 0.5)
|
|
|
|
# --- modified file: last price changes
|
|
f = write_history(root, "aaa", last_price=200.0)
|
|
touch(f)
|
|
b = reload(d)
|
|
check("changed file re-read incrementally", b.adj["aaa"].iloc[-1] == 200.0)
|
|
check("untouched symbols intact", b.adj["bbb"].iloc[-1] == 109.0)
|
|
|
|
# --- new file: symbol added
|
|
write_history(root, "ddd")
|
|
b = reload(d)
|
|
check("new symbol added", "ddd" in b.adj.columns)
|
|
|
|
# --- deleted file: symbol dropped
|
|
(root / "ccc-history.csv").unlink()
|
|
b = reload(d)
|
|
check("deleted symbol dropped", "ccc" not in b.adj.columns
|
|
and "ccc" not in b.div.columns)
|
|
|
|
# --- name file added for a new symbol
|
|
(root / "ddd.json").write_text(
|
|
json.dumps({"chart": {"result": [{"meta": {"longName": "Dee Dee"}}]}}))
|
|
b = reload(d)
|
|
check("name added incrementally", b.names.get("ddd") == "Dee Dee")
|
|
|
|
# --- dividend file modified
|
|
(root / "aaa-dividend.csv").write_text(
|
|
"Date,Dividends\n2024-01-05,0.5\n2024-01-06,1.25\n")
|
|
b = reload(d)
|
|
check("dividend change picked up", b.div["aaa"].loc["2024-01-06"] == 1.25)
|
|
|
|
# --- no-change reload: cheap, no rewrite
|
|
before = (cache / "panel_adj.parquet").stat().st_mtime_ns
|
|
b = reload(d)
|
|
after = (cache / "panel_adj.parquet").stat().st_mtime_ns
|
|
check("no-op reload does not rewrite parquet", before == after)
|
|
|
|
# --- forced full rebuild still works
|
|
b = D.load_bundle(root=root, cache=cache, rebuild=True)
|
|
check("forced full rebuild", list(b.adj.columns) == ["aaa", "bbb", "ddd"])
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
print(f"\n{PASS} passed, {FAIL} failed")
|
|
return 1 if FAIL else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|