From 81ef602d52dd0815c6271d11bd0eac8af936d4ed Mon Sep 17 00:00:00 2001 From: Greg Pomerantz Date: Fri, 28 Aug 2026 14:57:01 -0400 Subject: [PATCH] CEF app tab + server watchdog + full-universe verification batch - Fund Lab tab: CEF tax-arb shortlist table (char actual-vs-model, discount, 5y return/vol, scenario hits, tenders, score) built from cef_character.json + cef_annual.json. - fundlab/server_watchdog.sh: relaunches Streamlit on :8599 if the health endpoint stops answering (the old watchdog only covered the overnight screen and had exited). - fundlab/cef_universe_run.py: resumable batch extending per-share verification to all 295 CEFs -> cef_annual_all.json. --- app.py | 49 +++++++++++++++++++++++ fundlab/cef_universe_run.log | 1 + fundlab/cef_universe_run.py | 75 ++++++++++++++++++++++++++++++++++++ fundlab/server_watchdog.log | 2 + fundlab/server_watchdog.sh | 31 +++++++++++++++ 5 files changed, 158 insertions(+) create mode 100644 fundlab/cef_universe_run.log create mode 100644 fundlab/cef_universe_run.py create mode 100644 fundlab/server_watchdog.log create mode 100755 fundlab/server_watchdog.sh diff --git a/app.py b/app.py index ff26f50..ea550a1 100644 --- a/app.py +++ b/app.py @@ -1109,3 +1109,52 @@ with tab_fundlab: "counted where the layout exposes a dollar value; the bucket " "table is a rough keyword classification. N-PORT holdings are " "quarterly and up to ~60 days stale.") + + # --- closed-end funds (tax-arb shortlist) --------------------------- + import pandas as _pd + _fl_dir = Path(__file__).parent / "fundlab" + _char_path = _fl_dir / "cef_character.json" + if _char_path.exists(): + _char = json.loads(_char_path.read_text()) + _annual = {} + _ann_path = _fl_dir / "cef_annual.json" + if _ann_path.exists(): + _annual = json.loads(_ann_path.read_text()) + _rows = [] + for _s, _c in _char.items(): + _a = _annual.get(_s.upper(), {}) + _ch = _c.get("character") + _actual = False + if _a.get("char_actual") is not None: + _ch = _a["char_actual"] + _actual = True + _t5, _vol = _c.get("t5"), _c.get("vol5") + _score = (None if _ch is None or _t5 is None or _vol is None + else round(_ch * (_t5 + 0.4 * _vol), 4)) + _disc = _a.get("disc_now_approx") + _rows.append({ + "sym": _s.upper(), + "name": (_a.get("name") or _c.get("name") or "")[:44], + "char": _ch, + "actual": _actual, + "disc%": None if _disc is None else round(100 * _disc, 1), + "t5%": None if _t5 is None else round(100 * _t5, 1), + "vol5%": None if _vol is None else round(100 * _vol, 1), + "pos_scen": _c.get("n_pos_scen"), + "tenders": _a.get("n_tender"), + "score": _score}) + _cef_df = _pd.DataFrame(_rows) + st.divider() + st.subheader("Closed-end funds — tax-arb shortlist") + st.dataframe( + _cef_df.sort_values("score", ascending=False, na_position="last"), + width="stretch", height=460) + st.caption( + "char = distribution character 0–1 (0 = all ordinary income, 1 = " + "all capital-gains/ROC). actual = verified from the fund's own " + "per-share financial highlights (divs + gains + ROC == total " + "distributions per year, plus the full NAV chain); otherwise the " + "return-sleeve model. disc% = market vs NAV. pos_scen = severe-" + "drawdown scenarios with a positive 5y return. score = char × " + "(5y total return + 0.4 × 5y vol) — harvestable character " + "weighted by damage potential.") diff --git a/fundlab/cef_universe_run.log b/fundlab/cef_universe_run.log new file mode 100644 index 0000000..98ec982 --- /dev/null +++ b/fundlab/cef_universe_run.log @@ -0,0 +1 @@ +[2026-08-28 14:56:15] universe=295 done=0 todo=295 diff --git a/fundlab/cef_universe_run.py b/fundlab/cef_universe_run.py new file mode 100644 index 0000000..226abf7 --- /dev/null +++ b/fundlab/cef_universe_run.py @@ -0,0 +1,75 @@ +"""Extend per-share verification to the FULL CEF universe (295 tickers). + +The 50-fund shortlist run (fundlab.cef_annual) wrote fundlab/cef_annual.json. +This batch runs the same analyze() over every ticker in +fundlab/cef_universe.json and writes fundlab/cef_annual_all.json — +incrementally, so a killed run resumes by re-running this script (cached +funds are skipped, as are those already in the output file). + +Usage: + .venv/bin/python -m fundlab.cef_universe_run # all, resumable + .venv/bin/python -m fundlab.cef_universe_run --retry-err # redo failed ones +""" +from __future__ import annotations + +import json +import sys +import time +from pathlib import Path + +from fundlab.cef_annual import (CACHE, analyze, character_score, + current_nav_discount) + +HERE = Path(__file__).parent +UNIVERSE = HERE / "cef_universe.json" +OUT = HERE / "cef_annual_all.json" +LOG = HERE / "cef_universe_run.log" + + +def log(msg: str) -> None: + line = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] {msg}" + print(line, flush=True) + with LOG.open("a") as f: + f.write(line + "\n") + + +def run(retry_err: bool = False) -> dict: + uni = json.loads(UNIVERSE.read_text()) + prev: dict = {} + if OUT.exists(): + prev = json.loads(OUT.read_text()) + todo = [] + for s in sorted(uni): + p = prev.get(s) + if p is None or (retry_err and "error" in p): + todo.append(s) + log(f"universe={len(uni)} done={len(uni) - len(todo)} todo={len(todo)}" + + (" [retry-err]" if retry_err else "")) + res = dict(prev) + for i, s in enumerate(todo, 1): + u = uni[s] + cik = int(u.get("cik", 0)) + try: + a = analyze(s, cik) + except Exception as e: # noqa: BLE001 - batch must not die + a = {"sym": s, "error": f"analyze exception: {e}"} + a["char_actual"] = (round(character_score(a, u.get("name", "")), 2) + if "share_div" in a else None) + if "nav_end" in a: + d = current_nav_discount(a) + a["disc_now_approx"] = round(d, 4) if d is not None else None + res[s] = a + if "share_div" in a: + log(f"{i:3}/{len(todo)} {s:7} div {a['share_div']:.0%} " + f"gain {a['share_gains']:.0%} roc {a['share_roc']:.0%} " + f"char={a['char_actual']}") + else: + log(f"{i:3}/{len(todo)} {s:7} -- {a.get('error', 'no data')}") + OUT.write_text(json.dumps(res, indent=1, default=str)) + log(f"done: parsed={sum(1 for v in res.values() if 'share_div' in v)} " + f"of {len(res)}") + return res + + +if __name__ == "__main__": + run(retry_err="--retry-err" in sys.argv) diff --git a/fundlab/server_watchdog.log b/fundlab/server_watchdog.log new file mode 100644 index 0000000..84597d9 --- /dev/null +++ b/fundlab/server_watchdog.log @@ -0,0 +1,2 @@ +2026-08-28 14:52:00 +server_watchdog: no answer on :8599 - relaunching diff --git a/fundlab/server_watchdog.sh b/fundlab/server_watchdog.sh new file mode 100755 index 0000000..a3713a8 --- /dev/null +++ b/fundlab/server_watchdog.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Keep the Streamlit app alive on port 8599. +# +# The server (or the machine) can drop it at any time, so every 5 minutes +# this checks whether something is answering on the port and, if not, +# relaunches it detached. Streamlit only hot-reloads app.py, so module +# changes need this relaunch (or a manual one) to take effect. +# +# Usage: setsid nohup fundlab/server_watchdog.sh >fundlab/server_watchdog.log 2>&1 & +set -u +cd "$(dirname "$0")/.." +PORT=8599 +INTERVAL=300 + +alive() { + curl -s -o /dev/null --max-time 8 "http://127.0.0.1:${PORT}/_stcore/health" \ + && [ "$(curl -s --max-time 8 "http://127.0.0.1:${PORT}/_stcore/health")" = "ok" ] +} + +while true; do + if ! alive; then + { date "+%Y-%m-%d %H:%M:%S" + echo "server_watchdog: no answer on :${PORT} - relaunching" + } >> fundlab/server_watchdog.log + setsid nohup .venv/bin/streamlit run app.py \ + --server.port "${PORT}" --server.headless true \ + --browser.gatherUsageStats false >> fundlab/streamlit.log 2>&1 < /dev/null & + sleep 15 + fi + sleep "${INTERVAL}" +done