Greg Pomerantz
915e04c2c3
tools/jank_eval.py: strict PASS/FAIL jank evaluator (the referee).
- slot grid from VSYNC ftn edges (hardware truth); anchor fallback only
- PASS iff per active window: no dropped slot, no same-slot double, no
present past the SF latch, no >20ms present gap, phase spread <=0.5ms
- self-test: 7 synthetic log files through the same main() path, all
classified correctly; validated on the emulator (60 Hz emu display
detected, active windows found)
- re-judges slt4 honestly: my 'clean' drag-1 window is FAIL (30.6ms +
21ms stalls, 20 past-latch presents, 1.1ms spread) — the user saw it
lib.rs/stg.rs: slot-based wake guard replaces the interval freshness
guard (0.2ms false-reject margin). A wake renders iff its edge-grid slot
is strictly after the last rendered slot; a gap of >=2 slots emits
MISS {mon, slots, nowake} — the missing evidence for stall slots
(wakes lost vs loop blocked vs guard).
sltest: log drain batch-flush (64 lines) — per-line FUSE flush stalled
the drain thread for minutes (file frozen, process alive).
gallery: MISS lines added to the perf-log allowlist.
303 lines
12 KiB
Python
303 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
jank_eval.py — deterministic 120 Hz scroll-jank evaluator.
|
|
|
|
Reads app-side perf logs (PRES/VSYNC lines from the vendored stg/vkprobe
|
|
instrumentation) and produces a strict, repeatable PASS/FAIL report against
|
|
the zero-jank definition (docs/BUGS/hypotheses-120hz.md):
|
|
|
|
PASS iff, in every ACTIVE window (sustained rendering):
|
|
1. no DROPPED slot (present in slot k, next present in slot k+2+)
|
|
2. no DOUBLE (two presents in the same display slot)
|
|
3. no LATE present (edge-grid phase > +5.0 ms, past the SF latch)
|
|
4. no STALL (present-to-present > 20 ms)
|
|
5. phase spread p10-p90 <= 0.5 ms (edge grid: phase within slot)
|
|
|
|
SLOT GRID (the part that takes care to get right):
|
|
* PRIMARY: the VSYNC `ftn=` field is the hardware vsync EDGE timestamp.
|
|
slot(m) = floor((m - ftn0)/P), phase = (m-ftn0) mod P in [0,P).
|
|
Exact and unambiguous for any present phase. P = median 1-slot ftn
|
|
delta (hardware truth).
|
|
* FALLBACK (no VSYNC lines, e.g. emulator sanity runs): slot =
|
|
round((m - anchor)/P) with anchor = circular mean of present phases.
|
|
Ambiguous for presents >P/2 from the anchor (documented limitation);
|
|
LATE detection is unreliable in this mode.
|
|
|
|
Windows: --windows "startMs,endMs name; ..." (ms from first present)
|
|
or auto-detected (present rate > 60/s active, < 10/s idle).
|
|
|
|
Exit: 0 = PASS, 1 = FAIL, 2 = no usable data.
|
|
Self-test: --selftest (synthesizes log FILES and runs the same main()
|
|
path end-to-end; the tool must classify each case or exit non-zero).
|
|
"""
|
|
import argparse, math, os, re, sys, tempfile
|
|
from collections import defaultdict
|
|
|
|
PRES_RE = re.compile(r"^PRES f=(\d+)\w* mon=(\d+)")
|
|
VSYNC_RE = re.compile(r"^VSYNC ftn=(\d+) mon=(\d+)")
|
|
|
|
def parse_log(lines):
|
|
pres, ftns = [], []
|
|
for l in lines:
|
|
m = PRES_RE.match(l)
|
|
if m:
|
|
pres.append((int(m.group(1)), int(m.group(2))))
|
|
continue
|
|
m = VSYNC_RE.match(l)
|
|
if m:
|
|
ftns.append(int(m.group(1)))
|
|
return pres, ftns
|
|
|
|
def vsync_period(ftns):
|
|
"""Hardware period: median of the 1-slot ftn deltas."""
|
|
if len(ftns) < 30: return None
|
|
d = sorted(b - a for a, b in zip(ftns, ftns[1:]) if 4_000_000 < b - a < 25_000_000)
|
|
return d[len(d)//2] if d else None
|
|
|
|
def fallback_period(monots):
|
|
"""Mean of the deltas (telescoping — unbiased by correlated jitter).
|
|
Do NOT use a fitted period for the verdict on the phone; it is a
|
|
sanity-run convenience for logs without VSYNC edges."""
|
|
n = len(monots)
|
|
return int((monots[-1] - monots[0])/(n-1)) if n > 1 else 8_333_333
|
|
|
|
def anchor_phase(monots, period):
|
|
"""Circular mean of present phases (handles 0/P wrap and bimodal
|
|
wake-source phases; the anchor stays <P/2 from every present as long
|
|
as the modes are <P apart)."""
|
|
sx = sy = 0.0
|
|
for m in monots[:2000]:
|
|
q = (m % period)/period * 2*math.pi
|
|
sx += math.cos(q); sy += math.sin(q)
|
|
ang = math.atan2(sy, sx)
|
|
if ang < 0: ang += 2*math.pi
|
|
return int(ang/(2*math.pi)*period)
|
|
|
|
def slot_of(m, P, grid):
|
|
"""grid = ('edge', ftn0) or ('anchor', anchor). Returns (slot, phase_ns).
|
|
edge: phase in [0, P) from the vsync edge (hardware truth)
|
|
anchor: phase in (-P/2, P/2] centered on the anchor (fallback)"""
|
|
mode, ref = grid
|
|
if mode == "edge":
|
|
s = (m - ref)//P
|
|
return s, m - ref - s*P
|
|
x = (m - ref)/P
|
|
s = int(x + 0.5) if x >= 0 else -int(-x + 0.5)
|
|
return s, (m - ref) - s*P
|
|
|
|
def auto_windows(pres, period):
|
|
if not pres: return []
|
|
t0 = pres[0][1]
|
|
rate = 1000.0/(period/1e6) # display Hz (120 phone, 60 emulator)
|
|
hi, lo = 0.75*rate, 0.25*rate # active vs idle density thresholds
|
|
BUCK = 500_000_000
|
|
dens = defaultdict(int)
|
|
for f, m in pres:
|
|
dens[(m - t0)//BUCK] += 1
|
|
last = int((pres[-1][1] - t0)//BUCK)
|
|
wins, start = [], None
|
|
for b in range(last + 1):
|
|
r = dens.get(b, 0) * 2
|
|
if r > hi and start is None:
|
|
start = b
|
|
elif r < lo and start is not None:
|
|
wins.append((start, b)); start = None
|
|
if start is not None:
|
|
wins.append((start, last + 1))
|
|
out = []
|
|
for i, (b0, b1) in enumerate(wins):
|
|
idx0 = next((k for k, (f, m) in enumerate(pres) if (m - t0)//BUCK >= b0), None)
|
|
idx1 = len(pres)
|
|
for k in range(len(pres) - 1, -1, -1):
|
|
if (pres[k][1] - t0)//BUCK < b1:
|
|
idx1 = k + 1
|
|
break
|
|
if idx0 is not None:
|
|
out.append((f"w{i}", idx0, idx1))
|
|
return out
|
|
|
|
def explicit_windows(pres, t0, spec):
|
|
out = []
|
|
for part in spec.split(";"):
|
|
part = part.strip()
|
|
if not part: continue
|
|
rng, _, name = part.partition(" ")
|
|
a, b = rng.split(",")
|
|
a, b = int(a)*1_000_000, int(b)*1_000_000
|
|
idx0 = next((k for k, (f, m) in enumerate(pres) if m >= t0 + a), None)
|
|
idx1 = len(pres)
|
|
for k in range(len(pres) - 1, -1, -1):
|
|
if pres[k][1] <= t0 + b:
|
|
idx1 = k + 1
|
|
break
|
|
if idx0 is not None:
|
|
out.append((name or f"w{len(out)}", idx0, idx1))
|
|
return out
|
|
|
|
def evaluate(pres, P, grid, windows):
|
|
t0 = pres[0][1]
|
|
any_fail = False
|
|
total = 0
|
|
for name, i0, i1 in windows:
|
|
seg = pres[i0:i1]
|
|
total += len(seg)
|
|
if len(seg) < 10:
|
|
print(f"\n=== window '{name}' n={len(seg)} -> SKIP (short)")
|
|
continue
|
|
drops = doubles = lates = stalls = jit = 0
|
|
drop_at, stall_at, late_at, double_at = [], [], [], []
|
|
prev_s = prev_p = None
|
|
phases = []
|
|
for k in range(len(seg)):
|
|
f, m = seg[k]
|
|
s, p = slot_of(m, P, grid)
|
|
phases.append(p)
|
|
if prev_s is not None:
|
|
gap = s - prev_s
|
|
if gap >= 2:
|
|
drops += gap - 1
|
|
drop_at.append((k, gap, (m - seg[k-1][1])/1e6))
|
|
elif gap == 0:
|
|
doubles += 1
|
|
double_at.append(k)
|
|
if p > 5_000_000:
|
|
lates += 1
|
|
late_at.append((k, p/1e6))
|
|
if k and (m - seg[k-1][1]) > 20_000_000:
|
|
stalls += 1
|
|
stall_at.append((k, (m - seg[k-1][1])/1e6))
|
|
if prev_p is not None and abs(p - prev_p) > 500_000:
|
|
jit += 1
|
|
prev_s, prev_p = s, p
|
|
ps = sorted(phases)
|
|
spread = (ps[int(len(ps)*9)//10] - ps[len(ps)//10])/1e6
|
|
ok = drops == 0 and doubles == 0 and lates == 0 and stalls == 0 and spread <= 0.5
|
|
any_fail = any_fail or not ok
|
|
print(f"\n=== window '{name}' n={len(seg)} "
|
|
f"[{(seg[0][1]-t0)/1e6:.0f}..{(seg[-1][1]-t0)/1e6:.0f} ms] -> "
|
|
f"{'PASS' if ok else 'FAIL'}")
|
|
print(f" dropped={drops} doubles={doubles} late={lates} stalls={stalls} "
|
|
f"phase p10-p90={spread:.2f} ms phase-jumps>0.5ms={jit}")
|
|
for k, gap, ms in drop_at[:5]:
|
|
print(f" DROP #{k}: skipped {gap-1} slot(s), gap {ms:.1f} ms")
|
|
for k, ms in stall_at[:5]:
|
|
print(f" STALL #{k}: {ms:.1f} ms")
|
|
for k, ms in late_at[:5]:
|
|
print(f" LATE #{k}: phase +{ms:.2f} ms")
|
|
for k in double_at[:5]:
|
|
print(f" DOUBLE #{k}")
|
|
print(f"\n=== SUMMARY windows={len(windows)} frames={total} -> "
|
|
+ ("PASS" if not any_fail and windows else "FAIL"))
|
|
return (not any_fail) and bool(windows)
|
|
|
|
# ---------------------------------------------------------------- self-test
|
|
def selftest():
|
|
"""Synthesize LOG FILES (VSYNC ftn + PRES lines) and run the same
|
|
run_file() path as production. Cases must be classified exactly."""
|
|
P = 8_333_333
|
|
T0 = 100_000_000
|
|
def gen(name, off, skips=(), doubles=(), stall_after=None, n=600):
|
|
lines, i = [], 0
|
|
t = T0
|
|
for i in range(n):
|
|
if i in skips:
|
|
t += P
|
|
continue
|
|
lines.append(f"VSYNC ftn={t} mon={t+200_000}")
|
|
m = t + off(i)
|
|
lines.append(f"PRES f={i} mon={m}")
|
|
if i in doubles:
|
|
lines.append(f"PRES f={i}b mon={m+300_000}")
|
|
t += P
|
|
if stall_after is not None and i == stall_after:
|
|
t += 120_000_000
|
|
path = tempfile.NamedTemporaryFile("w", suffix=".log", delete=False)
|
|
path.write("\n".join(lines))
|
|
path.close()
|
|
return path.name
|
|
|
|
cases = [
|
|
("perfect 120Hz +1ms", lambda i: 1_000_000, {}, set(), None, True,
|
|
dict(drops=0, doubles=0, lates=0, stalls=0, max_spread=0.2)),
|
|
("3-slot drop at 200", lambda i: 1_000_000, {200, 201, 202}, set(), None, False,
|
|
dict(drops=3, doubles=0, lates=0, stalls=1, max_spread=99)),
|
|
("double at 300", lambda i: 1_000_000, {}, {300}, None, False,
|
|
dict(drops=0, doubles=1, lates=0, stalls=0, max_spread=0.2)),
|
|
("late i%7==0 +6ms (past latch)", lambda i: 6_000_000 if i % 7 == 0 else 1_000_000,
|
|
{}, set(), None, False,
|
|
dict(drops=0, doubles=0, lates=1, stalls=0, max_spread=99)),
|
|
("120ms stall at 400", lambda i: 1_000_000, {}, set(), 400, False,
|
|
dict(drops=14, doubles=0, lates=0, stalls=1, max_spread=99)),
|
|
("phase swing +/-.6ms", lambda i: 1_000_000 + (i % 5 - 2)*300_000,
|
|
{}, set(), None, False,
|
|
dict(drops=0, doubles=0, lates=0, stalls=0, max_spread=1.25)),
|
|
("tiny jitter 0.3ms", lambda i: 1_000_000 + (i % 3 - 1)*150_000,
|
|
{}, set(), None, True,
|
|
dict(drops=0, doubles=0, lates=0, stalls=0, max_spread=0.31)),
|
|
]
|
|
ok_all = True
|
|
for name, off, skips, dbl, stall_after, expect_pass, exp in cases:
|
|
path = gen(name, off, skips, dbl, stall_after)
|
|
import io, contextlib
|
|
buf = io.StringIO()
|
|
with contextlib.redirect_stdout(buf):
|
|
res = run_file(path, period_ms=None, windows_spec="")
|
|
os.unlink(path)
|
|
txt = buf.getvalue()
|
|
m = re.search(r"dropped=(\d+) doubles=(\d+) late=(\d+) stalls=(\d+).*p10-p90=([\d.]+)", txt)
|
|
if not m:
|
|
print(f" selftest {name:32s} *** could not parse report ***\n{txt}")
|
|
ok_all = False
|
|
continue
|
|
d, db, la, st, sp = int(m.group(1)), int(m.group(2)), int(m.group(3)), int(m.group(4)), float(m.group(5))
|
|
good = (res == expect_pass) and d >= exp["drops"] and db >= exp["doubles"] \
|
|
and la >= exp["lates"] and st >= exp["stalls"] and sp <= exp["max_spread"]
|
|
ok_all = ok_all and good
|
|
print(f" selftest {name:32s} expect={'PASS' if expect_pass else 'FAIL':4s} "
|
|
f"got={'PASS' if res else 'FAIL':4s} (drops={d} dbl={db} late={la} stalls={st} "
|
|
f"spread={sp:.2f}) {'OK' if good else '*** MISMATCH ***'}")
|
|
print("self-test:", "PASS" if ok_all else "FAIL")
|
|
return 0 if ok_all else 1
|
|
|
|
def run_file(path, period_ms, windows_spec):
|
|
lines = open(path, errors="replace").read().splitlines()
|
|
pres, ftns = parse_log(lines)
|
|
if len(pres) < 50:
|
|
print("no usable PRES data"); return False
|
|
monots = [m for _, m in pres]
|
|
src, P = None, None
|
|
if ftns:
|
|
P = vsync_period(ftns)
|
|
if P: src = "vsync-edges"
|
|
if period_ms:
|
|
P, src = int(period_ms*1e6), "override"
|
|
if not P:
|
|
P, src = fallback_period(monots), "fallback-fit"
|
|
grid = ("edge", ftns[0]) if (src == "vsync-edges" or (period_ms and ftns)) else \
|
|
("anchor", anchor_phase(monots, P))
|
|
t0 = monots[0]
|
|
print(f"display period: {P/1e6:.4f} ms (rate {1e6/(P/1e6):.1f} Hz, {src}; "
|
|
f"grid={'vsync-edge' if grid[0]=='edge' else 'anchor-fallback'})")
|
|
wins = explicit_windows(pres, t0, windows_spec) if windows_spec else auto_windows(pres, P)
|
|
if not wins:
|
|
print("no active windows found"); return False
|
|
return evaluate(pres, P, grid, wins)
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("file", nargs="?")
|
|
ap.add_argument("--windows", default="", help='"startMs,endMs name; ..." (ms from first present)')
|
|
ap.add_argument("--period-ms", type=float, default=None,
|
|
help="force the display period (phone: 8.3333)")
|
|
ap.add_argument("--selftest", action="store_true")
|
|
args = ap.parse_args()
|
|
if args.selftest:
|
|
sys.exit(selftest())
|
|
if not args.file:
|
|
ap.print_usage(); sys.exit(2)
|
|
ok = run_file(args.file, args.period_ms, args.windows)
|
|
sys.exit(0 if ok else 1)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|