// bench runs the corridor stop-optimization benchmark (bench/tasks.json) // against a router backend. It is the acceptance test for the // "search along the route" primitives: route, stop_cost, optimize_stops. // // Usage: // // bench [--tasks bench/tasks.json] [--record] // // --record writes observed golden values (direct route minutes and // optimized orders) back into the tasks file for regression use. package main import ( "context" "encoding/json" "flag" "fmt" "math" "os" "strings" "time" "maps/router/internal/geo" "maps/router/internal/plan" "maps/router/internal/route" ) type Place struct { Name string `json:"name"` Lat float64 `json:"lat"` Lon float64 `json:"lon"` } func (p Place) Point() geo.Point { return geo.Point{Lat: p.Lat, Lon: p.Lon} } type Candidate struct { ID string `json:"id"` Name string `json:"name"` Lat float64 `json:"lat"` Lon float64 `json:"lon"` DwellMin int `json:"dwellMin"` } type Check struct { Type string `json:"type"` ID string `json:"id"` Name string `json:"name"` Lat, Lon float64 `json:",omitempty"` MinDistKm float64 `json:"minDistKm,omitempty"` Candidate string `json:"candidate,omitempty"` MaxDetourMin int `json:"maxDetourMin,omitempty"` MinDetourMin int `json:"minDetourMin,omitempty"` MaxPct float64 `json:"maxPct,omitempty"` GoldMin float64 `json:"goldMin,omitempty"` GoldMax float64 `json:"goldMax,omitempty"` Want []string `json:"want,omitempty"` } type Task struct { ID string `json:"id"` Desc string `json:"desc"` Origin Place `json:"origin"` Dest Place `json:"dest"` K int `json:"k"` BudgetMin int `json:"budgetMin"` Candidates []Candidate `json:"candidates"` Checks []Check `json:"checks"` } type BenchFile struct { Backend string `json:"backend"` ValhallaURL string `json:"valhalla_url"` Tasks []Task `json:"tasks"` } type result struct { check string status string // PASS FAIL RECORD detail string } func main() { tasksPath := flag.String("tasks", "bench/tasks.json", "path to tasks.json") record := flag.Bool("record", false, "record golden values into the tasks file") flag.Parse() raw, err := os.ReadFile(*tasksPath) if err != nil { fatal(err) } var bf BenchFile if err := json.Unmarshal(raw, &bf); err != nil { fatal(err) } var r route.Router switch bf.Backend { case "valhalla": r = route.NewValhalla(bf.ValhallaURL) case "osrm": r = route.NewOSRM("http://localhost:5000") default: fatal(fmt.Errorf("unknown backend %q", bf.Backend)) } ctx := context.Background() delay := 150 * time.Millisecond if _, ok := r.(*route.OSRM); ok { delay = 0 } failures := 0 for ti := range bf.Tasks { t := &bf.Tasks[ti] fmt.Printf("=== %s: %s -> %s (k=%d)\n", t.ID, t.Origin.Name, t.Dest.Name, t.K) fmt.Printf(" %s\n", t.Desc) // Direct route (golden + corridor source). direct, err := r.Route(ctx, route.ProfileDrive, []geo.Point{t.Origin.Point(), t.Dest.Point()}) if err != nil { fatal(fmt.Errorf("task %s: direct route: %w", t.ID, err)) } fmt.Printf(" direct: %s (%.1f km)\n", fmtDur(direct.Duration), direct.Distance/1000) // Matrix over [candidates..., origin, dest]. pts := make([]geo.Point, 0, len(t.Candidates)+2) for _, c := range t.Candidates { pts = append(pts, geo.Point{Lat: c.Lat, Lon: c.Lon}) } pts = append(pts, t.Origin.Point(), t.Dest.Point()) fmt.Printf(" building %dx%d matrix (%d calls)...\n", len(pts), len(pts), len(pts)*(len(pts)-1)) t0 := time.Now() m, err := route.BuildMatrix(ctx, r, route.ProfileDrive, pts, delay) if err != nil { fatal(fmt.Errorf("task %s: matrix: %w", t.ID, err)) } fmt.Printf(" matrix done in %s\n", time.Since(t0).Round(time.Second)) stops := make([]plan.Stop, len(t.Candidates)) for i, c := range t.Candidates { stops[i] = plan.Stop{ID: c.ID, Name: c.Name, At: geo.Point{Lat: c.Lat, Lon: c.Lon}, DwellMin: c.DwellMin} } candByID := map[string]plan.Stop{} for _, s := range stops { candByID[s.ID] = s } opt := plan.OptimizeStops(m, stops, t.K, t.BudgetMin) optNames := make([]string, len(opt.Order)) for i, si := range opt.Order { optNames[i] = stops[si].ID } fmt.Printf(" optimized: %s (detour %d min, total %d min, exhaustive=%v)\n", strings.Join(optNames, " -> "), opt.DetourMin, opt.TotalMin, opt.Exhaustive) for ci := range t.Checks { ck := &t.Checks[ci] res := runCheck(ctx, r, t, m, stops, direct, opt, *ck) if res.status == "FAIL" { failures++ } fmt.Printf(" [%s] %-22s %s\n", res.status, ck.ID, res.detail) if *record { applyRecord(ck, direct, optNames) } } } if *record { out, _ := json.MarshalIndent(bf, "", " ") if err := os.WriteFile(*tasksPath, out, 0o644); err != nil { fatal(err) } fmt.Printf("\nrecorded goldens into %s\n", *tasksPath) } fmt.Printf("\n%s (%d failures)\n", outcome(failures), failures) if failures > 0 { os.Exit(1) } } func runCheck(ctx context.Context, r route.Router, t *Task, m route.Matrix, stops []plan.Stop, direct *route.Route, opt plan.OptimizeResult, ck Check) result { res := result{check: ck.ID} switch ck.Type { case "golden_route_min": mins := direct.Duration / 60 if ck.GoldMin == 0 && ck.GoldMax == 0 { res.status = "RECORD" res.detail = fmt.Sprintf("direct = %.0f min (no golden yet, run --record)", mins) return res } lo, hi := ck.GoldMin, ck.GoldMax if lo == 0 { lo = mins - math.Inf(1) } if hi == 0 { hi = math.Inf(1) } if mins < lo || mins > hi { res.status = "FAIL" res.detail = fmt.Sprintf("direct = %.0f min, golden [%v, %v]", mins, lo, hi) } else { res.status = "PASS" res.detail = fmt.Sprintf("direct = %.0f min in [%v, %v]", mins, lo, hi) } case "route_avoids": p := geo.Point{Lat: ck.Lat, Lon: ck.Lon} dist := distFromRoute(direct, p) if dist < ck.MinDistKm*1000 { res.status = "FAIL" res.detail = fmt.Sprintf("%s: route comes within %.1f km (< %.0f km)", ck.Name, dist/1000, ck.MinDistKm) } else { res.status = "PASS" res.detail = fmt.Sprintf("%s: route stays %.0f km away (min %.0f)", ck.Name, dist/1000, ck.MinDistKm) } case "stop_free": c, ok := stopIdx(stops, ck.Candidate) if !ok { res.status, res.detail = "FAIL", "unknown candidate" return res } cost := plan.StopCost(m, c, stops[c]) if cost.DetourMin > ck.MaxDetourMin { res.status = "FAIL" res.detail = fmt.Sprintf("detour = %d min > %d min cap", cost.DetourMin, ck.MaxDetourMin) } else { res.status = "PASS" res.detail = fmt.Sprintf("detour = %d min <= %d min", cost.DetourMin, ck.MaxDetourMin) } case "stop_detour_at_least": c, ok := stopIdx(stops, ck.Candidate) if !ok { res.status, res.detail = "FAIL", "unknown candidate" return res } cost := plan.StopCost(m, c, stops[c]) if cost.DetourMin < ck.MinDetourMin { res.status = "FAIL" res.detail = fmt.Sprintf("detour = %d min < %d min floor", cost.DetourMin, ck.MinDetourMin) } else { res.status = "PASS" res.detail = fmt.Sprintf("detour = %d min >= %d min", cost.DetourMin, ck.MinDetourMin) } case "stop_detour_band": // The computed detour must fall in [min, max] — a regression // guard on corridor shape (router/extract changes would show here). c, ok := stopIdx(stops, ck.Candidate) if !ok { res.status, res.detail = "FAIL", "unknown candidate" return res } cost := plan.StopCost(m, c, stops[c]) if cost.DetourMin < ck.MinDetourMin || cost.DetourMin > ck.MaxDetourMin { res.status = "FAIL" res.detail = fmt.Sprintf("detour = %d min outside [%d, %d]", cost.DetourMin, ck.MinDetourMin, ck.MaxDetourMin) } else { res.status = "PASS" res.detail = fmt.Sprintf("detour = %d min in [%d, %d]", cost.DetourMin, ck.MinDetourMin, ck.MaxDetourMin) } case "optimize_order": got := make([]string, len(opt.Order)) for i, si := range opt.Order { got[i] = stops[si].ID } if ck.Want == nil { res.status = "RECORD" res.detail = fmt.Sprintf("order = %s (no golden yet)", strings.Join(got, " -> ")) return res } if !sameSeq(got, ck.Want) { res.status = "FAIL" res.detail = fmt.Sprintf("got %s, want %s", strings.Join(got, " -> "), strings.Join(ck.Want, " -> ")) } else { res.status = "PASS" res.detail = fmt.Sprintf("order = %s", strings.Join(got, " -> ")) } case "crosscheck": // Re-route in the optimized order as a single call; compare // travel time against the matrix-summed prediction. if len(opt.Order) == 0 { res.status, res.detail = "PASS", "no stops, nothing to cross-check" return res } pts := []geo.Point{t.Origin.Point()} for _, si := range opt.Order { pts = append(pts, stops[si].At) } pts = append(pts, t.Dest.Point()) rr, err := r.Route(ctx, route.ProfileDrive, pts) if err != nil { res.status, res.detail = "FAIL", "cross-check route: "+err.Error() return res } // matrix-predicted travel time (no dwell): sum legs n := len(stops) via := 0.0 prev := n // origin index in matrix for _, si := range opt.Order { via += m[prev][si] prev = si } via += m[prev][n+1] diffPct := math.Abs(via-rr.Duration) / rr.Duration * 100 if diffPct > ck.MaxPct { res.status = "FAIL" res.detail = fmt.Sprintf("matrix %.0f s vs route %.0f s (%.1f%% drift > %.0f%%)", via, rr.Duration, diffPct, ck.MaxPct) } else { res.status = "PASS" res.detail = fmt.Sprintf("matrix %.0f s vs route %.0f s (%.1f%% drift)", via, rr.Duration, diffPct) } default: res.status, res.detail = "FAIL", "unknown check type "+ck.Type } return res } // applyRecord fills in goldens. func applyRecord(ck *Check, direct *route.Route, optNames []string) { mins := direct.Duration / 60 switch ck.Type { case "golden_route_min": // +/- 10% tolerance band around the observed value. ck.GoldMin = math.Floor(mins*0.9/5) * 5 ck.GoldMax = math.Ceil(mins*1.1/5) * 5 case "optimize_order": ck.Want = append([]string(nil), optNames...) } } // distFromRoute returns the min distance from p to the route: geometry // if available, else bbox (conservative: inside bbox => 0). func distFromRoute(r *route.Route, p geo.Point) float64 { if len(r.Geometry) >= 2 { return geo.DistToPolylineMeters(p, r.Geometry) } if !r.BBox.Empty() { return r.BBox.DistMeters(p) } return 0 } func stopIdx(stops []plan.Stop, id string) (int, bool) { for i, s := range stops { if s.ID == id { return i, true } } return 0, false } func sameSeq(a, b []string) bool { if len(a) != len(b) { return false } for i := range a { if a[i] != b[i] { return false } } return true } func fmtDur(sec float64) string { m := int(0.5 + sec/60) return fmt.Sprintf("%dh%02dm", m/60, m%60) } func outcome(failures int) string { if failures == 0 { return "BENCH PASS" } return "BENCH FAIL" } func fatal(err error) { fmt.Fprintln(os.Stderr, "bench error:", err) os.Exit(2) }