Live web enrichment (SearXNG) + PostGIS spatial stack

Milestone 1 — live web search:
- mock/server.js: /search + /search-status proxy to the self-hosted
  SearXNG (third leg of the 3-source blend; engine endpoint stays
  server-side), /spatial + /spatial-status proxy to the spatial service
- mock/app.js: background enrichment of suggestion cards — each
  candidate gets a web element (discovery cards + "you might like"
  rows) that is checked via SearXNG in the background (one in-flight
  query per place, 1 h TTL, failures retry in ~5 min) and APPENDS
  sourced facts with provenance URLs; never reorders/rewrites the plan.
  New LLM tools: web_search (cite URLs) and poi_near (PostGIS). Status
  badge gains 'web' / 'spatial'.
- styles.css: .cc-web live-facts blocks, t-web verified chip

Milestone 2 — PostGIS spatial DB (code complete; needs docker access
to run — see spatial/README.md):
- spatial/: docker-compose (postgis/postgis:16-3.4 + osm2pgsql:16
  one-shot importer + spatiald), schema.sql (poi table w/ GIST index,
  spatial_extract bookkeeping, refresh_poi()), import.sh for the PBF
  extracts in ../osm, spatiald (Go, lib/pq): /health, /near
  (ST_DWithin), /nearest (KNN), /corridor (ST_Buffer along a route),
  with kind/name/extract/limit filters
This commit is contained in:
Greg Pomerantz 2026-09-10 09:43:30 -04:00
parent ca32d20a50
commit c367eba5cb
12 changed files with 921 additions and 3 deletions

1
.gitignore vendored
View File

@ -8,6 +8,7 @@ osm/build/
# Go
router/router
spatial/spatiald
router/bench
router/scripts/__pycache__/

View File

@ -51,7 +51,82 @@ async function checkLLM() {
try { const r = await fetch('/llm-status'); const j = await r.json(); llmOn = !!j.llm; }
catch { llmOn = false; }
}
const badgeText = () => '📶 online' + (routerOn ? ' · router' : '') + (llmOn ? ' · llm' : '');
// ---------------- web search client (SearXNG via the /search proxy) ------
// Live facts (opening hours, fees, seasonal notes) with provenance URLs —
// the third leg of the 3-source blend. Enrichment APPENDS to suggestion
// cards; it never reorders, removes, or rewrites the plan (DESIGN.md
// "streaming enrichment").
let webOn = false;
async function checkWeb() {
try { const r = await fetch('/search-status'); webOn = !!(await r.json()).web; }
catch { webOn = false; }
}
// ---------------- spatial client (PostGIS spatiald via /spatial proxy) --
let spatialOn = false, spatialPoi = 0;
async function checkSpatial() {
try { const r = await fetch('/spatial-status'); const j = await r.json(); spatialOn = !!j.spatial; spatialPoi = j.poi_count || 0; }
catch { spatialOn = false; }
}
const badgeText = () => '📶 online' + (routerOn ? ' · router' : '') + (llmOn ? ' · llm' : '') + (webOn ? ' · web' : '') + (spatialOn ? ' · spatial' : '');
const WEB_TTL = 60 * 60 * 1000; // a cached live check is good for an hour
const tripCity = () => (M?.trip?.places?.[0]?.name) || (M?.trip?.title || '').split(' ')[0] || '';
const webQuery = (c, cat) => {
const tail = cat === 'hotel' ? 'price per night reviews'
: ['lunch', 'dinner', 'breakfast', 'snack'].includes(cat) ? 'menu prices opening hours'
: 'opening hours entry fee';
return `"${c.name}" ${tripCity()} ${tail}`;
};
// Suggestion surfaces (discovery cards, "you might like" rows) embed a web
// element via webEl(c, cat); it is re-rendered on every refresh, so results
// are kept on the candidate (c.web) and every live node for that place is
// refreshed via the data-webfor attribute.
const webBusy = new Set();
function webEl(c, cat) {
const n = el('div', 'cc-web');
n.dataset.webfor = c.id;
n.dataset.webcat = cat || '';
refreshWebEl(n, c);
return n;
}
function refreshWebEl(n, c) {
if (!n || !n.isConnected) return;
if (c.web) {
n.classList.remove('busy');
if (c.web.failed) { n.innerHTML = '🌐 live check unavailable (retry in a few minutes)'; return; }
const rs = (c.web.results || []).filter(r => r.url);
if (!rs.length) { n.innerHTML = '🌐 no live results'; return; }
n.innerHTML = rs.slice(0, 3).map(r => {
const host = (() => { try { return new URL(r.url).hostname.replace(/^www\./, ''); } catch { return r.source || 'web'; } })();
const snip = (r.snippet || r.title || '').replace(/Missing:.*$/i, '').trim();
return `<span class="web-src"><a href="${esc(r.url)}" target="_blank" rel="noopener">${esc(host)}</a></span>` +
`<span class="web-snip">${esc(snip.slice(0, 150))}${snip.length > 150 ? '…' : ''}</span>`;
}).join('<br>');
} else if (webBusy.has(c.id)) {
n.classList.add('busy');
n.innerHTML = '🌐 checking <b>' + esc(c.name) + '</b>…';
} else n.innerHTML = '';
}
// fire-and-forget: at most one in-flight query per place, cached for WEB_TTL
async function enrichWeb(c, cat) {
if (!webOn || !c) return;
if (c.web && Date.now() - c.web.at < WEB_TTL) return;
if (webBusy.has(c.id)) return;
webBusy.add(c.id);
document.querySelectorAll(`[data-webfor="${c.id}"]`).forEach(n => refreshWebEl(n, c));
try {
const r = await fetch('/search?q=' + encodeURIComponent(webQuery(c, cat)) + '&count=3');
const j = await r.json();
if (!j.ok) throw new Error(j.error || 'search failed');
c.web = { q: webQuery(c, cat), at: Date.now(), results: j.results || [] };
} catch {
// remember the failure so live nodes can show it, but age it out fast
// (retry after ~5 min, not after the full TTL)
c.web = { q: webQuery(c, cat), at: Date.now() - (WEB_TTL - 5 * 60 * 1000), results: [], failed: true };
} finally {
webBusy.delete(c.id);
document.querySelectorAll(`[data-webfor="${c.id}"]`).forEach(n => refreshWebEl(n, c));
}
}
const esc = s => String(s).replace(/[&<>"']/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' }[c]));
// Compact itinerary snapshot (the design doc's LLM context form): stops with
// times + stays + bookings. No geometry, no tool output. layout() fills
@ -311,6 +386,36 @@ const TOOLS = {
walk_from_hotel_min: hotelWalkMin(p.at), url: c?.url || null, summary: c?.detail?.summary || null };
},
// LIVE web search (SearXNG) — for anything that can change between the
// data snapshot and the trip: opening hours, fees, seasonal events, prices.
// Results carry provenance URLs; the model must cite the ones it used.
web_search: async (a) => {
const q = String(a.query || '').trim();
if (!q) return { error: 'query is required' };
if (!webOn) return { error: 'the web search backend is offline — say so and fall back to what the plan already knows' };
const r = await fetch('/search?q=' + encodeURIComponent(q) + '&count=5' + (a.categories ? '&categories=' + encodeURIComponent(a.categories) : ''));
const j = await r.json();
if (!j.ok) return { error: j.error || 'search failed' };
return { count: j.count, results: (j.results || []).map(x => ({ title: x.title, url: x.url, source: x.source, snippet: x.snippet.slice(0, 300) })) };
},
// POI lookup against the local PostGIS extract (native ST_DWithin / KNN /
// corridor queries — real OSM data, not the curated demo pools).
poi_near: async (a) => {
if (!spatialOn) return { error: 'the spatial database is offline — use search_places instead' };
const lat = +a.lat, lng = +a.lng, r = Math.min(50000, Math.max(100, +a.radius_m || 500));
if (!isFinite(lat) || !isFinite(lng)) return { error: 'lat/lng must be numbers (lat, lng in degrees)' };
let qs = `lat=${lat}&lng=${lng}&r=${r}`;
if (a.kind) qs += '&kind=' + encodeURIComponent(a.kind); // e.g. amenity=restaurant, tourism=museum
if (a.name) qs += '&name=' + encodeURIComponent(a.name);
if (a.corridor) qs += '&corridor=' + encodeURIComponent(a.corridor); // 'lng,lat;lng,lat;…' route polyline
const rj = await fetch('/spatial/' + (a.corridor ? 'corridor' : 'near') + '?' + qs);
const j = await rj.json();
if (j.error) return { error: j.error };
return { count: (j.results || []).length, results: (j.results || []).map(x =>
({ name: x.name, kind: x.kind, dist_m: Math.round(x.dist_m), opening_hours: x.opening_hours || null, fee: x.fee || null, website: x.website || null })) };
},
route_between: async (a) => {
const A = resolvePlace(a.from), B = resolvePlace(a.to);
if (!A) return { error: `cannot resolve "from" = "${a.from}"` };
@ -535,6 +640,8 @@ function llmSystemPrompt() {
' itinerary_summary() — the current plan: bookings, stays, and each day\'s scheduled stops with times.\n' +
' search_places(category, region?) — candidate places. category ∈ {' + cats.join(', ') + '} ; region ∈ {' + regions.join(', ') + '} or omit. Returns id, name, price_eur, dur_min, tags, pitch, walk_from_hotel_min.\n' +
' place_facts(id) — full facts for one place (id or name): price, duration, tags, pitch, walk from hotel, summary.\n' +
' web_search(query) — LIVE web search. Use for anything that can change since the data snapshot: opening hours, entry fees, seasonal events, current prices, whether a place still exists. Returns results with source URLs — always cite the URL(s) you relied on in your answer.\n' +
' poi_near(lat, lng, radius_m?, kind?, name?) — real POIs from the local OSM/PostGIS extract (e.g. kind "amenity=restaurant"). Use when the curated pools are empty or the user asks "what is around X".\n' +
' route_between(from, to) — real walking minutes + km between two places ("hotel" or a place id/name).\n' +
' review_plan(dayId?) — deterministic checks: day overrun vs the waking window, meals at implausible hours, duplicates. dayId optional.\n' +
' add_stop(dayId, slot, candidateId, state?) — put a candidate on a day. slot ∈ lunch,dinner,visit. If the slot is taken the current stop is demoted to a backup (a swap). state ∈ planned,idea,backup (default planned). candidateId comes from search_places.\n' +
@ -546,7 +653,7 @@ function llmSystemPrompt() {
'Guidelines:\n' +
'- dayId is the day id like "D1" or "D2" — shown in brackets in the plan rows, e.g. [D1 · 2026-09-12]. Never use the calendar date as a dayId.\n' +
'- Answer concisely (14 sentences), plain text, no markdown.\n' +
'- For any factual question about times, distances, or prices, CALL the tool — do not guess.\n' +
'- For any factual question about times, distances, or prices, CALL the tool — do not guess. For CURRENT facts ("is it open?", "how much now", seasonal), prefer web_search and cite its source URLs.\n' +
'- For a requested change, call the matching fix-op, then confirm in one sentence what changed. The user can undo any edit.\n' +
'- Before a big rework, consider review_plan() and mention any issues it reports.\n' +
'- Be efficient: call each tool at most once unless you need to, and never re-fetch facts you already have. A fix-op alone is enough — do not re-search or re-fetch the stop you are moving/editing.\n' +
@ -1097,6 +1204,8 @@ function candCard(c, anchor, isHotel) {
chips.append(el('span', 'pchip', c.price ? '~' + c.price + '€ / ' + (isHotel ? 'night' : 'person') : 'free'));
card.append(chips);
if (c.tags?.length) card.append(el('div', 'cc-tags', c.tags.map(t => '#' + t).join(' ')));
card.append(webEl(c, isHotel ? 'hotel' : disc.cat)); // live web facts + provenance (background)
enrichWeb(c, isHotel ? 'hotel' : disc.cat);
const acts = el('div', 'cc-acts');
const pick = el('button', 'btn primary small', isHotel ? 'Make this the hotel' : 'Choose this');
pick.onclick = () => chooseCand(c);
@ -1616,6 +1725,8 @@ function renderSugg() {
row.append(el('span', 'sr-name', c.name));
row.append(el('span', 'pchip', `${walkMin(a.at, c.at)} min`));
row.append(el('span', 'pchip', c.price ? '~' + c.price + '€' : 'free'));
const wn = webEl(c, slot); wn.classList.add('cc-web-sugg'); row.append(wn);
enrichWeb(c, slot);
const cmp = el('button', 'btn ghost tiny', '⚖');
cmp.title = 'Compare';
cmp.onclick = () => toggleCompare(itemFromCand(c, slot));
@ -2366,10 +2477,11 @@ function enterTrip(id) {
renderHotelMks(); renderModeToggle();
renderDocChips();
// this trip may live on a different OSRM extract — re-check before routing
Promise.all([checkRouter(), checkLLM()]).then(() => {
Promise.all([checkRouter(), checkLLM(), checkWeb(), checkSpatial()]).then(() => {
const b = $('#offline-badge');
if (b && offlineState !== 'ready') b.textContent = badgeText();
if (routerOn) enrichLegs(day);
if (webOn && $('#sugg-block')) renderSugg(); // web came up late → enrich the suggestion rows now
});
if (!day.legs.length) rebuildLegs(day); else enrichLegs(day);
renderAll();

View File

@ -36,6 +36,12 @@ const LLM = {
ctx: 131072,
maxTokens: +(process.env.LLM_MAX_TOKENS || 8192), // thinking + answer share this budget
};
// Web search backend: self-hosted SearXNG (JSON API). The UI only ever sends
// questions; the engine endpoint stays server-side. This is the third leg of
// the DESIGN.md 3-source blend (OSM ground truth → LLM knowledge → live web).
const SEARXNG = { base: process.env.SEARXNG_BASE || 'http://192.168.3.4:30053' };
// PostGIS spatial service (see ../spatial/): live once the import has run.
const SPATIAL = { url: process.env.SPATIAL_URL || 'http://localhost:5005' };
const routerFor = (u) => ROUTERS[u.searchParams.get('router') || 'northeast'] || ROUTERS.northeast;
const PROFILE = { foot: 'walking', walk: 'walking', bike: 'cycling', cycle: 'cycling', car: 'driving', drive: 'driving', transit: 'driving' };
const CACHE = path.join(__dirname, '.tilecache');
@ -371,6 +377,79 @@ http.createServer((req, res) => {
return;
}
// ---- web search (proxy -> SearXNG) --------------------------------
// GET /search-status — availability probe (HEAD on the root; it does NOT
// touch the upstream engines, unlike a real query)
if (u.pathname === '/search-status') {
fetch(SEARXNG.base + '/', { method: 'HEAD', signal: AbortSignal.timeout(4000) })
.then(r => {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Content-Type', 'application/json');
res.writeHead(200);
res.end(JSON.stringify({ web: r.ok, base: SEARXNG.base }));
})
.catch(() => {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Content-Type', 'application/json'); res.writeHead(200);
res.end(JSON.stringify({ web: false, base: SEARXNG.base }));
});
return;
}
// GET /search?q=…&categories=…&count=… — normalized result list with
// provenance (url + engine). SearXNG's JSON API wants format=json; the
// proxy caps results and strips the noise fields the UI doesn't render.
if (u.pathname === '/search') {
const q = u.searchParams.get('q') || '';
const count = Math.min(10, Math.max(1, +(u.searchParams.get('count') || 5)));
const cats = u.searchParams.get('categories') || 'general';
if (!q.trim()) { res.writeHead(400, { 'Content-Type': 'application/json' }); return res.end(JSON.stringify({ ok: false, error: 'q is required' })); }
const up = `${SEARXNG.base}/search?q=${encodeURIComponent(q)}&format=json&categories=${encodeURIComponent(cats)}&pageno=1`;
const t0 = Date.now();
fetch(up, { signal: AbortSignal.timeout(20000) }).then(async r => {
if (!r.ok) throw new Error('searxng http ' + r.status);
const j = await r.json();
const results = (j.results || []).slice(0, count).map(x => ({
title: String(x.title || ''), url: String(x.url || ''),
source: String(x.engine || ''), snippet: String(x.content || ''),
})).filter(x => x.url);
console.log(`[search] ${JSON.stringify(q).slice(0, 80)}${results.length} results in ${Date.now() - t0}ms`);
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Content-Type', 'application/json');
res.writeHead(200);
res.end(JSON.stringify({ ok: true, query: q, count: results.length, results }));
}).catch(e => {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Content-Type', 'application/json'); res.writeHead(200);
res.end(JSON.stringify({ ok: false, error: 'search failed: ' + e.message }));
});
return;
}
// ---- spatial backend (proxy -> PostGIS spatiald) -------------------
// /spatial-status probes /health; /spatial/<path> forwards verbatim
// (query string included) — /near, /nearest, /corridor.
if (u.pathname === '/spatial-status' || u.pathname.startsWith('/spatial/')) {
const up = u.pathname === '/spatial-status'
? SPATIAL.url + '/health'
: SPATIAL.url + u.pathname.slice('/spatial'.length); // keep query string
const isProbe = u.pathname === '/spatial-status';
fetch(up, { signal: AbortSignal.timeout(15000) }).then(async r => {
const t = await r.text();
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Content-Type', 'application/json');
res.writeHead(200);
if (!isProbe) return res.end(t); // forward the service body verbatim
let j = {}; try { j = JSON.parse(t); } catch {}
res.end(JSON.stringify({ spatial: r.ok && !!j.ok, url: SPATIAL.url, ...j }));
}).catch(() => {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Content-Type', 'application/json'); res.writeHead(200);
if (u.pathname === '/spatial-status') return res.end(JSON.stringify({ spatial: false, url: SPATIAL.url }));
res.end(JSON.stringify({ error: 'spatial backend unreachable at ' + SPATIAL.url }));
});
return;
}
// ---- document ingestion (flight/booking → text) --------------------
// POST /parse-doc — body { filename, mime, b64 }. Returns { ok, kind,
// name, text, chars, truncated }. PDFs go through pdftotext; emails (.eml

View File

@ -132,6 +132,17 @@ button { font: inherit; }
.t-search::before { content: "~"; font-weight: 800; }
.t-llm { background: transparent; border: 1.5px dashed #b9c0cc; color: var(--ink2); }
.t-llm::after { content: "est"; font-size: 9px; opacity: .8; }
.t-web { background: #e3f4e6; color: #1d7a35; }
.t-web::before { content: "✓"; font-weight: 800; }
/* live web facts on suggestion cards (background SearXNG enrichment) */
.cc-web { margin-top: 7px; font-size: 11px; color: var(--ink2); line-height: 1.5;
border-top: 1px dashed var(--line); padding-top: 6px; }
.cc-web:empty { display: none; }
.cc-web.busy { opacity: .75; font-style: italic; }
.cc-web .web-src a { color: #1d4f9c; font-weight: 700; text-decoration: none; }
.cc-web .web-src a:hover { text-decoration: underline; }
.cc-web .web-snip { display: block; margin: 1px 0 3px; }
.cc-web-sugg { border-top: none; padding-top: 2px; }
.pchip { font-size: 11.5px; color: var(--ink2); font-weight: 600; white-space: nowrap; }
.sug-note { font-size: 11px; color: var(--ink2); font-style: italic; }

12
spatial/Dockerfile Normal file
View File

@ -0,0 +1,12 @@
# spatiald — PostGIS query service (see main.go for endpoints)
FROM golang:1.27 AS build
WORKDIR /src
COPY go.mod go.sum ./
RUN go mod download
COPY main.go .
RUN CGO_ENABLED=0 go build -o /spatiald .
FROM debian:bookworm-slim
COPY --from=build /spatiald /usr/local/bin/spatiald
EXPOSE 5005
ENTRYPOINT ["spatiald"]

81
spatial/README.md Normal file
View File

@ -0,0 +1,81 @@
# spatial — PostGIS source of truth
Native spatial queries for the trip planner (DESIGN.md §Data: "PostGIS
(Postgres) — import PBF via osm2pgsql; gives SQL spatial queries (radius,
nearest, bbox)"). Replaces the flat-file/Overpass approximations.
## Stack
| Piece | What it is |
|---|---|
| `docker-compose.yml` | `postgis/postgis:16-3.4` (DB, port 5432) + one-shot `osm2pgsql:16` importer + `spatiald` (query service, port 5005) |
| `schema.sql` | runs on first boot: `poi` table (named POIs as points, GIST index), `spatial_extract` bookkeeping, `refresh_poi()` |
| `import.sh` | loads a PBF from `../osm/` via osm2pgsql, refreshes `poi` |
| `main.go` (spatiald) | JSON query service over `poi` |
| `Dockerfile` | builds spatiald |
## Prerequisites
* Docker with the current user in the `docker` group (on this box:
one-time `sudo usermod -aG docker $USER`, then re-login).
* PBF extracts in `../osm/` (already present: `nh.osm.pbf`,
`colombia.osm.pbf`, US state extracts).
## Usage
```bash
docker compose up -d postgis # first: runs schema.sql
./import.sh ../osm/nh.osm.pbf nh # load New England (default)
./import.sh ../osm/colombia.osm.pbf colombia
docker compose up -d spatiald # query service on :5005
```
The mock server (`mock/server.js`) proxies it: `GET /spatial/*``:5005`,
plus `GET /spatial-status` for the UI badge. When the service is up, the
chat assistant gains the `poi_near` tool automatically.
## spatiald endpoints
All return `{"count":N,"results":[{extract,osm_id,name,kind,opening_hours,
fee,website,addr_city,dist_m},…]}`. Common filters:
* `extract=nh` — one loaded extract
* `kind=amenity=restaurant` — exact kind, or `kind=restaurant` (any family)
* `name=café` — ILIKE substring
* `limit=20` (max 200)
| Endpoint | Query | SQL core |
|---|---|---|
| `GET /health` | — | extract bookkeeping |
| `GET /near` | `lat,lng,r(m; default 500)` | `ST_DWithin(geography, …, r)` + KNN ordering |
| `GET /nearest` | `lat,lng` | `ORDER BY geom <-> point` |
| `GET /corridor` | `points=lng,lat;…`, `r` | `ST_DWithin(geom::geography, ST_Buffer(line::geography, r))` |
Examples:
```bash
curl 'localhost:5005/near?lat=42.35&lng=-71.06&r=1000&kind=restaurant'
curl 'localhost:5005/nearest?lat=10.40&lng=-75.54&kind=tourism&limit=5'
curl 'localhost:5005/corridor?r=300&kind=fuel&points=-71.06,42.35;-71.07,42.36'
```
## Data notes
* `poi` covers every **named** node (or polygon way, via centroid) with an
`amenity`/`tourism`/`shop`/`leisure`/`historic`/`place` tag — the kinds the
planner asks about. Unnamed amenities (e.g. a nameless kiosk) are out of
scope for v1.
* `opening_hours`, `fee`, `website`, `addr_city` are carried straight from
the OSM tags; the live web enrichment (SearXNG, `mock/app.js`) fills gaps
the tags don't have (the 3-source blend).
* Multiple extracts coexist in one DB, disambiguated by `poi.extract`
(set automatically by `import.sh`).
* pgvector (semantic POI index, later milestone) gets its own init script +
embedding importer — the base image doesn't ship the `vector` package.
## Roadmap hooks
* `bbox` queries: trivial addition (`ST_Contains(ST_MakeEnvelope,…)`).
* Routing-graph join: OSRM/GraphHopper geometries can be loaded into the
same DB (`route_geom` table) for true along-route analytics instead of
the current buffer-over-polyline.

View File

@ -0,0 +1,60 @@
# PostGIS spatial stack for the trip planner.
#
# docker compose up -d postgis # start the DB (schema.sql runs on first boot)
# ./import.sh ../osm/nh.osm.pbf # import an OSM extract (one-shot osm2pgsql)
# docker compose run --rm spatiald # start the query service on :5005
#
# The PBF extracts live in ../osm (gitignored). osm2pgsql runs as a one-shot
# container so the host needs no osm2pgsql/osmium installation.
services:
postgis:
image: postgis/postgis:16-3.4
container_name: trips-postgis
environment:
POSTGRES_DB: trips
POSTGRES_USER: trips
POSTGRES_PASSWORD: trips
ports:
- "5432:5432"
volumes:
- pgdata:/var/lib/postgresql/data
# initdb scripts run in lexicographic order, once, on an empty volume
- ./schema.sql:/docker-entrypoint-initdb.d/10-schema.sql:ro
healthcheck:
test: ["CMD-SHELL", "pg_isready -U trips -d trips"]
interval: 5s
timeout: 3s
retries: 12
# One-shot importer. `docker compose run --rm importer` (or import.sh)
# loads the PBF named in PBF_FILE into the DB. The osm2pgsql image runs
# as uid 1001 and needs the PBF readable — import.sh bind-mounts ../osm.
importer:
image: osm2pgsql/osm2pgsql:16
user: "1001:1001"
network_mode: "service:postgis"
environment:
PBF_FILE: /data/nh.osm.pbf
# --create --clean: fresh load every time (extracts are small enough);
# slim mode keeps the RAM footprint sane; disable-operations saves time.
ARGS: >-
--host=postgis --port=5432 --database=trips --user=trips
--password=trips --create --clean --slim
--flat-nodes=/tmp/flat.nodes --disable-operations
--import-strips=10
volumes:
- ../osm:/data:ro
spatiald:
build: .
container_name: trips-spatiald
environment:
SPATIAL_DSN: "host=postgis port=5432 user=trips password=trips dbname=trips sslmode=disable"
ports:
- "5005:5005"
depends_on:
postgis:
condition: service_healthy
volumes:
pgdata:

5
spatial/go.mod Normal file
View File

@ -0,0 +1,5 @@
module trips/spatial
go 1.27
require github.com/lib/pq v1.10.9

2
spatial/go.sum Normal file
View File

@ -0,0 +1,2 @@
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=

52
spatial/import.sh Executable file
View File

@ -0,0 +1,52 @@
#!/usr/bin/env bash
# Import an OSM PBF extract into the local PostGIS stack via osm2pgsql.
#
# ./import.sh [file.osm.pbf] [extract-name]
#
# file.osm.pbf default: ../osm/nh.osm.pbf (path may be absolute)
# extract-name default: basename without extension
#
# The PBF must be readable by uid 1001 (the osm2pgsql image user) — the
# script bind-mounts its directory read-only. After the load, the poi
# table is refreshed (refresh_poi()) and the spatial_extract row updated.
set -euo pipefail
cd "$(dirname "$0")"
PBF=${1:-../osm/nh.osm.pbf}
NAME=${2:-$(basename "${PBF%.osm.pbf}")}
ABS=$(realpath "$PBF")
[[ -f $ABS ]] || { echo "no such PBF: $ABS" >&2; exit 1; }
# 1. DB up (first boot runs schema.sql)
docker compose up -d postgis
echo "waiting for postgis to be healthy…"
for i in $(seq 1 60); do
if docker compose exec -T postgis pg_isready -U trips -d trips >/dev/null 2>&1; then break; fi
sleep 2
[[ $i -eq 60 ]] && { echo "postgis never became healthy" >&2; exit 1; }
done
# 2. osm2pgsql load (one-shot container, PBF bind-mounted ro)
docker run --rm \
--user 1001:1001 \
--network "trips_default" \
-v "$(dirname "$ABS"):/data:ro" \
-e PBF_FILE="/data/$(basename "$ABS")" \
-e ARGS="--host=postgis --port=5432 --database=trips --user=trips --password=trips \
--create --clean --slim --flat-nodes=/tmp/flat.nodes --disable-operations" \
osm2pgsql/osm2pgsql:16
# 3. refresh the queryable poi table + extract bookkeeping
docker compose exec -T postgis psql -U trips -d trips -v ON_ERROR_STOP=1 <<SQL
INSERT INTO spatial_extract (name) VALUES ('${NAME}')
ON CONFLICT (name) DO NOTHING;
SELECT refresh_poi('${NAME}');
SQL
# 4. report
docker compose exec -T postgis psql -U trips -d trips -c \
"SELECT name, poi_count, to_char(imported_at,'YYYY-MM-DD HH24:MI') AS imported
FROM spatial_extract ORDER BY name;"
echo "done — extract '$NAME' is queryable. Start the service:"
echo " docker compose up -d spatiald # :5005, proxied by the mock server at /spatial/*"

405
spatial/main.go Normal file
View File

@ -0,0 +1,405 @@
// spatiald — the PostGIS query service for the trip planner.
//
// Native spatial queries over the osm2pgsql-loaded OSM extracts:
//
// GET /health { ok, extracts: {name: poi_count} }
// GET /near?lat=&lng=&r= ST_DWithin radius search
// GET /nearest?lat=&lng= KNN (ORDER BY geom <-> point)
// GET /corridor?points=&r= corridor buffering along a route
//
// Optional filters on all three:
//
// extract=nh restrict to one loaded extract
// kind=amenity=restaurant exact kind match, or one of the bare values
// (restaurant, museum, cafe, …)
// name=café ILIKE substring on the POI name
// limit=20 default 20, max 200
//
// /corridor's `points` is "lng,lat;lng,lat;…" (a router polyline, downsampled
// is fine — the buffer does the smoothing work).
package main
import (
"database/sql"
"encoding/json"
"flag"
"fmt"
"log"
"net/http"
"strconv"
"strings"
"time"
_ "github.com/lib/pq"
)
var (
dsn *sql.DB
addr string
)
type poi struct {
Extract string `json:"extract"`
OsmID int64 `json:"osm_id"`
OsmType string `json:"osm_type"`
Name string `json:"name"`
Kind string `json:"kind"`
OpenH *string `json:"opening_hours,omitempty"`
Fee *string `json:"fee,omitempty"`
Website *string `json:"website,omitempty"`
AddrCity *string `json:"addr_city,omitempty"`
DistM float64 `json:"dist_m"`
}
func main() {
dsnFlag := flag.String("dsn",
"host=localhost port=5432 user=trips password=trips dbname=trips sslmode=disable",
"PostgreSQL/PostGIS DSN")
addrFlag := flag.String("addr", ":5005", "listen address")
flag.Parse()
addr = *addrFlag
var err error
dsn, err = sql.Open("postgres", *dsnFlag)
if err != nil {
log.Fatalf("sql.Open: %v", err)
}
dsn.SetMaxOpenConns(8)
dsn.SetConnMaxLifetime(30 * time.Minute)
if err = dsn.Ping(); err != nil {
log.Printf("spatiald: DB not reachable yet (%v) — serving /health as down until it is", err)
}
mux := http.NewServeMux()
mux.HandleFunc("/health", handleHealth)
mux.HandleFunc("/near", handleNear)
mux.HandleFunc("/nearest", handleNearest)
mux.HandleFunc("/corridor", handleCorridor)
log.Printf("spatiald: listening on %s (dsn: %s)", addr, *dsnFlag)
srv := &http.Server{Addr: addr, Handler: mux, ReadHeaderTimeout: 5 * time.Second}
if err = srv.ListenAndServe(); err != nil {
log.Fatal(err)
}
}
func cors(w http.ResponseWriter) {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Content-Type", "application/json")
}
func jerr(w http.ResponseWriter, code int, msg string) {
cors(w)
w.WriteHeader(code)
fmt.Fprintf(w, `{"error":%q}`, strings.ReplaceAll(msg, `"`, `\"`))
}
// common filters → (where clause, args). kind accepts "amenity=restaurant"
// (exact) or "restaurant" (match the tag value across all kind families).
func filters(q map[string][]string) (string, []any, error) {
var conds []string
var args []any
if e := first(q, "extract"); e != "" {
conds = append(conds, "poi.extract = $%d")
args = append(args, e)
}
if k := first(q, "kind"); k != "" {
if strings.Contains(k, "=") {
conds = append(conds, "poi.kind = $%d")
args = append(args, k)
} else {
// bare value: match the part after '=' in any kind family
conds = append(conds, "poi.kind LIKE $%d")
args = append(args, "%="+k)
}
}
if n := first(q, "name"); n != "" {
conds = append(conds, "poi.name ILIKE $%d")
args = append(args, "%"+n+"%")
}
if len(conds) == 0 {
return "", args, nil
}
return " WHERE " + strings.Join(conds, " AND "), args, nil
}
func first(q map[string][]string, k string) string {
if v := q[k]; len(v) > 0 {
return strings.TrimSpace(v[0])
}
return ""
}
func limitOf(q map[string][]string) int {
l, err := strconv.Atoi(first(q, "limit"))
if err != nil || l < 1 {
return 20
}
if l > 200 {
return 200
}
return l
}
// rows → pois; the SELECT must end with a distance column in metres.
func scanPois(rs *sql.Rows) ([]poi, error) {
var out []poi
for rs.Next() {
var p poi
if err := rs.Scan(&p.Extract, &p.OsmID, &p.OsmType, &p.Name, &p.Kind,
&p.OpenH, &p.Fee, &p.Website, &p.AddrCity, &p.DistM); err != nil {
return nil, err
}
out = append(out, p)
}
return out, rs.Err()
}
func writePois(w http.ResponseWriter, pois []poi) {
if pois == nil {
pois = []poi{}
}
cors(w)
fmt.Fprintf(w, `{"count":%d,"results":`, len(pois))
b, _ := json.Marshal(pois)
w.Write(b)
w.Write([]byte("}"))
}
func handleHealth(w http.ResponseWriter, r *http.Request) {
cors(w)
type ext struct {
Name string `json:"name"`
PoiCount *int64 `json:"poi_count"`
Imported *string `json:"imported_at,omitempty"`
}
healthy := dsn.Ping() == nil
var extracts []ext
if healthy {
rows, err := dsn.Query(`SELECT name, poi_count, to_char(imported_at, 'YYYY-MM-DD"T"HH24:MI:SSOF') FROM spatial_extract ORDER BY name`)
if err == nil {
for rows.Next() {
var e ext
rows.Scan(&e.Name, &e.PoiCount, &e.Imported)
extracts = append(extracts, e)
}
rows.Close()
}
}
if extracts == nil {
extracts = []ext{}
}
fmt.Fprintf(w, `{"ok":%v,"postgis":%v,"extracts":`, healthy, healthy)
b, _ := json.Marshal(extracts)
w.Write(b)
w.Write([]byte("}"))
}
const poiCols = `poi.extract, poi.osm_id, poi.osm_type, poi.name, poi.kind,
poi.opening_hours, poi.fee, poi.website, poi.addr_city`
func handleNear(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
lat, lng, err := point(q)
if err != nil {
jerr(w, 400, err.Error())
return
}
rad, err := radius(q)
if err != nil {
jerr(w, 400, err.Error())
return
}
where, args, _ := filters(q)
args = append(args, lat, lng, rad, limitOf(q))
// point geometry first, then geography cast for the radius test, then KNN
// ordering among hits — ST_DWithin(geography) uses the GIST index.
query := `SELECT ` + poiCols + `,
ST_Distance_Sphere(poi.geom, ST_SetSRID(ST_MakePoint($2, $1), 4326)) AS dist_m
FROM poi` + where + `
AND ST_DWithin(poi.geom::geography, ST_SetSRID(ST_MakePoint($2, $1), 4326)::geography, $3)
ORDER BY poi.geom <-> ST_SetSRID(ST_MakePoint($2, $1), 4326)
LIMIT $4`
// renumber: filters consume $1..$n, so shift the point/radius/limit params
query, args = renumber(query, args, q)
if query == "" {
jerr(w, 500, "param renumbering failed")
return
}
rows, err := dsn.Query(query, args...)
if err != nil {
jerr(w, 502, "query: "+err.Error())
return
}
pois, err := scanPois(rows)
rows.Close()
if err != nil {
jerr(w, 502, err.Error())
return
}
writePois(w, pois)
}
func handleNearest(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
lat, lng, err := point(q)
if err != nil {
jerr(w, 400, err.Error())
return
}
where, args, _ := filters(q)
args = append(args, lat, lng, limitOf(q))
query := `SELECT ` + poiCols + `,
ST_Distance_Sphere(poi.geom, ST_SetSRID(ST_MakePoint($2, $1), 4326)) AS dist_m
FROM poi` + where + `
ORDER BY poi.geom <-> ST_SetSRID(ST_MakePoint($2, $1), 4326)
LIMIT $3`
query, args = renumber(query, args, q)
if query == "" {
jerr(w, 500, "param renumbering failed")
return
}
rows, err := dsn.Query(query, args...)
if err != nil {
jerr(w, 502, "query: "+err.Error())
return
}
pois, err := scanPois(rows)
rows.Close()
if err != nil {
jerr(w, 502, err.Error())
return
}
writePois(w, pois)
}
func handleCorridor(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
rad, err := radius(q)
if err != nil {
jerr(w, 400, err.Error())
return
}
pts := parsePoints(first(q, "points"))
if len(pts) < 2 {
jerr(w, 400, "points must be 'lng,lat;lng,lat;…' with at least 2 points")
return
}
where, args, _ := filters(q)
var coords []string
for _, p := range pts {
coords = append(coords, fmt.Sprintf("%s %s", strconv.FormatFloat(p[1], 'f', 6, 64), strconv.FormatFloat(p[0], 'f', 6, 64)))
}
line := "LINESTRING(" + strings.Join(coords, ",") + ")"
args = append(args, line, rad, limitOf(q))
// distance to a line uses ST_Distance(geography) (ST_Distance_Sphere is
// point-to-point only); the corridor is a buffered geography band.
query := `SELECT ` + poiCols + `,
ST_Distance(poi.geom::geography, ST_SetSRID(ST_GeomFromText($1), 4326)::geography) AS dist_m
FROM poi` + where + `
AND ST_DWithin(poi.geom::geography,
ST_Buffer(ST_SetSRID(ST_GeomFromText($1), 4326)::geography, $2), true)
ORDER BY dist_m
LIMIT $3`
query, args = renumber(query, args, q)
if query == "" {
jerr(w, 500, "param renumbering failed")
return
}
rows, err := dsn.Query(query, args...)
if err != nil {
jerr(w, 502, "query: "+err.Error())
return
}
pois, err := scanPois(rows)
rows.Close()
if err != nil {
jerr(w, 502, err.Error())
return
}
writePois(w, pois)
}
// point/radius parse the lat/lng (degrees) and r (metres) query params.
func point(q map[string][]string) (lat, lng float64, err error) {
lat, err = strconv.ParseFloat(first(q, "lat"), 64)
if err != nil {
return 0, 0, fmt.Errorf("lat is required (decimal degrees)")
}
lng, err = strconv.ParseFloat(first(q, "lng"), 64)
if err != nil {
return 0, 0, fmt.Errorf("lng is required (decimal degrees)")
}
if lat < -90 || lat > 90 || lng < -180 || lng > 180 {
return 0, 0, fmt.Errorf("lat/lng out of range")
}
return lat, lng, nil
}
func radius(q map[string][]string) (float64, error) {
s := first(q, "r")
if s == "" {
return 500, nil
}
v, err := strconv.ParseFloat(s, 64)
if err != nil || v < 50 || v > 50000 {
return 0, fmt.Errorf("r must be 50..50000 metres")
}
return v, nil
}
// parsePoints: "lng,lat;lng,lat" → [[lng,lat],…] in that order.
func parsePoints(s string) [][2]float64 {
var out [][2]float64
for _, pair := range strings.Split(s, ";") {
pair = strings.TrimSpace(pair)
if pair == "" {
continue
}
xy := strings.Split(pair, ",")
if len(xy) != 2 {
continue
}
lng, e1 := strconv.ParseFloat(strings.TrimSpace(xy[0]), 64)
lat, e2 := strconv.ParseFloat(strings.TrimSpace(xy[1]), 64)
if e1 == nil && e2 == nil {
out = append(out, [2]float64{lng, lat})
}
}
return out
}
// renumber: the WHERE clause (from filters) already contains $1..$n in the
// order filters() appended its args. The trailing point/radius/limit params
// were written as $1,$2,$3,$4 — rewrite them to $n+1, $n+2, … so the arg
// list (filter args first, then point args) lines up.
func renumber(query string, args []any, q map[string][]string) (string, []any) {
n := 0
if e := first(q, "extract"); e != "" {
n++
}
if k := first(q, "kind"); k != "" {
n++
}
if nn := first(q, "name"); nn != "" {
n++
}
var b strings.Builder
for i := 0; i < len(query); i++ {
c := query[i]
if c == '$' && i+1 < len(query) && query[i+1] >= '1' && query[i+1] <= '9' {
// count-digit placeholders only — the filter params are < 10
d := int(query[i+1] - '0')
if d <= n {
b.WriteByte(c)
b.WriteByte(query[i+1])
i++
continue
}
fmt.Fprintf(&b, "$%d", n+d)
i++
continue
}
b.WriteByte(c)
}
return b.String(), args
}

98
spatial/schema.sql Normal file
View File

@ -0,0 +1,98 @@
-- PostGIS schema for the trip planner (runs once, on first container boot).
-- osm2pgsql loads the standard planet_osm_* tables; we then materialize the
-- POI points we query on, with a GIST index for ST_DWithin / KNN.
CREATE EXTENSION IF NOT EXISTS postgis;
-- Named data sets (so multi-extract setups — e.g. northeast + colombia — can
-- be imported into one DB and disambiguated).
CREATE TABLE IF NOT EXISTS spatial_extract (
name text PRIMARY KEY,
imported_at timestamptz NOT NULL DEFAULT now(),
bounds box, -- from osmium (in degrees)
poi_count bigint
);
-- Searchable POIs: every named node/way of the interesting kinds, as a point.
-- osm2pgsql's default table schema (see README: --default-schema / stock).
CREATE TABLE IF NOT EXISTS poi (
extract text NOT NULL REFERENCES spatial_extract(name) ON DELETE CASCADE,
osm_id bigint NOT NULL,
osm_type char NOT NULL, -- 'N' | 'W'
name text NOT NULL,
kind text NOT NULL, -- dominant tag: amenity=, tourism=, …
amenity text,
tourism text,
shop text,
leisure text,
historic text,
place text,
opening_hours text,
fee text, -- OSM fee tag / fee:* values
website text,
addr_city text,
geom geometry(Point, 4326) NOT NULL,
PRIMARY KEY (extract, osm_type, osm_id)
);
CREATE INDEX IF NOT EXISTS poi_geom_gist ON poi USING GIST (geom);
CREATE INDEX IF NOT EXISTS poi_kind ON poi (extract, kind);
CREATE INDEX IF NOT EXISTS poi_name_trgm ON poi USING GIN (to_tsvector('simple', name));
-- pgvector (semantic POI index) is a later milestone: it needs the vector
-- package in the image (postgis/postgis doesn't ship it), so it gets its own
-- init script + embedding importer instead of living in this base schema.
-- Helper: refresh the poi table from a freshly loaded extract.
-- (Invoked by import.sh; kept here so the SQL lives in one place.)
CREATE OR REPLACE FUNCTION refresh_poi(p_extract text) RETURNS void AS $$
BEGIN
DELETE FROM poi WHERE extract = p_extract;
INSERT INTO poi (extract, osm_id, osm_type, name, kind, amenity, tourism, shop,
leisure, historic, place, opening_hours, fee, website, addr_city, geom)
WITH cand AS (
SELECT 'N'::char AS t, n.osm_id, n.name, n.amenity, n.tourism, n.shop,
n.leisure, n.historic, n.place, n.opening_hours, n.fee, n.website,
n.addr_city, ST_SetSRID(ST_MakePoint(n.lon, n.lat), 4326) AS geom
FROM planet_osm_node n
WHERE n.name IS NOT NULL AND n.name <> ''
AND (n.amenity IS NOT NULL OR n.tourism IS NOT NULL OR n.shop IS NOT NULL
OR n.leisure IS NOT NULL OR n.historic IS NOT NULL OR n.place IS NOT NULL)
UNION ALL
SELECT 'W'::char, w.osm_id, w.name, w.amenity, w.tourism, w.shop,
w.leisure, w.historic, w.place, w.opening_hours, w.fee, w.website,
w.addr_city, ST_Centroid(w.geom)
FROM planet_osm_way w
WHERE w.name IS NOT NULL AND w.name <> ''
AND (w.amenity IS NOT NULL OR w.tourism IS NOT NULL OR w.shop IS NOT NULL
OR w.leisure IS NOT NULL OR w.historic IS NOT NULL)
AND ST_GeometryType(w.geom) = 'ST_Polygon' -- areas only (gardens, parks…)
)
SELECT p_extract, osm_id, t, name,
COALESCE(
CASE WHEN amenity IS NOT NULL THEN 'amenity=' || amenity
END,
CASE WHEN tourism IS NOT NULL THEN 'tourism=' || tourism
END,
CASE WHEN shop IS NOT NULL THEN 'shop=' || shop
END,
CASE WHEN leisure IS NOT NULL THEN 'leisure=' || leisure
END,
CASE WHEN historic IS NOT NULL THEN 'historic=' || historic
END,
CASE WHEN place IS NOT NULL THEN 'place=' || place
END
),
amenity, tourism, shop, leisure, historic, place,
opening_hours, fee, website, addr_city, geom
FROM cand
WHERE COALESCE(amenity, tourism, shop, leisure, historic, place) IS NOT NULL;
UPDATE spatial_extract
SET poi_count = (SELECT count(*) FROM poi WHERE extract = p_extract),
imported_at = now()
WHERE name = p_extract;
ANALYZE poi;
END;
$$ LANGUAGE plpgsql;