gmp/trips
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PostGIS: live — imports, spatiald, and fixes from real runs

Browse Source 
Ran the full stack (docker now available via sg docker):
- postgis/postgis:16-3.4 up, both extracts imported:
  northeast 291,000 POIs (from ~/osm-build/data/northeast.osm.pbf),
  colombia 228,446 POIs
- spatiald live on :5005, proxied by the mock server (/spatial/*,
  /spatial-status); UI badge + poi_near tool activate automatically

Fixes discovered by running against real data:
- osm2pgsql image: osm2pgsql/osm2pgsql does not exist on Docker Hub —
  use iboates/osm2pgsql, bypass its DB-probing entrypoint (--entrypoint
  osm2pgsql), password via PGPASSWORD (this build's --password flag
  forces an interactive prompt)
- osm2pgsql 2.x schema: planet_osm_point/_polygon (not _node/_way),
  geometry in EPSG:3857 'way' column, -k hstore for opening_hours/
  fee/website/addr:*, refresh_poi() dedupes relation polygons and
  uses hstore -> (not ->>)
- compose file downgraded to v1-compatible 3.8 (host has docker-compose
  1.29, no v2 plugin); OSM_DIR exported by import.sh
- spatiald: proper SQL parameterization (@@i@@ tokens, body ..
  shifted past the filter args — the first renumber attempt was
  unsound), ST_Distance(geography) instead of the geometry-only
  ST_Distance_Sphere, 4-arg ST_DWithin for the corridor band, WKT in
  lng lat order, kind= accepts exact (amenity=restaurant), bare tag
  value (restaurant) or family (tourism)
- mock server /spatial proxy: forward u.search (was dropping it)
- app.js poi_near: 'points' param, semicolons must be %3B-encoded
  (Go url.Parse drops the tail of a value containing a raw ';')
- README: osm2pgsql 2.x data notes + kind semantics
This commit is contained in:
Greg Pomerantz 2026-09-10 12:23:49 -04:00
parent 3da04c111e
commit d05ebfa72f
8 changed files with 290 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mock/app.js
View File

@ -408,7 +408,7 @@ const TOOLS = {
let qs = `lat=${lat}&lng=${lng}&r=${r}`; let qs = `lat=${lat}&lng=${lng}&r=${r}`;
if (a.kind) qs += '&kind=' + encodeURIComponent(a.kind); // e.g. amenity=restaurant, tourism=museum if (a.kind) qs += '&kind=' + encodeURIComponent(a.kind); // e.g. amenity=restaurant, tourism=museum
if (a.name) qs += '&name=' + encodeURIComponent(a.name); if (a.name) qs += '&name=' + encodeURIComponent(a.name);
if (a.corridor) qs += '&corridor=' + encodeURIComponent(a.corridor); // 'lng,lat;lng,lat;…' route polyline if (a.corridor) qs += '&points=' + encodeURIComponent(a.corridor); // 'lng,lat;lng,lat;…' route polyline (encoded: Go parses raw ';' as a param separator)
const rj = await fetch('/spatial/' + (a.corridor ? 'corridor' : 'near') + '?' + qs); const rj = await fetch('/spatial/' + (a.corridor ? 'corridor' : 'near') + '?' + qs);
const j = await rj.json(); const j = await rj.json();
if (j.error) return { error: j.error }; if (j.error) return { error: j.error };

2
mock/server.js
View File

@ -435,7 +435,7 @@ http.createServer((req, res) => {
if (u.pathname === '/spatial-status' || u.pathname.startsWith('/spatial/')) { if (u.pathname === '/spatial-status' || u.pathname.startsWith('/spatial/')) {
const up = u.pathname === '/spatial-status' const up = u.pathname === '/spatial-status'
? SPATIAL.url + '/health' ? SPATIAL.url + '/health'
: SPATIAL.url + u.pathname.slice('/spatial'.length); // keep query string : SPATIAL.url + u.pathname.slice('/spatial'.length) + (u.search || '');
const isProbe = u.pathname === '/spatial-status'; const isProbe = u.pathname === '/spatial-status';
fetch(up, { signal: AbortSignal.timeout(15000) }).then(async r => { fetch(up, { signal: AbortSignal.timeout(15000) }).then(async r => {
const t = await r.text(); const t = await r.text();

109
spatial/.nfs000000000016f2b2000000e7 Normal file
View File

@ -0,0 +1,109 @@
-- PostGIS schema for the trip planner (runs once, on first container boot).
-- osm2pgsql loads the standard planet_osm_* tables; we then materialize the
-- POI points we query on, with a GIST index for ST_DWithin / KNN.
CREATE EXTENSION IF NOT EXISTS postgis;
-- osm2pgsql --hstore puts every tag that has no dedicated column here
CREATE EXTENSION IF NOT EXISTS hstore;
-- Named data sets (so multi-extract setups — e.g. northeast + colombia — can
-- be imported into one DB and disambiguated).
CREATE TABLE IF NOT EXISTS spatial_extract (
name text PRIMARY KEY,
imported_at timestamptz NOT NULL DEFAULT now(),
bounds box, -- from osmium (in degrees)
poi_count bigint
);
-- Searchable POIs: every named point/polygon of the interesting kinds.
-- osm2pgsql 2.x pgsql output schema: planet_osm_point / planet_osm_polygon,
-- geometry in EPSG:3857 in a column called `way`; tags without a dedicated
-- column (opening_hours, fee, website, addr:*) live in the hstore `tags`.
CREATE TABLE IF NOT EXISTS poi (
extract text NOT NULL REFERENCES spatial_extract(name) ON DELETE CASCADE,
osm_id bigint NOT NULL,
osm_type char NOT NULL, -- 'N' | 'W'
name text NOT NULL,
kind text NOT NULL, -- dominant tag: amenity=, tourism=, …
amenity text,
tourism text,
shop text,
leisure text,
historic text,
place text,
opening_hours text,
fee text, -- OSM fee tag / fee:* values
website text,
addr_city text,
geom geometry(Point, 4326) NOT NULL, -- reprojected from 3857
PRIMARY KEY (extract, osm_type, osm_id)
);
CREATE INDEX IF NOT EXISTS poi_geom_gist ON poi USING GIST (geom);
CREATE INDEX IF NOT EXISTS poi_kind ON poi (extract, kind);
CREATE INDEX IF NOT EXISTS poi_name_trgm ON poi USING GIN (to_tsvector('simple', name));
-- pgvector (semantic POI index) is a later milestone: it needs the vector
-- package in the image (postgis/postgis doesn't ship it), so it gets its own
-- init script + embedding importer instead of living in this base schema.
-- Helper: refresh the poi table from a freshly loaded extract.
-- (Invoked by import.sh; kept here so the SQL lives in one place.)
CREATE OR REPLACE FUNCTION refresh_poi(p_extract text) RETURNS void AS $$
BEGIN
DELETE FROM poi WHERE extract = p_extract;
INSERT INTO poi (extract, osm_id, osm_type, name, kind, amenity, tourism, shop,
leisure, historic, place, opening_hours, fee, website, addr_city, geom)
WITH cand AS (
SELECT 'N'::char AS t, n.osm_id, n.name, n.amenity, n.tourism, n.shop,
n.leisure, n.historic, n.place,
n.tags->>'opening_hours' AS opening_hours,
COALESCE(n.tags->>'fee', n.tags->>'fee:tourism') AS fee,
COALESCE(n.tags->>'website', n.tags->>'url') AS website,
n.tags->>'addr:city' AS addr_city,
ST_Transform(n.way, 4326) AS geom
FROM planet_osm_point n
WHERE n.name IS NOT NULL AND n.name <> ''
AND (n.amenity IS NOT NULL OR n.tourism IS NOT NULL OR n.shop IS NOT NULL
OR n.leisure IS NOT NULL OR n.historic IS NOT NULL OR n.place IS NOT NULL)
UNION ALL
SELECT 'W'::char, w.osm_id, w.name, w.amenity, w.tourism, w.shop,
w.leisure, w.historic, w.place,
w.tags->>'opening_hours',
COALESCE(w.tags->>'fee', w.tags->>'fee:tourism'),
COALESCE(w.tags->>'website', w.tags->>'url'),
w.tags->>'addr:city',
ST_Centroid(ST_Transform(w.way, 4326))
FROM planet_osm_polygon w
WHERE w.name IS NOT NULL AND w.name <> ''
AND (w.amenity IS NOT NULL OR w.tourism IS NOT NULL OR w.shop IS NOT NULL
OR w.leisure IS NOT NULL OR w.historic IS NOT NULL)
)
SELECT p_extract, osm_id, t, name,
COALESCE(
CASE WHEN amenity IS NOT NULL THEN 'amenity=' || amenity
END,
CASE WHEN tourism IS NOT NULL THEN 'tourism=' || tourism
END,
CASE WHEN shop IS NOT NULL THEN 'shop=' || shop
END,
CASE WHEN leisure IS NOT NULL THEN 'leisure=' || leisure
END,
CASE WHEN historic IS NOT NULL THEN 'historic=' || historic
END,
CASE WHEN place IS NOT NULL THEN 'place=' || place
END
),
amenity, tourism, shop, leisure, historic, place,
opening_hours, fee, website, addr_city, geom
FROM cand
WHERE COALESCE(amenity, tourism, shop, leisure, historic, place) IS NOT NULL;
UPDATE spatial_extract
SET poi_count = (SELECT count(*) FROM poi WHERE extract = p_extract),
imported_at = now()
WHERE name = p_extract;
ANALYZE poi;
END;
$$ LANGUAGE plpgsql;

29
spatial/README.md
View File

@ -8,7 +8,7 @@ nearest, bbox)"). Replaces the flat-file/Overpass approximations.
| Piece | What it is | | Piece | What it is |
|---|---| |---|---|
| `docker-compose.yml` | `postgis/postgis:16-3.4` (DB, port 5432) + one-shot `osm2pgsql:16` importer + `spatiald` (query service, port 5005) | | `docker-compose.yml` | `postgis/postgis:16-3.4` (DB, port 5432) + one-shot `iboates/osm2pgsql` importer + `spatiald` (query service, port 5005) |
| `schema.sql` | runs on first boot: `poi` table (named POIs as points, GIST index), `spatial_extract` bookkeeping, `refresh_poi()` | | `schema.sql` | runs on first boot: `poi` table (named POIs as points, GIST index), `spatial_extract` bookkeeping, `refresh_poi()` |
| `import.sh` | loads a PBF from `../osm/` via osm2pgsql, refreshes `poi` | | `import.sh` | loads a PBF from `../osm/` via osm2pgsql, refreshes `poi` |
| `main.go` (spatiald) | JSON query service over `poi` | | `main.go` (spatiald) | JSON query service over `poi` |
@ -24,10 +24,10 @@ nearest, bbox)"). Replaces the flat-file/Overpass approximations.
## Usage ## Usage
```bash ```bash
docker compose up -d postgis # first: runs schema.sql docker-compose up -d postgis # first: runs schema.sql
./import.sh ~/trips/osm/nh.osm.pbf nh # load New England (default)
./import.sh ~/trips/osm/colombia.osm.pbf colombia ./import.sh ~/trips/osm/colombia.osm.pbf colombia
docker compose up -d spatiald # query service on :5005 ./import.sh ~/osm-build/data/northeast.osm.pbf northeast # merged NE extract
docker-compose up -d spatiald # query service on :5005
``` ```
The mock server (`mock/server.js`) proxies it: `GET /spatial/*` → `:5005`, The mock server (`mock/server.js`) proxies it: `GET /spatial/*` → `:5005`,
@ -61,13 +61,20 @@ curl 'localhost:5005/corridor?r=300&kind=fuel&points=-71.06,42.35;-71.07,42.36'
## Data notes ## Data notes
* `poi` covers every **named** node (or polygon way, via centroid) with an * Built for **osm2pgsql 2.x** (`iboates/osm2pgsql`): tables are
`amenity`/`tourism`/`shop`/`leisure`/`historic`/`place` tag — the kinds the `planet_osm_point` / `planet_osm_polygon` (not the 1.x `_node`/`_way`),
planner asks about. Unnamed amenities (e.g. a nameless kiosk) are out of geometry lives in a `way` column in EPSG:3857, and the import runs with
scope for v1. `-k` so tags without a fixed column (`opening_hours`, `fee`, `website`,
* `opening_hours`, `fee`, `website`, `addr_city` are carried straight from `addr:city`) land in the hstore `tags` column that `refresh_poi()` reads.
the OSM tags; the live web enrichment (SearXNG, `mock/app.js`) fills gaps `refresh_poi()` also dedupes relation-derived polygons (2.x repeats them
the tags don't have (the 3-source blend). per relation with negated ids).
* `poi` covers every **named** point/polygon with an `amenity`/`tourism`/
`shop`/`leisure`/`historic`/`place` tag. Unnamed amenities (a nameless
kiosk) are out of scope for v1.
* `kind` filter semantics: `kind=amenity=restaurant` (exact),
`kind=restaurant` (tag value, any family), `kind=tourism` (whole family).
* Corridor `points` must URL-encode the semicolons (`%3B`) — Go's
`url.Parse` drops the tail of a value that contains a raw `;`.
* Multiple extracts coexist in one DB, disambiguated by `poi.extract` * Multiple extracts coexist in one DB, disambiguated by `poi.extract`
(set automatically by `import.sh`). (set automatically by `import.sh`).
* pgvector (semantic POI index, later milestone) gets its own init script + * pgvector (semantic POI index, later milestone) gets its own init script +

52
spatial/docker-compose.yml
View File

@ -1,14 +1,17 @@
# PostGIS spatial stack for the trip planner. # PostGIS spatial stack for the trip planner.
# #
# docker compose up -d postgis # start the DB (schema.sql runs on first boot) # docker-compose up -d postgis # start the DB (schema.sql runs on first boot)
# ./import.sh # import ~/trips/osm/nh.osm.pbf (one-shot osm2pgsql) # ./import.sh ~/trips/osm/nh.osm.pbf nh # import an OSM extract (one-shot osm2pgsql)
# docker-compose up -d spatiald # start the query service on :5005
#
# Composed for docker-compose v1 (3.8) AND v2 (the same file works with
# `docker compose` once the v2 plugin is available): no healthchecks or
# `depends_on` conditions — import.sh polls pg_isready itself, and spatiald
# tolerates a not-yet-ready DB at boot.
# #
# OSM_DIR (default ~/trips/osm) is where the PBF extracts live — outside the # OSM_DIR (default ~/trips/osm) is where the PBF extracts live — outside the
# dev tree. Export it to override. # dev tree. Export it to override.
# docker compose run --rm spatiald # start the query service on :5005 version: "3.8"
#
# The PBF extracts live in ../osm (gitignored). osm2pgsql runs as a one-shot
# container so the host needs no osm2pgsql/osmium installation.
services: services:
postgis: postgis:
image: postgis/postgis:16-3.4 image: postgis/postgis:16-3.4
@ -23,30 +26,23 @@ services:
- pgdata:/var/lib/postgresql/data - pgdata:/var/lib/postgresql/data
# initdb scripts run in lexicographic order, once, on an empty volume # initdb scripts run in lexicographic order, once, on an empty volume
- ./schema.sql:/docker-entrypoint-initdb.d/10-schema.sql:ro - ./schema.sql:/docker-entrypoint-initdb.d/10-schema.sql:ro
healthcheck: restart: unless-stopped
test: ["CMD-SHELL", "pg_isready -U trips -d trips"]
interval: 5s
timeout: 3s
retries: 12
# One-shot importer. `docker compose run --rm importer` (or import.sh) # One-shot importer (also usable as: docker-compose run --rm importer
# loads the PBF named in PBF_FILE into the DB. The osm2pgsql image runs # -- /data/<file>.osm.pbf ...). import.sh does the same with a bare
# as uid 1001 and needs the PBF readable — import.sh bind-mounts ../osm. # `docker run`, since the PBF path varies per import.
importer: importer:
image: osm2pgsql/osm2pgsql:16 image: iboates/osm2pgsql:latest
user: "1001:1001" entrypoint: osm2pgsql
network_mode: "service:postgis"
environment: environment:
PBF_FILE: /data/nh.osm.pbf PGPASSWORD: trips # the image build's --password flag forces a prompt
# --create --clean: fresh load every time (extracts are small enough); command: >-
# slim mode keeps the RAM footprint sane; disable-operations saves time. /data/colombia.osm.pbf --host=postgis --port=5432 --database=trips
ARGS: >- --user=trips -k --slim
--host=postgis --port=5432 --database=trips --user=trips --flat-nodes=/tmp/flat.nodes
--password=trips --create --clean --slim network_mode: "service:postgis"
--flat-nodes=/tmp/flat.nodes --disable-operations
--import-strips=10
volumes: volumes:
- ${OSM_DIR:-$HOME/trips/osm}:/data:ro - ${OSM_DIR}:/data:ro # OSM_DIR is exported by import.sh (default ~/trips/osm)
spatiald: spatiald:
build: . build: .
@ -56,8 +52,8 @@ services:
ports: ports:
- "5005:5005" - "5005:5005"
depends_on: depends_on:
postgis: - postgis
condition: service_healthy restart: unless-stopped
volumes: volumes:
pgdata: pgdata:

62
spatial/import.sh
View File

@ -3,52 +3,72 @@
# #
# ./import.sh [file.osm.pbf] [extract-name] # ./import.sh [file.osm.pbf] [extract-name]
# #
# file.osm.pbf default: $OSM_DIR/nh.osm.pbf (OSM_DIR defaults to # file.osm.pbf default: $OSM_DIR/colombia.osm.pbf (OSM_DIR defaults to
# ~/trips/osm — the PBFs live outside the dev tree) # ~/trips/osm — the PBFs live outside the dev tree).
# The merged New England file lives at
# $HOME/osm-build/data/northeast.osm.pbf (setup-osrm.sh);
# any absolute path works, e.g.:
# ./import.sh ~/osm-build/data/northeast.osm.pbf northeast
# extract-name default: basename without extension # extract-name default: basename without extension
# #
# The PBF must be readable by uid 1001 (the osm2pgsql image user) — the # The osm2pgsql container reaches the DB through the host-published port
# script bind-mounts its directory read-only. After the load, the poi # 5432 (see docker-compose.yml), so no shared compose network is needed.
# table is refreshed (refresh_poi()) and the spatial_extract row updated. # After the load, the poi table is refreshed (refresh_poi()) and the
# spatial_extract row updated.
set -euo pipefail set -euo pipefail
cd "$(dirname "$0")" cd "$(dirname "$0")"
# docker-compose v1 (python) or v2 (plugin) — whichever the host has
DC=docker-compose
command -v docker >/dev/null && docker compose version >/dev/null 2>&1 && DC="docker compose"
# osm2pgsql image: iboates/osm2pgsql (the osm2pgsql/osm2pgsql repo image does
# not exist on Docker Hub). Its /entrypoint.sh does its own DB probing, so we
# bypass it and exec osm2pgsql directly.
IMG=${OSM2PGSQL_IMAGE:-iboates/osm2pgsql:latest}
OSM_DIR="${OSM_DIR:-$HOME/trips/osm}" OSM_DIR="${OSM_DIR:-$HOME/trips/osm}"
PBF=${1:-$OSM_DIR/nh.osm.pbf} export OSM_DIR
PBF=${1:-$OSM_DIR/colombia.osm.pbf}
NAME=${2:-$(basename "${PBF%.osm.pbf}")} NAME=${2:-$(basename "${PBF%.osm.pbf}")}
ABS=$(realpath "$PBF") ABS=$(realpath "$PBF")
[[ -f $ABS ]] || { echo "no such PBF: $ABS" >&2; exit 1; } [[ -f $ABS && $(stat -c%s "$ABS") -gt 100000 ]] || { echo "no such PBF: $ABS" >&2; exit 1; }
# 1. DB up (first boot runs schema.sql) # 1. DB up (first boot runs schema.sql)
docker compose up -d postgis $DC up -d postgis
echo "waiting for postgis to be healthy…" echo "waiting for postgis to be ready…"
for i in $(seq 1 60); do for i in $(seq 1 60); do
if docker compose exec -T postgis pg_isready -U trips -d trips >/dev/null 2>&1; then break; fi if $DC exec -T postgis pg_isready -U trips -d trips >/dev/null 2>&1; then break; fi
sleep 2 sleep 2
[[ $i -eq 60 ]] && { echo "postgis never became healthy" >&2; exit 1; } [[ $i -eq 60 ]] && { echo "postgis never became ready" >&2; exit 1; }
done done
# 2. osm2pgsql load (one-shot container, PBF bind-mounted ro) # 2. osm2pgsql load (one-shot container, PBF bind-mounted ro), on the
# postgis compose network. Note: this osm2pgsql build's --password flag
# FORCES an interactive prompt, so the password goes via PGPASSWORD.
# --create is the 2.x default; the 1.x ops machinery is gone in 2.x.
NET=$(docker inspect trips-postgis | python3 -c \
"import json,sys; print(next(iter(json.load(sys.stdin)[0]['NetworkSettings']['Networks'])))")
docker run --rm \ docker run --rm \
--user 1001:1001 \ --entrypoint osm2pgsql \
--network "trips_default" \ --network "$NET" \
-e PGPASSWORD=trips \
-v "$(dirname "$ABS"):/data:ro" \ -v "$(dirname "$ABS"):/data:ro" \
-e PBF_FILE="/data/$(basename "$ABS")" \ "$IMG" "/data/$(basename "$ABS")" \
-e ARGS="--host=postgis --port=5432 --database=trips --user=trips --password=trips \ --host=postgis --port=5432 --database=trips --user=trips \
--create --clean --slim --flat-nodes=/tmp/flat.nodes --disable-operations" \ -k --slim --flat-nodes=/tmp/flat.nodes # -k: unmatched tags → hstore `tags`
osm2pgsql/osm2pgsql:16
# 3. refresh the queryable poi table + extract bookkeeping # 3. refresh the queryable poi table + extract bookkeeping
docker compose exec -T postgis psql -U trips -d trips -v ON_ERROR_STOP=1 <<SQL $DC exec -T postgis psql -U trips -d trips -v ON_ERROR_STOP=1 <<SQL
INSERT INTO spatial_extract (name) VALUES ('${NAME}') INSERT INTO spatial_extract (name) VALUES ('${NAME}')
ON CONFLICT (name) DO NOTHING; ON CONFLICT (name) DO NOTHING;
SELECT refresh_poi('${NAME}'); SELECT refresh_poi('${NAME}');
SQL SQL
# 4. report # 4. report
docker compose exec -T postgis psql -U trips -d trips -c \ $DC exec -T postgis psql -U trips -d trips -c \
"SELECT name, poi_count, to_char(imported_at,'YYYY-MM-DD HH24:MI') AS imported "SELECT name, poi_count, to_char(imported_at,'YYYY-MM-DD HH24:MI') AS imported
FROM spatial_extract ORDER BY name;" FROM spatial_extract ORDER BY name;"
echo "done — extract '$NAME' is queryable. Start the service:" echo "done — extract '$NAME' is queryable. Start the service:"
echo " docker compose up -d spatiald # :5005, proxied by the mock server at /spatial/*" echo " $DC up -d spatiald # :5005, proxied by the mock server at /spatial/*"

128
spatial/main.go
View File

@ -26,6 +26,7 @@ import (
"fmt" "fmt"
"log" "log"
"net/http" "net/http"
"os"
"strconv" "strconv"
"strings" "strings"
"time" "time"
@ -57,6 +58,9 @@ func main() {
"PostgreSQL/PostGIS DSN") "PostgreSQL/PostGIS DSN")
addrFlag := flag.String("addr", ":5005", "listen address") addrFlag := flag.String("addr", ":5005", "listen address")
flag.Parse() flag.Parse()
if env := os.Getenv("SPATIAL_DSN"); env != "" {
*dsnFlag = env // docker-compose sets the in-network DSN
}
addr = *addrFlag addr = *addrFlag
var err error var err error
@ -95,31 +99,50 @@ func jerr(w http.ResponseWriter, code int, msg string) {
// common filters → (where clause, args). kind accepts "amenity=restaurant" // common filters → (where clause, args). kind accepts "amenity=restaurant"
// (exact) or "restaurant" (match the tag value across all kind families). // (exact) or "restaurant" (match the tag value across all kind families).
func filters(q map[string][]string) (string, []any, error) { // filters → (WHERE clause with @@i@@ placeholder tokens, args). Tokens are
// resolved to $i by renumber(); the query body's own $1..$4 placeholders
// would otherwise collide with the filter params.
func filters(q map[string][]string) (string, []any) {
var conds []string var conds []string
var args []any var args []any
param := func(arg any) string {
args = append(args, arg)
return "@@" + strconv.Itoa(len(args)) + "@@"
}
if e := first(q, "extract"); e != "" { if e := first(q, "extract"); e != "" {
conds = append(conds, "poi.extract = $%d") conds = append(conds, "poi.extract = "+param(e))
args = append(args, e)
} }
if k := first(q, "kind"); k != "" { if k := first(q, "kind"); k != "" {
if strings.Contains(k, "=") { if strings.Contains(k, "=") {
conds = append(conds, "poi.kind = $%d") conds = append(conds, "poi.kind = "+param(k))
args = append(args, k) } else if isFamily(k) {
// bare family name: the whole family ("tourism", "leisure", …)
conds = append(conds, "poi.kind LIKE '"+k+"=%'")
} else { } else {
// bare value: match the part after '=' in any kind family // bare tag value: exact match in any kind family
conds = append(conds, "poi.kind LIKE $%d") v := param(k)
args = append(args, "%="+k) var parts []string
for _, fam := range []string{"amenity", "tourism", "shop", "leisure", "historic", "place"} {
parts = append(parts, "poi.kind = '"+fam+"='||"+v)
}
conds = append(conds, "("+strings.Join(parts, " OR ")+")")
} }
} }
if n := first(q, "name"); n != "" { if n := first(q, "name"); n != "" {
conds = append(conds, "poi.name ILIKE $%d") conds = append(conds, "poi.name ILIKE "+param("%"+n+"%"))
args = append(args, "%"+n+"%")
} }
if len(conds) == 0 { if len(conds) == 0 {
return "", args, nil return "", args
} }
return " WHERE " + strings.Join(conds, " AND "), args, nil return " WHERE " + strings.Join(conds, " AND "), args
}
func isFamily(k string) bool {
for _, f := range []string{"amenity", "tourism", "shop", "leisure", "historic", "place"} {
if k == f {
return true
}
}
return false
} }
func first(q map[string][]string, k string) string { func first(q map[string][]string, k string) string {
@ -209,22 +232,17 @@ func handleNear(w http.ResponseWriter, r *http.Request) {
jerr(w, 400, err.Error()) jerr(w, 400, err.Error())
return return
} }
where, args, _ := filters(q) where, args := filters(q)
args = append(args, lat, lng, rad, limitOf(q)) args = append(args, lat, lng, rad, limitOf(q))
// point geometry first, then geography cast for the radius test, then KNN // point geometry first, then geography cast for the radius test, then KNN
// ordering among hits — ST_DWithin(geography) uses the GIST index. // ordering among hits — ST_DWithin(geography) uses the GIST index.
query := `SELECT ` + poiCols + `, query := `SELECT ` + poiCols + `,
ST_Distance_Sphere(poi.geom, ST_SetSRID(ST_MakePoint($2, $1), 4326)) AS dist_m ST_Distance(poi.geom::geography, ST_SetSRID(ST_MakePoint($2, $1), 4326)::geography) AS dist_m
FROM poi` + where + ` FROM poi` + where + `
AND ST_DWithin(poi.geom::geography, ST_SetSRID(ST_MakePoint($2, $1), 4326)::geography, $3) AND ST_DWithin(poi.geom::geography, ST_SetSRID(ST_MakePoint($2, $1), 4326)::geography, $3)
ORDER BY poi.geom <-> ST_SetSRID(ST_MakePoint($2, $1), 4326) ORDER BY poi.geom <-> ST_SetSRID(ST_MakePoint($2, $1), 4326)
LIMIT $4` LIMIT $4`
// renumber: filters consume $1..$n, so shift the point/radius/limit params query = renumber(query, len(args)-4)
query, args = renumber(query, args, q)
if query == "" {
jerr(w, 500, "param renumbering failed")
return
}
rows, err := dsn.Query(query, args...) rows, err := dsn.Query(query, args...)
if err != nil { if err != nil {
jerr(w, 502, "query: "+err.Error()) jerr(w, 502, "query: "+err.Error())
@ -246,18 +264,14 @@ func handleNearest(w http.ResponseWriter, r *http.Request) {
jerr(w, 400, err.Error()) jerr(w, 400, err.Error())
return return
} }
where, args, _ := filters(q) where, args := filters(q)
args = append(args, lat, lng, limitOf(q)) args = append(args, lat, lng, limitOf(q))
query := `SELECT ` + poiCols + `, query := `SELECT ` + poiCols + `,
ST_Distance_Sphere(poi.geom, ST_SetSRID(ST_MakePoint($2, $1), 4326)) AS dist_m ST_Distance(poi.geom::geography, ST_SetSRID(ST_MakePoint($2, $1), 4326)::geography) AS dist_m
FROM poi` + where + ` FROM poi` + where + `
ORDER BY poi.geom <-> ST_SetSRID(ST_MakePoint($2, $1), 4326) ORDER BY poi.geom <-> ST_SetSRID(ST_MakePoint($2, $1), 4326)
LIMIT $3` LIMIT $3`
query, args = renumber(query, args, q) query = renumber(query, len(args)-3)
if query == "" {
jerr(w, 500, "param renumbering failed")
return
}
rows, err := dsn.Query(query, args...) rows, err := dsn.Query(query, args...)
if err != nil { if err != nil {
jerr(w, 502, "query: "+err.Error()) jerr(w, 502, "query: "+err.Error())
@ -284,10 +298,10 @@ func handleCorridor(w http.ResponseWriter, r *http.Request) {
jerr(w, 400, "points must be 'lng,lat;lng,lat;…' with at least 2 points") jerr(w, 400, "points must be 'lng,lat;lng,lat;…' with at least 2 points")
return return
} }
where, args, _ := filters(q) where, args := filters(q)
var coords []string var coords []string
for _, p := range pts { for _, p := range pts {
coords = append(coords, fmt.Sprintf("%s %s", strconv.FormatFloat(p[1], 'f', 6, 64), strconv.FormatFloat(p[0], 'f', 6, 64))) coords = append(coords, fmt.Sprintf("%s %s", strconv.FormatFloat(p[0], 'f', 6, 64), strconv.FormatFloat(p[1], 'f', 6, 64))) // WKT: lng lat
} }
line := "LINESTRING(" + strings.Join(coords, ",") + ")" line := "LINESTRING(" + strings.Join(coords, ",") + ")"
args = append(args, line, rad, limitOf(q)) args = append(args, line, rad, limitOf(q))
@ -297,14 +311,10 @@ func handleCorridor(w http.ResponseWriter, r *http.Request) {
ST_Distance(poi.geom::geography, ST_SetSRID(ST_GeomFromText($1), 4326)::geography) AS dist_m ST_Distance(poi.geom::geography, ST_SetSRID(ST_GeomFromText($1), 4326)::geography) AS dist_m
FROM poi` + where + ` FROM poi` + where + `
AND ST_DWithin(poi.geom::geography, AND ST_DWithin(poi.geom::geography,
ST_Buffer(ST_SetSRID(ST_GeomFromText($1), 4326)::geography, $2), true) ST_Buffer(ST_SetSRID(ST_GeomFromText($1), 4326)::geography, $2), 0, true)
ORDER BY dist_m ORDER BY dist_m
LIMIT $3` LIMIT $3`
query, args = renumber(query, args, q) query = renumber(query, len(args)-3)
if query == "" {
jerr(w, 500, "param renumbering failed")
return
}
rows, err := dsn.Query(query, args...) rows, err := dsn.Query(query, args...)
if err != nil { if err != nil {
jerr(w, 502, "query: "+err.Error()) jerr(w, 502, "query: "+err.Error())
@ -368,38 +378,30 @@ func parsePoints(s string) [][2]float64 {
return out return out
} }
// renumber: the WHERE clause (from filters) already contains $1..$n in the // renumber resolves the placeholder scheme: the query body was written with
// order filters() appended its args. The trailing point/radius/limit params // $1..$4 for its own params (lat, lng, radius, limit) and the WHERE clause
// were written as $1,$2,$3,$4 — rewrite them to $n+1, $n+2, … so the arg // carries @@i@@ tokens for the n filter args, which come FIRST in the arg
// list (filter args first, then point args) lines up. // list. So: body $d → $(n+d), then @@i@@ → $i. n = number of filter args
func renumber(query string, args []any, q map[string][]string) (string, []any) { // (= len(args) minus the body params, known at the call site).
n := 0 func renumber(query string, n int) string {
if e := first(q, "extract"); e != "" {
n++
}
if k := first(q, "kind"); k != "" {
n++
}
if nn := first(q, "name"); nn != "" {
n++
}
var b strings.Builder var b strings.Builder
for i := 0; i < len(query); i++ { for i := 0; i < len(query); i++ {
c := query[i] if query[i] == '$' && i+1 < len(query) && query[i+1] >= '1' && query[i+1] <= '4' {
if c == '$' && i+1 < len(query) && query[i+1] >= '1' && query[i+1] <= '9' { fmt.Fprintf(&b, "$%d", n+int(query[i+1]-'0'))
// count-digit placeholders only — the filter params are < 10
d := int(query[i+1] - '0')
if d <= n {
b.WriteByte(c)
b.WriteByte(query[i+1])
i++
continue
}
fmt.Fprintf(&b, "$%d", n+d)
i++ i++
continue continue
} }
b.WriteByte(c) if strings.HasPrefix(query[i:], "@@") {
end := strings.Index(query[i+2:], "@@")
if end >= 0 {
if d, err := strconv.Atoi(query[i+2 : i+2+end]); err == nil {
fmt.Fprintf(&b, "$%d", d)
i += 2 + end + 1 // loop's i++ finishes the last '@'
continue
}
}
}
b.WriteByte(query[i])
} }
return b.String(), args return b.String()
} }

45
spatial/schema.sql
View File

@ -3,6 +3,8 @@
-- POI points we query on, with a GIST index for ST_DWithin / KNN. -- POI points we query on, with a GIST index for ST_DWithin / KNN.
CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS postgis;
-- osm2pgsql --hstore puts every tag that has no dedicated column here
CREATE EXTENSION IF NOT EXISTS hstore;
-- Named data sets (so multi-extract setups — e.g. northeast + colombia — can -- Named data sets (so multi-extract setups — e.g. northeast + colombia — can
-- be imported into one DB and disambiguated). -- be imported into one DB and disambiguated).
@ -13,8 +15,10 @@ CREATE TABLE IF NOT EXISTS spatial_extract (
poi_count bigint poi_count bigint
); );
-- Searchable POIs: every named node/way of the interesting kinds, as a point. -- Searchable POIs: every named point/polygon of the interesting kinds.
-- osm2pgsql's default table schema (see README: --default-schema / stock). -- osm2pgsql 2.x pgsql output schema: planet_osm_point / planet_osm_polygon,
-- geometry in EPSG:3857 in a column called `way`; tags without a dedicated
-- column (opening_hours, fee, website, addr:*) live in the hstore `tags`.
CREATE TABLE IF NOT EXISTS poi ( CREATE TABLE IF NOT EXISTS poi (
extract text NOT NULL REFERENCES spatial_extract(name) ON DELETE CASCADE, extract text NOT NULL REFERENCES spatial_extract(name) ON DELETE CASCADE,
osm_id bigint NOT NULL, osm_id bigint NOT NULL,
@ -31,7 +35,7 @@ CREATE TABLE IF NOT EXISTS poi (
fee text, -- OSM fee tag / fee:* values fee text, -- OSM fee tag / fee:* values
website text, website text,
addr_city text, addr_city text,
geom geometry(Point, 4326) NOT NULL, geom geometry(Point, 4326) NOT NULL, -- reprojected from 3857
PRIMARY KEY (extract, osm_type, osm_id) PRIMARY KEY (extract, osm_type, osm_id)
); );
@ -52,21 +56,34 @@ BEGIN
leisure, historic, place, opening_hours, fee, website, addr_city, geom) leisure, historic, place, opening_hours, fee, website, addr_city, geom)
WITH cand AS ( WITH cand AS (
SELECT 'N'::char AS t, n.osm_id, n.name, n.amenity, n.tourism, n.shop, SELECT 'N'::char AS t, n.osm_id, n.name, n.amenity, n.tourism, n.shop,
n.leisure, n.historic, n.place, n.opening_hours, n.fee, n.website, n.leisure, n.historic, n.place,
n.addr_city, ST_SetSRID(ST_MakePoint(n.lon, n.lat), 4326) AS geom n.tags->'opening_hours' AS opening_hours,
FROM planet_osm_node n COALESCE(n.tags->'fee', n.tags->'fee:tourism') AS fee,
COALESCE(n.tags->'website', n.tags->'url') AS website,
n.tags->'addr:city' AS addr_city,
ST_Transform(n.way, 4326) AS geom
FROM planet_osm_point n
WHERE n.name IS NOT NULL AND n.name <> '' WHERE n.name IS NOT NULL AND n.name <> ''
AND (n.amenity IS NOT NULL OR n.tourism IS NOT NULL OR n.shop IS NOT NULL AND (n.amenity IS NOT NULL OR n.tourism IS NOT NULL OR n.shop IS NOT NULL
OR n.leisure IS NOT NULL OR n.historic IS NOT NULL OR n.place IS NOT NULL) OR n.leisure IS NOT NULL OR n.historic IS NOT NULL OR n.place IS NOT NULL)
UNION ALL UNION ALL
SELECT 'W'::char, w.osm_id, w.name, w.amenity, w.tourism, w.shop, -- relation-derived polygons repeat per relation (osm_id negated) —
w.leisure, w.historic, w.place, w.opening_hours, w.fee, w.website, -- keep one row per way id
w.addr_city, ST_Centroid(w.geom) SELECT * FROM (
FROM planet_osm_way w SELECT DISTINCT ON (w.osm_id)
WHERE w.name IS NOT NULL AND w.name <> '' 'W'::char AS t, w.osm_id, w.name, w.amenity, w.tourism, w.shop,
AND (w.amenity IS NOT NULL OR w.tourism IS NOT NULL OR w.shop IS NOT NULL w.leisure, w.historic, w.place,
OR w.leisure IS NOT NULL OR w.historic IS NOT NULL) w.tags->'opening_hours' AS opening_hours,
AND ST_GeometryType(w.geom) = 'ST_Polygon' -- areas only (gardens, parks…) COALESCE(w.tags->'fee', w.tags->'fee:tourism') AS fee,
COALESCE(w.tags->'website', w.tags->'url') AS website,
w.tags->'addr:city' AS addr_city,
ST_Centroid(ST_Transform(w.way, 4326)) AS geom
FROM planet_osm_polygon w
WHERE w.name IS NOT NULL AND w.name <> ''
AND (w.amenity IS NOT NULL OR w.tourism IS NOT NULL OR w.shop IS NOT NULL
OR w.leisure IS NOT NULL OR w.historic IS NOT NULL)
ORDER BY w.osm_id
) w
) )
SELECT p_extract, osm_id, t, name, SELECT p_extract, osm_id, t, name,
COALESCE( COALESCE(