PostGIS: live — imports, spatiald, and fixes from real runs
Ran the full stack (docker now available via sg docker): - postgis/postgis:16-3.4 up, both extracts imported: northeast 291,000 POIs (from ~/osm-build/data/northeast.osm.pbf), colombia 228,446 POIs - spatiald live on :5005, proxied by the mock server (/spatial/*, /spatial-status); UI badge + poi_near tool activate automatically Fixes discovered by running against real data: - osm2pgsql image: osm2pgsql/osm2pgsql does not exist on Docker Hub — use iboates/osm2pgsql, bypass its DB-probing entrypoint (--entrypoint osm2pgsql), password via PGPASSWORD (this build's --password flag forces an interactive prompt) - osm2pgsql 2.x schema: planet_osm_point/_polygon (not _node/_way), geometry in EPSG:3857 'way' column, -k hstore for opening_hours/ fee/website/addr:*, refresh_poi() dedupes relation polygons and uses hstore -> (not ->>) - compose file downgraded to v1-compatible 3.8 (host has docker-compose 1.29, no v2 plugin); OSM_DIR exported by import.sh - spatiald: proper SQL parameterization (@@i@@ tokens, body .. shifted past the filter args — the first renumber attempt was unsound), ST_Distance(geography) instead of the geometry-only ST_Distance_Sphere, 4-arg ST_DWithin for the corridor band, WKT in lng lat order, kind= accepts exact (amenity=restaurant), bare tag value (restaurant) or family (tourism) - mock server /spatial proxy: forward u.search (was dropping it) - app.js poi_near: 'points' param, semicolons must be %3B-encoded (Go url.Parse drops the tail of a value containing a raw ';') - README: osm2pgsql 2.x data notes + kind semantics
This commit is contained in:
parent
3da04c111e
commit
d05ebfa72f
|
|
@ -408,7 +408,7 @@ const TOOLS = {
|
|||
let qs = `lat=${lat}&lng=${lng}&r=${r}`;
|
||||
if (a.kind) qs += '&kind=' + encodeURIComponent(a.kind); // e.g. amenity=restaurant, tourism=museum
|
||||
if (a.name) qs += '&name=' + encodeURIComponent(a.name);
|
||||
if (a.corridor) qs += '&corridor=' + encodeURIComponent(a.corridor); // 'lng,lat;lng,lat;…' route polyline
|
||||
if (a.corridor) qs += '&points=' + encodeURIComponent(a.corridor); // 'lng,lat;lng,lat;…' route polyline (encoded: Go parses raw ';' as a param separator)
|
||||
const rj = await fetch('/spatial/' + (a.corridor ? 'corridor' : 'near') + '?' + qs);
|
||||
const j = await rj.json();
|
||||
if (j.error) return { error: j.error };
|
||||
|
|
|
|||
|
|
@ -435,7 +435,7 @@ http.createServer((req, res) => {
|
|||
if (u.pathname === '/spatial-status' || u.pathname.startsWith('/spatial/')) {
|
||||
const up = u.pathname === '/spatial-status'
|
||||
? SPATIAL.url + '/health'
|
||||
: SPATIAL.url + u.pathname.slice('/spatial'.length); // keep query string
|
||||
: SPATIAL.url + u.pathname.slice('/spatial'.length) + (u.search || '');
|
||||
const isProbe = u.pathname === '/spatial-status';
|
||||
fetch(up, { signal: AbortSignal.timeout(15000) }).then(async r => {
|
||||
const t = await r.text();
|
||||
|
|
|
|||
109
spatial/.nfs000000000016f2b2000000e7
Normal file
109
spatial/.nfs000000000016f2b2000000e7
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
-- PostGIS schema for the trip planner (runs once, on first container boot).
|
||||
-- osm2pgsql loads the standard planet_osm_* tables; we then materialize the
|
||||
-- POI points we query on, with a GIST index for ST_DWithin / KNN.
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS postgis;
|
||||
-- osm2pgsql --hstore puts every tag that has no dedicated column here
|
||||
CREATE EXTENSION IF NOT EXISTS hstore;
|
||||
|
||||
-- Named data sets (so multi-extract setups — e.g. northeast + colombia — can
|
||||
-- be imported into one DB and disambiguated).
|
||||
CREATE TABLE IF NOT EXISTS spatial_extract (
|
||||
name text PRIMARY KEY,
|
||||
imported_at timestamptz NOT NULL DEFAULT now(),
|
||||
bounds box, -- from osmium (in degrees)
|
||||
poi_count bigint
|
||||
);
|
||||
|
||||
-- Searchable POIs: every named point/polygon of the interesting kinds.
|
||||
-- osm2pgsql 2.x pgsql output schema: planet_osm_point / planet_osm_polygon,
|
||||
-- geometry in EPSG:3857 in a column called `way`; tags without a dedicated
|
||||
-- column (opening_hours, fee, website, addr:*) live in the hstore `tags`.
|
||||
CREATE TABLE IF NOT EXISTS poi (
|
||||
extract text NOT NULL REFERENCES spatial_extract(name) ON DELETE CASCADE,
|
||||
osm_id bigint NOT NULL,
|
||||
osm_type char NOT NULL, -- 'N' | 'W'
|
||||
name text NOT NULL,
|
||||
kind text NOT NULL, -- dominant tag: amenity=, tourism=, …
|
||||
amenity text,
|
||||
tourism text,
|
||||
shop text,
|
||||
leisure text,
|
||||
historic text,
|
||||
place text,
|
||||
opening_hours text,
|
||||
fee text, -- OSM fee tag / fee:* values
|
||||
website text,
|
||||
addr_city text,
|
||||
geom geometry(Point, 4326) NOT NULL, -- reprojected from 3857
|
||||
PRIMARY KEY (extract, osm_type, osm_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS poi_geom_gist ON poi USING GIST (geom);
|
||||
CREATE INDEX IF NOT EXISTS poi_kind ON poi (extract, kind);
|
||||
CREATE INDEX IF NOT EXISTS poi_name_trgm ON poi USING GIN (to_tsvector('simple', name));
|
||||
|
||||
-- pgvector (semantic POI index) is a later milestone: it needs the vector
|
||||
-- package in the image (postgis/postgis doesn't ship it), so it gets its own
|
||||
-- init script + embedding importer instead of living in this base schema.
|
||||
|
||||
-- Helper: refresh the poi table from a freshly loaded extract.
|
||||
-- (Invoked by import.sh; kept here so the SQL lives in one place.)
|
||||
CREATE OR REPLACE FUNCTION refresh_poi(p_extract text) RETURNS void AS $$
|
||||
BEGIN
|
||||
DELETE FROM poi WHERE extract = p_extract;
|
||||
INSERT INTO poi (extract, osm_id, osm_type, name, kind, amenity, tourism, shop,
|
||||
leisure, historic, place, opening_hours, fee, website, addr_city, geom)
|
||||
WITH cand AS (
|
||||
SELECT 'N'::char AS t, n.osm_id, n.name, n.amenity, n.tourism, n.shop,
|
||||
n.leisure, n.historic, n.place,
|
||||
n.tags->>'opening_hours' AS opening_hours,
|
||||
COALESCE(n.tags->>'fee', n.tags->>'fee:tourism') AS fee,
|
||||
COALESCE(n.tags->>'website', n.tags->>'url') AS website,
|
||||
n.tags->>'addr:city' AS addr_city,
|
||||
ST_Transform(n.way, 4326) AS geom
|
||||
FROM planet_osm_point n
|
||||
WHERE n.name IS NOT NULL AND n.name <> ''
|
||||
AND (n.amenity IS NOT NULL OR n.tourism IS NOT NULL OR n.shop IS NOT NULL
|
||||
OR n.leisure IS NOT NULL OR n.historic IS NOT NULL OR n.place IS NOT NULL)
|
||||
UNION ALL
|
||||
SELECT 'W'::char, w.osm_id, w.name, w.amenity, w.tourism, w.shop,
|
||||
w.leisure, w.historic, w.place,
|
||||
w.tags->>'opening_hours',
|
||||
COALESCE(w.tags->>'fee', w.tags->>'fee:tourism'),
|
||||
COALESCE(w.tags->>'website', w.tags->>'url'),
|
||||
w.tags->>'addr:city',
|
||||
ST_Centroid(ST_Transform(w.way, 4326))
|
||||
FROM planet_osm_polygon w
|
||||
WHERE w.name IS NOT NULL AND w.name <> ''
|
||||
AND (w.amenity IS NOT NULL OR w.tourism IS NOT NULL OR w.shop IS NOT NULL
|
||||
OR w.leisure IS NOT NULL OR w.historic IS NOT NULL)
|
||||
)
|
||||
SELECT p_extract, osm_id, t, name,
|
||||
COALESCE(
|
||||
CASE WHEN amenity IS NOT NULL THEN 'amenity=' || amenity
|
||||
END,
|
||||
CASE WHEN tourism IS NOT NULL THEN 'tourism=' || tourism
|
||||
END,
|
||||
CASE WHEN shop IS NOT NULL THEN 'shop=' || shop
|
||||
END,
|
||||
CASE WHEN leisure IS NOT NULL THEN 'leisure=' || leisure
|
||||
END,
|
||||
CASE WHEN historic IS NOT NULL THEN 'historic=' || historic
|
||||
END,
|
||||
CASE WHEN place IS NOT NULL THEN 'place=' || place
|
||||
END
|
||||
),
|
||||
amenity, tourism, shop, leisure, historic, place,
|
||||
opening_hours, fee, website, addr_city, geom
|
||||
FROM cand
|
||||
WHERE COALESCE(amenity, tourism, shop, leisure, historic, place) IS NOT NULL;
|
||||
|
||||
UPDATE spatial_extract
|
||||
SET poi_count = (SELECT count(*) FROM poi WHERE extract = p_extract),
|
||||
imported_at = now()
|
||||
WHERE name = p_extract;
|
||||
|
||||
ANALYZE poi;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
|
@ -8,7 +8,7 @@ nearest, bbox)"). Replaces the flat-file/Overpass approximations.
|
|||
|
||||
| Piece | What it is |
|
||||
|---|---|
|
||||
| `docker-compose.yml` | `postgis/postgis:16-3.4` (DB, port 5432) + one-shot `osm2pgsql:16` importer + `spatiald` (query service, port 5005) |
|
||||
| `docker-compose.yml` | `postgis/postgis:16-3.4` (DB, port 5432) + one-shot `iboates/osm2pgsql` importer + `spatiald` (query service, port 5005) |
|
||||
| `schema.sql` | runs on first boot: `poi` table (named POIs as points, GIST index), `spatial_extract` bookkeeping, `refresh_poi()` |
|
||||
| `import.sh` | loads a PBF from `../osm/` via osm2pgsql, refreshes `poi` |
|
||||
| `main.go` (spatiald) | JSON query service over `poi` |
|
||||
|
|
@ -24,10 +24,10 @@ nearest, bbox)"). Replaces the flat-file/Overpass approximations.
|
|||
## Usage
|
||||
|
||||
```bash
|
||||
docker compose up -d postgis # first: runs schema.sql
|
||||
./import.sh ~/trips/osm/nh.osm.pbf nh # load New England (default)
|
||||
docker-compose up -d postgis # first: runs schema.sql
|
||||
./import.sh ~/trips/osm/colombia.osm.pbf colombia
|
||||
docker compose up -d spatiald # query service on :5005
|
||||
./import.sh ~/osm-build/data/northeast.osm.pbf northeast # merged NE extract
|
||||
docker-compose up -d spatiald # query service on :5005
|
||||
```
|
||||
|
||||
The mock server (`mock/server.js`) proxies it: `GET /spatial/*` → `:5005`,
|
||||
|
|
@ -61,13 +61,20 @@ curl 'localhost:5005/corridor?r=300&kind=fuel&points=-71.06,42.35;-71.07,42.36'
|
|||
|
||||
## Data notes
|
||||
|
||||
* `poi` covers every **named** node (or polygon way, via centroid) with an
|
||||
`amenity`/`tourism`/`shop`/`leisure`/`historic`/`place` tag — the kinds the
|
||||
planner asks about. Unnamed amenities (e.g. a nameless kiosk) are out of
|
||||
scope for v1.
|
||||
* `opening_hours`, `fee`, `website`, `addr_city` are carried straight from
|
||||
the OSM tags; the live web enrichment (SearXNG, `mock/app.js`) fills gaps
|
||||
the tags don't have (the 3-source blend).
|
||||
* Built for **osm2pgsql 2.x** (`iboates/osm2pgsql`): tables are
|
||||
`planet_osm_point` / `planet_osm_polygon` (not the 1.x `_node`/`_way`),
|
||||
geometry lives in a `way` column in EPSG:3857, and the import runs with
|
||||
`-k` so tags without a fixed column (`opening_hours`, `fee`, `website`,
|
||||
`addr:city`) land in the hstore `tags` column that `refresh_poi()` reads.
|
||||
`refresh_poi()` also dedupes relation-derived polygons (2.x repeats them
|
||||
per relation with negated ids).
|
||||
* `poi` covers every **named** point/polygon with an `amenity`/`tourism`/
|
||||
`shop`/`leisure`/`historic`/`place` tag. Unnamed amenities (a nameless
|
||||
kiosk) are out of scope for v1.
|
||||
* `kind` filter semantics: `kind=amenity=restaurant` (exact),
|
||||
`kind=restaurant` (tag value, any family), `kind=tourism` (whole family).
|
||||
* Corridor `points` must URL-encode the semicolons (`%3B`) — Go's
|
||||
`url.Parse` drops the tail of a value that contains a raw `;`.
|
||||
* Multiple extracts coexist in one DB, disambiguated by `poi.extract`
|
||||
(set automatically by `import.sh`).
|
||||
* pgvector (semantic POI index, later milestone) gets its own init script +
|
||||
|
|
|
|||
|
|
@ -1,14 +1,17 @@
|
|||
# PostGIS spatial stack for the trip planner.
|
||||
#
|
||||
# docker compose up -d postgis # start the DB (schema.sql runs on first boot)
|
||||
# ./import.sh # import ~/trips/osm/nh.osm.pbf (one-shot osm2pgsql)
|
||||
# docker-compose up -d postgis # start the DB (schema.sql runs on first boot)
|
||||
# ./import.sh ~/trips/osm/nh.osm.pbf nh # import an OSM extract (one-shot osm2pgsql)
|
||||
# docker-compose up -d spatiald # start the query service on :5005
|
||||
#
|
||||
# Composed for docker-compose v1 (3.8) AND v2 (the same file works with
|
||||
# `docker compose` once the v2 plugin is available): no healthchecks or
|
||||
# `depends_on` conditions — import.sh polls pg_isready itself, and spatiald
|
||||
# tolerates a not-yet-ready DB at boot.
|
||||
#
|
||||
# OSM_DIR (default ~/trips/osm) is where the PBF extracts live — outside the
|
||||
# dev tree. Export it to override.
|
||||
# docker compose run --rm spatiald # start the query service on :5005
|
||||
#
|
||||
# The PBF extracts live in ../osm (gitignored). osm2pgsql runs as a one-shot
|
||||
# container so the host needs no osm2pgsql/osmium installation.
|
||||
version: "3.8"
|
||||
services:
|
||||
postgis:
|
||||
image: postgis/postgis:16-3.4
|
||||
|
|
@ -23,30 +26,23 @@ services:
|
|||
- pgdata:/var/lib/postgresql/data
|
||||
# initdb scripts run in lexicographic order, once, on an empty volume
|
||||
- ./schema.sql:/docker-entrypoint-initdb.d/10-schema.sql:ro
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U trips -d trips"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
restart: unless-stopped
|
||||
|
||||
# One-shot importer. `docker compose run --rm importer` (or import.sh)
|
||||
# loads the PBF named in PBF_FILE into the DB. The osm2pgsql image runs
|
||||
# as uid 1001 and needs the PBF readable — import.sh bind-mounts ../osm.
|
||||
# One-shot importer (also usable as: docker-compose run --rm importer
|
||||
# -- /data/<file>.osm.pbf ...). import.sh does the same with a bare
|
||||
# `docker run`, since the PBF path varies per import.
|
||||
importer:
|
||||
image: osm2pgsql/osm2pgsql:16
|
||||
user: "1001:1001"
|
||||
network_mode: "service:postgis"
|
||||
image: iboates/osm2pgsql:latest
|
||||
entrypoint: osm2pgsql
|
||||
environment:
|
||||
PBF_FILE: /data/nh.osm.pbf
|
||||
# --create --clean: fresh load every time (extracts are small enough);
|
||||
# slim mode keeps the RAM footprint sane; disable-operations saves time.
|
||||
ARGS: >-
|
||||
--host=postgis --port=5432 --database=trips --user=trips
|
||||
--password=trips --create --clean --slim
|
||||
--flat-nodes=/tmp/flat.nodes --disable-operations
|
||||
--import-strips=10
|
||||
PGPASSWORD: trips # the image build's --password flag forces a prompt
|
||||
command: >-
|
||||
/data/colombia.osm.pbf --host=postgis --port=5432 --database=trips
|
||||
--user=trips -k --slim
|
||||
--flat-nodes=/tmp/flat.nodes
|
||||
network_mode: "service:postgis"
|
||||
volumes:
|
||||
- ${OSM_DIR:-$HOME/trips/osm}:/data:ro
|
||||
- ${OSM_DIR}:/data:ro # OSM_DIR is exported by import.sh (default ~/trips/osm)
|
||||
|
||||
spatiald:
|
||||
build: .
|
||||
|
|
@ -56,8 +52,8 @@ services:
|
|||
ports:
|
||||
- "5005:5005"
|
||||
depends_on:
|
||||
postgis:
|
||||
condition: service_healthy
|
||||
- postgis
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
|
|
|
|||
|
|
@ -3,52 +3,72 @@
|
|||
#
|
||||
# ./import.sh [file.osm.pbf] [extract-name]
|
||||
#
|
||||
# file.osm.pbf default: $OSM_DIR/nh.osm.pbf (OSM_DIR defaults to
|
||||
# ~/trips/osm — the PBFs live outside the dev tree)
|
||||
# file.osm.pbf default: $OSM_DIR/colombia.osm.pbf (OSM_DIR defaults to
|
||||
# ~/trips/osm — the PBFs live outside the dev tree).
|
||||
# The merged New England file lives at
|
||||
# $HOME/osm-build/data/northeast.osm.pbf (setup-osrm.sh);
|
||||
# any absolute path works, e.g.:
|
||||
# ./import.sh ~/osm-build/data/northeast.osm.pbf northeast
|
||||
# extract-name default: basename without extension
|
||||
#
|
||||
# The PBF must be readable by uid 1001 (the osm2pgsql image user) — the
|
||||
# script bind-mounts its directory read-only. After the load, the poi
|
||||
# table is refreshed (refresh_poi()) and the spatial_extract row updated.
|
||||
# The osm2pgsql container reaches the DB through the host-published port
|
||||
# 5432 (see docker-compose.yml), so no shared compose network is needed.
|
||||
# After the load, the poi table is refreshed (refresh_poi()) and the
|
||||
# spatial_extract row updated.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
# docker-compose v1 (python) or v2 (plugin) — whichever the host has
|
||||
DC=docker-compose
|
||||
command -v docker >/dev/null && docker compose version >/dev/null 2>&1 && DC="docker compose"
|
||||
|
||||
# osm2pgsql image: iboates/osm2pgsql (the osm2pgsql/osm2pgsql repo image does
|
||||
# not exist on Docker Hub). Its /entrypoint.sh does its own DB probing, so we
|
||||
# bypass it and exec osm2pgsql directly.
|
||||
IMG=${OSM2PGSQL_IMAGE:-iboates/osm2pgsql:latest}
|
||||
|
||||
OSM_DIR="${OSM_DIR:-$HOME/trips/osm}"
|
||||
PBF=${1:-$OSM_DIR/nh.osm.pbf}
|
||||
export OSM_DIR
|
||||
PBF=${1:-$OSM_DIR/colombia.osm.pbf}
|
||||
NAME=${2:-$(basename "${PBF%.osm.pbf}")}
|
||||
ABS=$(realpath "$PBF")
|
||||
[[ -f $ABS ]] || { echo "no such PBF: $ABS" >&2; exit 1; }
|
||||
[[ -f $ABS && $(stat -c%s "$ABS") -gt 100000 ]] || { echo "no such PBF: $ABS" >&2; exit 1; }
|
||||
|
||||
# 1. DB up (first boot runs schema.sql)
|
||||
docker compose up -d postgis
|
||||
echo "waiting for postgis to be healthy…"
|
||||
$DC up -d postgis
|
||||
echo "waiting for postgis to be ready…"
|
||||
for i in $(seq 1 60); do
|
||||
if docker compose exec -T postgis pg_isready -U trips -d trips >/dev/null 2>&1; then break; fi
|
||||
if $DC exec -T postgis pg_isready -U trips -d trips >/dev/null 2>&1; then break; fi
|
||||
sleep 2
|
||||
[[ $i -eq 60 ]] && { echo "postgis never became healthy" >&2; exit 1; }
|
||||
[[ $i -eq 60 ]] && { echo "postgis never became ready" >&2; exit 1; }
|
||||
done
|
||||
|
||||
# 2. osm2pgsql load (one-shot container, PBF bind-mounted ro)
|
||||
# 2. osm2pgsql load (one-shot container, PBF bind-mounted ro), on the
|
||||
# postgis compose network. Note: this osm2pgsql build's --password flag
|
||||
# FORCES an interactive prompt, so the password goes via PGPASSWORD.
|
||||
# --create is the 2.x default; the 1.x ops machinery is gone in 2.x.
|
||||
NET=$(docker inspect trips-postgis | python3 -c \
|
||||
"import json,sys; print(next(iter(json.load(sys.stdin)[0]['NetworkSettings']['Networks'])))")
|
||||
docker run --rm \
|
||||
--user 1001:1001 \
|
||||
--network "trips_default" \
|
||||
--entrypoint osm2pgsql \
|
||||
--network "$NET" \
|
||||
-e PGPASSWORD=trips \
|
||||
-v "$(dirname "$ABS"):/data:ro" \
|
||||
-e PBF_FILE="/data/$(basename "$ABS")" \
|
||||
-e ARGS="--host=postgis --port=5432 --database=trips --user=trips --password=trips \
|
||||
--create --clean --slim --flat-nodes=/tmp/flat.nodes --disable-operations" \
|
||||
osm2pgsql/osm2pgsql:16
|
||||
"$IMG" "/data/$(basename "$ABS")" \
|
||||
--host=postgis --port=5432 --database=trips --user=trips \
|
||||
-k --slim --flat-nodes=/tmp/flat.nodes # -k: unmatched tags → hstore `tags`
|
||||
|
||||
# 3. refresh the queryable poi table + extract bookkeeping
|
||||
docker compose exec -T postgis psql -U trips -d trips -v ON_ERROR_STOP=1 <<SQL
|
||||
$DC exec -T postgis psql -U trips -d trips -v ON_ERROR_STOP=1 <<SQL
|
||||
INSERT INTO spatial_extract (name) VALUES ('${NAME}')
|
||||
ON CONFLICT (name) DO NOTHING;
|
||||
SELECT refresh_poi('${NAME}');
|
||||
SQL
|
||||
|
||||
# 4. report
|
||||
docker compose exec -T postgis psql -U trips -d trips -c \
|
||||
$DC exec -T postgis psql -U trips -d trips -c \
|
||||
"SELECT name, poi_count, to_char(imported_at,'YYYY-MM-DD HH24:MI') AS imported
|
||||
FROM spatial_extract ORDER BY name;"
|
||||
|
||||
echo "done — extract '$NAME' is queryable. Start the service:"
|
||||
echo " docker compose up -d spatiald # :5005, proxied by the mock server at /spatial/*"
|
||||
echo " $DC up -d spatiald # :5005, proxied by the mock server at /spatial/*"
|
||||
|
|
|
|||
128
spatial/main.go
128
spatial/main.go
|
|
@ -26,6 +26,7 @@ import (
|
|||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
|
@ -57,6 +58,9 @@ func main() {
|
|||
"PostgreSQL/PostGIS DSN")
|
||||
addrFlag := flag.String("addr", ":5005", "listen address")
|
||||
flag.Parse()
|
||||
if env := os.Getenv("SPATIAL_DSN"); env != "" {
|
||||
*dsnFlag = env // docker-compose sets the in-network DSN
|
||||
}
|
||||
addr = *addrFlag
|
||||
|
||||
var err error
|
||||
|
|
@ -95,31 +99,50 @@ func jerr(w http.ResponseWriter, code int, msg string) {
|
|||
|
||||
// common filters → (where clause, args). kind accepts "amenity=restaurant"
|
||||
// (exact) or "restaurant" (match the tag value across all kind families).
|
||||
func filters(q map[string][]string) (string, []any, error) {
|
||||
// filters → (WHERE clause with @@i@@ placeholder tokens, args). Tokens are
|
||||
// resolved to $i by renumber(); the query body's own $1..$4 placeholders
|
||||
// would otherwise collide with the filter params.
|
||||
func filters(q map[string][]string) (string, []any) {
|
||||
var conds []string
|
||||
var args []any
|
||||
param := func(arg any) string {
|
||||
args = append(args, arg)
|
||||
return "@@" + strconv.Itoa(len(args)) + "@@"
|
||||
}
|
||||
if e := first(q, "extract"); e != "" {
|
||||
conds = append(conds, "poi.extract = $%d")
|
||||
args = append(args, e)
|
||||
conds = append(conds, "poi.extract = "+param(e))
|
||||
}
|
||||
if k := first(q, "kind"); k != "" {
|
||||
if strings.Contains(k, "=") {
|
||||
conds = append(conds, "poi.kind = $%d")
|
||||
args = append(args, k)
|
||||
conds = append(conds, "poi.kind = "+param(k))
|
||||
} else if isFamily(k) {
|
||||
// bare family name: the whole family ("tourism", "leisure", …)
|
||||
conds = append(conds, "poi.kind LIKE '"+k+"=%'")
|
||||
} else {
|
||||
// bare value: match the part after '=' in any kind family
|
||||
conds = append(conds, "poi.kind LIKE $%d")
|
||||
args = append(args, "%="+k)
|
||||
// bare tag value: exact match in any kind family
|
||||
v := param(k)
|
||||
var parts []string
|
||||
for _, fam := range []string{"amenity", "tourism", "shop", "leisure", "historic", "place"} {
|
||||
parts = append(parts, "poi.kind = '"+fam+"='||"+v)
|
||||
}
|
||||
conds = append(conds, "("+strings.Join(parts, " OR ")+")")
|
||||
}
|
||||
}
|
||||
if n := first(q, "name"); n != "" {
|
||||
conds = append(conds, "poi.name ILIKE $%d")
|
||||
args = append(args, "%"+n+"%")
|
||||
conds = append(conds, "poi.name ILIKE "+param("%"+n+"%"))
|
||||
}
|
||||
if len(conds) == 0 {
|
||||
return "", args, nil
|
||||
return "", args
|
||||
}
|
||||
return " WHERE " + strings.Join(conds, " AND "), args, nil
|
||||
return " WHERE " + strings.Join(conds, " AND "), args
|
||||
}
|
||||
func isFamily(k string) bool {
|
||||
for _, f := range []string{"amenity", "tourism", "shop", "leisure", "historic", "place"} {
|
||||
if k == f {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func first(q map[string][]string, k string) string {
|
||||
|
|
@ -209,22 +232,17 @@ func handleNear(w http.ResponseWriter, r *http.Request) {
|
|||
jerr(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
where, args, _ := filters(q)
|
||||
where, args := filters(q)
|
||||
args = append(args, lat, lng, rad, limitOf(q))
|
||||
// point geometry first, then geography cast for the radius test, then KNN
|
||||
// ordering among hits — ST_DWithin(geography) uses the GIST index.
|
||||
query := `SELECT ` + poiCols + `,
|
||||
ST_Distance_Sphere(poi.geom, ST_SetSRID(ST_MakePoint($2, $1), 4326)) AS dist_m
|
||||
ST_Distance(poi.geom::geography, ST_SetSRID(ST_MakePoint($2, $1), 4326)::geography) AS dist_m
|
||||
FROM poi` + where + `
|
||||
AND ST_DWithin(poi.geom::geography, ST_SetSRID(ST_MakePoint($2, $1), 4326)::geography, $3)
|
||||
ORDER BY poi.geom <-> ST_SetSRID(ST_MakePoint($2, $1), 4326)
|
||||
LIMIT $4`
|
||||
// renumber: filters consume $1..$n, so shift the point/radius/limit params
|
||||
query, args = renumber(query, args, q)
|
||||
if query == "" {
|
||||
jerr(w, 500, "param renumbering failed")
|
||||
return
|
||||
}
|
||||
query = renumber(query, len(args)-4)
|
||||
rows, err := dsn.Query(query, args...)
|
||||
if err != nil {
|
||||
jerr(w, 502, "query: "+err.Error())
|
||||
|
|
@ -246,18 +264,14 @@ func handleNearest(w http.ResponseWriter, r *http.Request) {
|
|||
jerr(w, 400, err.Error())
|
||||
return
|
||||
}
|
||||
where, args, _ := filters(q)
|
||||
where, args := filters(q)
|
||||
args = append(args, lat, lng, limitOf(q))
|
||||
query := `SELECT ` + poiCols + `,
|
||||
ST_Distance_Sphere(poi.geom, ST_SetSRID(ST_MakePoint($2, $1), 4326)) AS dist_m
|
||||
ST_Distance(poi.geom::geography, ST_SetSRID(ST_MakePoint($2, $1), 4326)::geography) AS dist_m
|
||||
FROM poi` + where + `
|
||||
ORDER BY poi.geom <-> ST_SetSRID(ST_MakePoint($2, $1), 4326)
|
||||
LIMIT $3`
|
||||
query, args = renumber(query, args, q)
|
||||
if query == "" {
|
||||
jerr(w, 500, "param renumbering failed")
|
||||
return
|
||||
}
|
||||
query = renumber(query, len(args)-3)
|
||||
rows, err := dsn.Query(query, args...)
|
||||
if err != nil {
|
||||
jerr(w, 502, "query: "+err.Error())
|
||||
|
|
@ -284,10 +298,10 @@ func handleCorridor(w http.ResponseWriter, r *http.Request) {
|
|||
jerr(w, 400, "points must be 'lng,lat;lng,lat;…' with at least 2 points")
|
||||
return
|
||||
}
|
||||
where, args, _ := filters(q)
|
||||
where, args := filters(q)
|
||||
var coords []string
|
||||
for _, p := range pts {
|
||||
coords = append(coords, fmt.Sprintf("%s %s", strconv.FormatFloat(p[1], 'f', 6, 64), strconv.FormatFloat(p[0], 'f', 6, 64)))
|
||||
coords = append(coords, fmt.Sprintf("%s %s", strconv.FormatFloat(p[0], 'f', 6, 64), strconv.FormatFloat(p[1], 'f', 6, 64))) // WKT: lng lat
|
||||
}
|
||||
line := "LINESTRING(" + strings.Join(coords, ",") + ")"
|
||||
args = append(args, line, rad, limitOf(q))
|
||||
|
|
@ -297,14 +311,10 @@ func handleCorridor(w http.ResponseWriter, r *http.Request) {
|
|||
ST_Distance(poi.geom::geography, ST_SetSRID(ST_GeomFromText($1), 4326)::geography) AS dist_m
|
||||
FROM poi` + where + `
|
||||
AND ST_DWithin(poi.geom::geography,
|
||||
ST_Buffer(ST_SetSRID(ST_GeomFromText($1), 4326)::geography, $2), true)
|
||||
ST_Buffer(ST_SetSRID(ST_GeomFromText($1), 4326)::geography, $2), 0, true)
|
||||
ORDER BY dist_m
|
||||
LIMIT $3`
|
||||
query, args = renumber(query, args, q)
|
||||
if query == "" {
|
||||
jerr(w, 500, "param renumbering failed")
|
||||
return
|
||||
}
|
||||
query = renumber(query, len(args)-3)
|
||||
rows, err := dsn.Query(query, args...)
|
||||
if err != nil {
|
||||
jerr(w, 502, "query: "+err.Error())
|
||||
|
|
@ -368,38 +378,30 @@ func parsePoints(s string) [][2]float64 {
|
|||
return out
|
||||
}
|
||||
|
||||
// renumber: the WHERE clause (from filters) already contains $1..$n in the
|
||||
// order filters() appended its args. The trailing point/radius/limit params
|
||||
// were written as $1,$2,$3,$4 — rewrite them to $n+1, $n+2, … so the arg
|
||||
// list (filter args first, then point args) lines up.
|
||||
func renumber(query string, args []any, q map[string][]string) (string, []any) {
|
||||
n := 0
|
||||
if e := first(q, "extract"); e != "" {
|
||||
n++
|
||||
}
|
||||
if k := first(q, "kind"); k != "" {
|
||||
n++
|
||||
}
|
||||
if nn := first(q, "name"); nn != "" {
|
||||
n++
|
||||
}
|
||||
// renumber resolves the placeholder scheme: the query body was written with
|
||||
// $1..$4 for its own params (lat, lng, radius, limit) and the WHERE clause
|
||||
// carries @@i@@ tokens for the n filter args, which come FIRST in the arg
|
||||
// list. So: body $d → $(n+d), then @@i@@ → $i. n = number of filter args
|
||||
// (= len(args) minus the body params, known at the call site).
|
||||
func renumber(query string, n int) string {
|
||||
var b strings.Builder
|
||||
for i := 0; i < len(query); i++ {
|
||||
c := query[i]
|
||||
if c == '$' && i+1 < len(query) && query[i+1] >= '1' && query[i+1] <= '9' {
|
||||
// count-digit placeholders only — the filter params are < 10
|
||||
d := int(query[i+1] - '0')
|
||||
if d <= n {
|
||||
b.WriteByte(c)
|
||||
b.WriteByte(query[i+1])
|
||||
i++
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, "$%d", n+d)
|
||||
if query[i] == '$' && i+1 < len(query) && query[i+1] >= '1' && query[i+1] <= '4' {
|
||||
fmt.Fprintf(&b, "$%d", n+int(query[i+1]-'0'))
|
||||
i++
|
||||
continue
|
||||
}
|
||||
b.WriteByte(c)
|
||||
if strings.HasPrefix(query[i:], "@@") {
|
||||
end := strings.Index(query[i+2:], "@@")
|
||||
if end >= 0 {
|
||||
if d, err := strconv.Atoi(query[i+2 : i+2+end]); err == nil {
|
||||
fmt.Fprintf(&b, "$%d", d)
|
||||
i += 2 + end + 1 // loop's i++ finishes the last '@'
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
b.WriteByte(query[i])
|
||||
}
|
||||
return b.String(), args
|
||||
return b.String()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@
|
|||
-- POI points we query on, with a GIST index for ST_DWithin / KNN.
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS postgis;
|
||||
-- osm2pgsql --hstore puts every tag that has no dedicated column here
|
||||
CREATE EXTENSION IF NOT EXISTS hstore;
|
||||
|
||||
-- Named data sets (so multi-extract setups — e.g. northeast + colombia — can
|
||||
-- be imported into one DB and disambiguated).
|
||||
|
|
@ -13,8 +15,10 @@ CREATE TABLE IF NOT EXISTS spatial_extract (
|
|||
poi_count bigint
|
||||
);
|
||||
|
||||
-- Searchable POIs: every named node/way of the interesting kinds, as a point.
|
||||
-- osm2pgsql's default table schema (see README: --default-schema / stock).
|
||||
-- Searchable POIs: every named point/polygon of the interesting kinds.
|
||||
-- osm2pgsql 2.x pgsql output schema: planet_osm_point / planet_osm_polygon,
|
||||
-- geometry in EPSG:3857 in a column called `way`; tags without a dedicated
|
||||
-- column (opening_hours, fee, website, addr:*) live in the hstore `tags`.
|
||||
CREATE TABLE IF NOT EXISTS poi (
|
||||
extract text NOT NULL REFERENCES spatial_extract(name) ON DELETE CASCADE,
|
||||
osm_id bigint NOT NULL,
|
||||
|
|
@ -31,7 +35,7 @@ CREATE TABLE IF NOT EXISTS poi (
|
|||
fee text, -- OSM fee tag / fee:* values
|
||||
website text,
|
||||
addr_city text,
|
||||
geom geometry(Point, 4326) NOT NULL,
|
||||
geom geometry(Point, 4326) NOT NULL, -- reprojected from 3857
|
||||
PRIMARY KEY (extract, osm_type, osm_id)
|
||||
);
|
||||
|
||||
|
|
@ -52,21 +56,34 @@ BEGIN
|
|||
leisure, historic, place, opening_hours, fee, website, addr_city, geom)
|
||||
WITH cand AS (
|
||||
SELECT 'N'::char AS t, n.osm_id, n.name, n.amenity, n.tourism, n.shop,
|
||||
n.leisure, n.historic, n.place, n.opening_hours, n.fee, n.website,
|
||||
n.addr_city, ST_SetSRID(ST_MakePoint(n.lon, n.lat), 4326) AS geom
|
||||
FROM planet_osm_node n
|
||||
n.leisure, n.historic, n.place,
|
||||
n.tags->'opening_hours' AS opening_hours,
|
||||
COALESCE(n.tags->'fee', n.tags->'fee:tourism') AS fee,
|
||||
COALESCE(n.tags->'website', n.tags->'url') AS website,
|
||||
n.tags->'addr:city' AS addr_city,
|
||||
ST_Transform(n.way, 4326) AS geom
|
||||
FROM planet_osm_point n
|
||||
WHERE n.name IS NOT NULL AND n.name <> ''
|
||||
AND (n.amenity IS NOT NULL OR n.tourism IS NOT NULL OR n.shop IS NOT NULL
|
||||
OR n.leisure IS NOT NULL OR n.historic IS NOT NULL OR n.place IS NOT NULL)
|
||||
UNION ALL
|
||||
SELECT 'W'::char, w.osm_id, w.name, w.amenity, w.tourism, w.shop,
|
||||
w.leisure, w.historic, w.place, w.opening_hours, w.fee, w.website,
|
||||
w.addr_city, ST_Centroid(w.geom)
|
||||
FROM planet_osm_way w
|
||||
WHERE w.name IS NOT NULL AND w.name <> ''
|
||||
AND (w.amenity IS NOT NULL OR w.tourism IS NOT NULL OR w.shop IS NOT NULL
|
||||
OR w.leisure IS NOT NULL OR w.historic IS NOT NULL)
|
||||
AND ST_GeometryType(w.geom) = 'ST_Polygon' -- areas only (gardens, parks…)
|
||||
-- relation-derived polygons repeat per relation (osm_id negated) —
|
||||
-- keep one row per way id
|
||||
SELECT * FROM (
|
||||
SELECT DISTINCT ON (w.osm_id)
|
||||
'W'::char AS t, w.osm_id, w.name, w.amenity, w.tourism, w.shop,
|
||||
w.leisure, w.historic, w.place,
|
||||
w.tags->'opening_hours' AS opening_hours,
|
||||
COALESCE(w.tags->'fee', w.tags->'fee:tourism') AS fee,
|
||||
COALESCE(w.tags->'website', w.tags->'url') AS website,
|
||||
w.tags->'addr:city' AS addr_city,
|
||||
ST_Centroid(ST_Transform(w.way, 4326)) AS geom
|
||||
FROM planet_osm_polygon w
|
||||
WHERE w.name IS NOT NULL AND w.name <> ''
|
||||
AND (w.amenity IS NOT NULL OR w.tourism IS NOT NULL OR w.shop IS NOT NULL
|
||||
OR w.leisure IS NOT NULL OR w.historic IS NOT NULL)
|
||||
ORDER BY w.osm_id
|
||||
) w
|
||||
)
|
||||
SELECT p_extract, osm_id, t, name,
|
||||
COALESCE(
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user