Greg Pomerantz
d192b1dcab
The embedding approach was overkill: it embedded only 'name — kind' (short strings), spent a 4B model + ~2.5GB VRAM + 4.6GB of vectors + a one-time 450k-row job to do what indexed SQL does directly. New default — no embedding model, no extra VRAM: - The LLM agent is the semantic layer: it maps the user's concept to OSM kinds + local-language name keywords. It is already in VRAM for chat, so this costs nothing. - /kinds: returns the tag vocabulary that actually exists (GROUP BY kind with counts) so the agent grounds its choices in real data. - /search: indexed retrieval — kind IN/ILIKE (poi_kind), name FTS (to_tsvector) + trigram (pg_trgm) for fuzzy/substring, optional ST_DWithin radius. Ranked by trigram similarity then distance. - schema.sql: real trigram GIN index (poi_name_trgm_ops); renamed the misnamed FTS index to poi_name_fts. - Agent tools: poi_semantic -> poi_kinds + poi_search (both pin results on the map). pgvector demoted to an opt-in path (embed/ + /semantic) — still works if poi_vec is built, but no longer the default. Dropped the half-built poi_vec and stopped the background embed run.
119 lines
5.5 KiB
PL/PgSQL
119 lines
5.5 KiB
PL/PgSQL
-- PostGIS schema for the trip planner (runs once, on first container boot).
|
|
-- osm2pgsql loads the standard planet_osm_* tables; we then materialize the
|
|
-- POI points we query on, with a GIST index for ST_DWithin / KNN.
|
|
|
|
CREATE EXTENSION IF NOT EXISTS postgis;
|
|
-- osm2pgsql --hstore puts every tag that has no dedicated column here
|
|
CREATE EXTENSION IF NOT EXISTS hstore;
|
|
|
|
-- Named data sets (so multi-extract setups — e.g. northeast + colombia — can
|
|
-- be imported into one DB and disambiguated).
|
|
CREATE TABLE IF NOT EXISTS spatial_extract (
|
|
name text PRIMARY KEY,
|
|
imported_at timestamptz NOT NULL DEFAULT now(),
|
|
bounds box, -- from osmium (in degrees)
|
|
poi_count bigint
|
|
);
|
|
|
|
-- Searchable POIs: every named point/polygon of the interesting kinds.
|
|
-- osm2pgsql 2.x pgsql output schema: planet_osm_point / planet_osm_polygon,
|
|
-- geometry in EPSG:3857 in a column called `way`; tags without a dedicated
|
|
-- column (opening_hours, fee, website, addr:*) live in the hstore `tags`.
|
|
CREATE TABLE IF NOT EXISTS poi (
|
|
extract text NOT NULL REFERENCES spatial_extract(name) ON DELETE CASCADE,
|
|
osm_id bigint NOT NULL,
|
|
osm_type char NOT NULL, -- 'N' | 'W'
|
|
name text NOT NULL,
|
|
kind text NOT NULL, -- dominant tag: amenity=, tourism=, …
|
|
amenity text,
|
|
tourism text,
|
|
shop text,
|
|
leisure text,
|
|
historic text,
|
|
place text,
|
|
opening_hours text,
|
|
fee text, -- OSM fee tag / fee:* values
|
|
website text,
|
|
addr_city text,
|
|
geom geometry(Point, 4326) NOT NULL, -- reprojected from 3857
|
|
PRIMARY KEY (extract, osm_type, osm_id)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS poi_geom_gist ON poi USING GIST (geom);
|
|
CREATE INDEX IF NOT EXISTS poi_kind ON poi (extract, kind);
|
|
-- name matching for the /search endpoint, two complementary indexes:
|
|
-- * word-level full-text (to_tsvector) — exact word hits
|
|
CREATE INDEX IF NOT EXISTS poi_name_fts ON poi USING GIN (to_tsvector('simple', name));
|
|
-- * trigram (fuzzy / substring, language-agnostic) — needs pg_trgm
|
|
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
|
CREATE INDEX IF NOT EXISTS poi_name_trgm_ops ON poi USING gin (name gin_trgm_ops);
|
|
|
|
-- pgvector (opt-in semantic index) is separate: 20-vector.sql + embed/.
|
|
|
|
-- Helper: refresh the poi table from a freshly loaded extract.
|
|
-- (Invoked by import.sh; kept here so the SQL lives in one place.)
|
|
CREATE OR REPLACE FUNCTION refresh_poi(p_extract text) RETURNS void AS $$
|
|
BEGIN
|
|
DELETE FROM poi WHERE extract = p_extract;
|
|
INSERT INTO poi (extract, osm_id, osm_type, name, kind, amenity, tourism, shop,
|
|
leisure, historic, place, opening_hours, fee, website, addr_city, geom)
|
|
WITH cand AS (
|
|
SELECT 'N'::char AS t, n.osm_id, n.name, n.amenity, n.tourism, n.shop,
|
|
n.leisure, n.historic, n.place,
|
|
n.tags->'opening_hours' AS opening_hours,
|
|
COALESCE(n.tags->'fee', n.tags->'fee:tourism') AS fee,
|
|
COALESCE(n.tags->'website', n.tags->'url') AS website,
|
|
n.tags->'addr:city' AS addr_city,
|
|
ST_Transform(n.way, 4326) AS geom
|
|
FROM planet_osm_point n
|
|
WHERE n.name IS NOT NULL AND n.name <> ''
|
|
AND (n.amenity IS NOT NULL OR n.tourism IS NOT NULL OR n.shop IS NOT NULL
|
|
OR n.leisure IS NOT NULL OR n.historic IS NOT NULL OR n.place IS NOT NULL)
|
|
UNION ALL
|
|
-- relation-derived polygons repeat per relation (osm_id negated) —
|
|
-- keep one row per way id
|
|
SELECT * FROM (
|
|
SELECT DISTINCT ON (w.osm_id)
|
|
'W'::char AS t, w.osm_id, w.name, w.amenity, w.tourism, w.shop,
|
|
w.leisure, w.historic, w.place,
|
|
w.tags->'opening_hours' AS opening_hours,
|
|
COALESCE(w.tags->'fee', w.tags->'fee:tourism') AS fee,
|
|
COALESCE(w.tags->'website', w.tags->'url') AS website,
|
|
w.tags->'addr:city' AS addr_city,
|
|
ST_Centroid(ST_Transform(w.way, 4326)) AS geom
|
|
FROM planet_osm_polygon w
|
|
WHERE w.name IS NOT NULL AND w.name <> ''
|
|
AND (w.amenity IS NOT NULL OR w.tourism IS NOT NULL OR w.shop IS NOT NULL
|
|
OR w.leisure IS NOT NULL OR w.historic IS NOT NULL)
|
|
ORDER BY w.osm_id
|
|
) w
|
|
)
|
|
SELECT p_extract, osm_id, t, name,
|
|
COALESCE(
|
|
CASE WHEN amenity IS NOT NULL THEN 'amenity=' || amenity
|
|
END,
|
|
CASE WHEN tourism IS NOT NULL THEN 'tourism=' || tourism
|
|
END,
|
|
CASE WHEN shop IS NOT NULL THEN 'shop=' || shop
|
|
END,
|
|
CASE WHEN leisure IS NOT NULL THEN 'leisure=' || leisure
|
|
END,
|
|
CASE WHEN historic IS NOT NULL THEN 'historic=' || historic
|
|
END,
|
|
CASE WHEN place IS NOT NULL THEN 'place=' || place
|
|
END
|
|
),
|
|
amenity, tourism, shop, leisure, historic, place,
|
|
opening_hours, fee, website, addr_city, geom
|
|
FROM cand
|
|
WHERE COALESCE(amenity, tourism, shop, leisure, historic, place) IS NOT NULL;
|
|
|
|
UPDATE spatial_extract
|
|
SET poi_count = (SELECT count(*) FROM poi WHERE extract = p_extract),
|
|
imported_at = now()
|
|
WHERE name = p_extract;
|
|
|
|
ANALYZE poi;
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|