diff --git a/.gitignore b/.gitignore index d03370a..6a3bb50 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ router/router router/bench spatial/spatiald +spatial/spatial router/scripts/__pycache__/ # Runtime artifacts live OUTSIDE this source tree, in ~/trips: diff --git a/spatial/20-vector.sql b/spatial/20-vector.sql deleted file mode 100644 index 7954b39..0000000 --- a/spatial/20-vector.sql +++ /dev/null @@ -1,5 +0,0 @@ --- pgvector (runs on first boot, after 10-schema.sql; the `vector` package is --- baked into the db/ image — see db/Dockerfile). --- The poi_vec table itself is created by embedpoi (it needs to know the --- embedding dimension, which is detected from the model's first response). -CREATE EXTENSION IF NOT EXISTS vector; diff --git a/spatial/README.md b/spatial/README.md index 55cc9e1..886a0d9 100644 --- a/spatial/README.md +++ b/spatial/README.md @@ -8,13 +8,10 @@ nearest, bbox)"). Replaces the flat-file/Overpass approximations. | Piece | What it is | |---|---| -| `docker-compose.yml` | `postgis/postgis:16-3.4` + `postgresql-16-pgvector` (DB, port 5432) + one-shot `iboates/osm2pgsql` importer + `spatiald` (query service, port 5005) | +| `docker-compose.yml` | `postgis/postgis:16-3.4` (DB, port 5432) + one-shot `iboates/osm2pgsql` importer + `spatiald` (query service, port 5005) | | `schema.sql` | runs on first boot: `poi` table (named POIs as points, GIST index), `spatial_extract` bookkeeping, `refresh_poi()` | -| `20-vector.sql` | runs on first boot: `CREATE EXTENSION vector` (the `vector` package is baked into the DB image — see `db/Dockerfile`) | -| `db/Dockerfile` | `postgis/postgis:16-3.4` + `postgresql-16-pgvector` (workaround for the EOL bullseye repos — see the file) | | `import.sh` | loads a PBF from `../osm/` via osm2pgsql, refreshes `poi` | -| `main.go` (spatiald) | JSON query service over `poi` + `poi_vec` (radius / KNN / corridor / **semantic**) | -| `embed/main.go` (embedpoi) | builds the `poi_vec` semantic index from `poi` (zembed embeddings → pgvector, HNSW) | +| `main.go` (spatiald) | JSON query service over `poi` (radius / KNN / corridor / targeted tag+name search) | | `Dockerfile` | builds spatiald | ## Prerequisites @@ -56,7 +53,6 @@ fee,website,addr_city,dist_m},…]}`. Common filters: | `GET /corridor` | `points=lng,lat;…`, `r` | `ST_DWithin(geom::geography, ST_Buffer(line::geography, r))` | | `GET /search` | `kinds=a\|b`, `terms=x\|y`, optional `lat,lng,r` | indexed `kind` + FTS/trigram `name` match | | `GET /kinds` | `q?`, `extract?`, `limit` | `GROUP BY kind` — the tag vocabulary | -| `GET /semantic` | `q` (free text), `limit` | **opt-in**: cosine `<=>` over `poi_vec` (see below) | `/search` and `/kinds` are the agent's main POI lookups (see the next section). All spatial results carry `lat`/`lng` so the UI can drop map pins. @@ -94,35 +90,6 @@ This replaces the old pgvector approach for the agent: no embedding model, no extra VRAM, no ~4.6 GB vector table, no one-time 450k-row embed job — and it is arguably more accurate here, because `kind` is ground truth. -### Optional: pgvector semantic index (off by default) - -If you ever *do* want true free-text semantic search, the machinery is still -here but **not wired to the agent** by default. `poi_vec` mirrors `poi` for -named POIs and stores a 2560-dim zembed embedding of `"name — kind"`; build it -with `embedpoi` (resumable; rebuilds only if the embedding dim changed): - -```bash -go run ./embed -dsn "host=localhost port=5432 user=trips password=trips dbname=trips sslmode=disable" -go run ./embed -extract colombia -qps 2 -batch 64 # one extract, gentler on the host -``` - -Then `GET /semantic?q=…` works (it embeds `q` with the local model — -`EMBED_BASE` / `EMBED_MODEL` env, defaults `http://192.168.3.7:1234/v1` + -`zembed-1-Q4_K_M` — and cosine-ranks `poi_vec`). Note it needs the embedding -model resident in GPU VRAM while running, which is why targeted `/search` is -the default. - -* `embedpoi` retries patiently (up to 20× with backoff) because the shared - llama.cpp host swaps models in/out and 500s briefly while zembed reloads. -* Zembed uses a `query: ` prompt prefix for search-time queries — `spatiald` - adds it in `/semantic`; `embedpoi` stores raw `"name — kind"` passages. -* After the run it builds `poi_vec_hnsw` (`hnsw (vec vector_cosine_ops)`) so - `/semantic` stays fast at full scale. -* The DB image is `postgis/postgis:16-3.4` + `postgresql-16-pgvector` (see - `db/Dockerfile`). The stock postgis images for PG16 are bullseye-based and - their Debian release files are expired, so the image disables - `Check-Valid-Until` and adds the pgdg repo just to fetch `postgresql-16-pgvector`. - ## Data notes * Built for **osm2pgsql 2.x** (`iboates/osm2pgsql`): tables are @@ -141,15 +108,10 @@ the default. `url.Parse` drops the tail of a value that contains a raw `;`. * Multiple extracts coexist in one DB, disambiguated by `poi.extract` (set automatically by `import.sh`). -* `lib/pq` has no `[]float32` → `vector` codec, so `/semantic` sends the - embedding as a `[f1,f2,…]` **text** literal and casts with `::vector` in SQL. -* `poi_vec` is rebuilt by `embedpoi` (not `import.sh`); it is keyed - `(extract, osm_type, osm_id)` to match `poi`. ## Roadmap hooks * `bbox` queries: trivial addition (`ST_Contains(ST_MakeEnvelope,…)`). -* Rerun `embedpoi` after a new `import.sh` so `poi_vec` tracks the new `poi` rows. * Routing-graph join: OSRM/GraphHopper geometries can be loaded into the same DB (`route_geom` table) for true along-route analytics instead of the current buffer-over-polyline. diff --git a/spatial/db/Dockerfile b/spatial/db/Dockerfile deleted file mode 100644 index 6d34265..0000000 --- a/spatial/db/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# PostGIS + pgvector. -# -# postgis/postgis images for PG16 are bullseye-based and their Debian release -# files have expired (EOL), so a plain apt update fails. Two fixes baked in: -# - Acquire::Check-Valid-Until=false (the repos are still served) -# - the pgdg signing key (trusted.gpg.d accepts ASCII-armored keys) -# then postgresql-16-pgvector from pgdg gives us the `vector` extension. -FROM postgis/postgis:16-3.4 -COPY pgdg.asc /etc/apt/trusted.gpg.d/pgdg.asc -RUN echo 'deb http://apt.postgresql.org/pub/repos/apt bullseye-pgdg main' \ - > /etc/apt/sources.list.d/pgdg.list \ - && echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/99noexpire \ - && apt-get update -qq \ - && apt-get install -y -qq --no-install-recommends postgresql-16-pgvector \ - && rm -rf /var/lib/apt/lists/* diff --git a/spatial/db/pgdg.asc b/spatial/db/pgdg.asc deleted file mode 100644 index 8480576..0000000 --- a/spatial/db/pgdg.asc +++ /dev/null @@ -1,77 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- - -mQINBE6XR8IBEACVdDKT2HEH1IyHzXkb4nIWAY7echjRxo7MTcj4vbXAyBKOfjja -UrBEJWHN6fjKJXOYWXHLIYg0hOGeW9qcSiaa1/rYIbOzjfGfhE4x0Y+NJHS1db0V -G6GUj3qXaeyqIJGS2z7m0Thy4Lgr/LpZlZ78Nf1fliSzBlMo1sV7PpP/7zUO+aA4 -bKa8Rio3weMXQOZgclzgeSdqtwKnyKTQdXY5MkH1QXyFIk1nTfWwyqpJjHlgtwMi -c2cxjqG5nnV9rIYlTTjYG6RBglq0SmzF/raBnF4Lwjxq4qRqvRllBXdFu5+2pMfC -IZ10HPRdqDCTN60DUix+BTzBUT30NzaLhZbOMT5RvQtvTVgWpeIn20i2NrPWNCUh -hj490dKDLpK/v+A5/i8zPvN4c6MkDHi1FZfaoz3863dylUBR3Ip26oM0hHXf4/2U -A/oA4pCl2W0hc4aNtozjKHkVjRx5Q8/hVYu+39csFWxo6YSB/KgIEw+0W8DiTII3 -RQj/OlD68ZDmGLyQPiJvaEtY9fDrcSpI0Esm0i4sjkNbuuh0Cvwwwqo5EF1zfkVj -Tqz2REYQGMJGc5LUbIpk5sMHo1HWV038TWxlDRwtOdzw08zQA6BeWe9FOokRPeR2 -AqhyaJJwOZJodKZ76S+LDwFkTLzEKnYPCzkoRwLrEdNt1M7wQBThnC5z6wARAQAB -tBxQb3N0Z3JlU1FMIERlYmlhbiBSZXBvc2l0b3J5iQJOBBMBCAA4AhsDBQsJCAcD -BRUKCQgLBRYCAwEAAh4BAheAFiEEuXsK/KoaR/BE8kSgf8x9RqzMTPgFAlhtCD8A -CgkQf8x9RqzMTPgECxAAk8uL+dwveTv6eH21tIHcltt8U3Ofajdo+D/ayO53LiYO -xi27kdHD0zvFMUWXLGxQtWyeqqDRvDagfWglHucIcaLxoxNwL8+e+9hVFIEskQAY -kVToBCKMXTQDLarz8/J030Pmcv3ihbwB+jhnykMuyyNmht4kq0CNgnlcMCdVz0d3 -z/09puryIHJrD+A8y3TD4RM74snQuwc9u5bsckvRtRJKbP3GX5JaFZAqUyZNRJRJ -Tn2OQRBhCpxhlZ2afkAPFIq2aVnEt/Ie6tmeRCzsW3lOxEH2K7MQSfSu/kRz7ELf -Cz3NJHj7rMzC+76Rhsas60t9CjmvMuGONEpctijDWONLCuch3Pdj6XpC+MVxpgBy -2VUdkunb48YhXNW0jgFGM/BFRj+dMQOUbY8PjJjsmVV0joDruWATQG/M4C7O8iU0 -B7o6yVv4m8LDEN9CiR6r7H17m4xZseT3f+0QpMe7iQjz6XxTUFRQxXqzmNnloA1T -7VjwPqIIzkj/u0V8nICG/ktLzp1OsCFatWXh7LbU+hwYl6gsFH/mFDqVxJ3+DKQi -vyf1NatzEwl62foVjGUSpvh3ymtmtUQ4JUkNDsXiRBWczaiGSuzD9Qi0ONdkAX3b -ewqmN4TfE+XIpCPxxHXwGq9Rv1IFjOdCX0iG436GHyTLC1tTUIKF5xV4Y0+cXIOI -RgQQEQgABgUCTpdI7gAKCRDFr3dKWFELWqaPAKD1TtT5c3sZz92Fj97KYmqbNQZP -+ACfSC6+hfvlj4GxmUjp1aepoVTo3weJAhwEEAEIAAYFAk6XSQsACgkQTFprqxLS -p64F8Q//cCcutwrH50UoRFejg0EIZav6LUKejC6kpLeubbEtuaIH3r2zMblPGc4i -+eMQKo/PqyQrceRXeNNlqO6/exHozYi2meudxa6IudhwJIOn1MQykJbNMSC2sGUp -1W5M1N5EYgt4hy+qhlfnD66LR4G+9t5FscTJSy84SdiOuqgCOpQmPkVRm1HX5X1+ -dmnzMOCk5LHHQuiacV0qeGO7JcBCVEIDr+uhU1H2u5GPFNHm5u15n25tOxVivb94 -xg6NDjouECBH7cCVuW79YcExH/0X3/9G45rjdHlKPH1OIUJiiX47OTxdG3dAbB4Q -fnViRJhjehFscFvYWSqXo3pgWqUsEvv9qJac2ZEMSz9x2mj0ekWxuM6/hGWxJdB+ -+985rIelPmc7VRAXOjIxWknrXnPCZAMlPlDLu6+vZ5BhFX0Be3y38f7GNCxFkJzl -hWZ4Cj3WojMj+0DaC1eKTj3rJ7OJlt9S9xnO7OOPEUTGyzgNIDAyCiu8F4huLPaT -ape6RupxOMHZeoCVlqx3ouWctelB2oNXcxxiQ/8y+21aHfD4n/CiIFwDvIQjl7dg -mT3u5Lr6yxuosR3QJx1P6rP5ZrDTP9khT30t+HZCbvs5Pq+v/9m6XDmi+NlU7Zuh -Ehy97tL3uBDgoL4b/5BpFL5U9nruPlQzGq1P9jj40dxAaDAX/WKJAj0EEwEIACcC -GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlB5KywFCQPDFt8ACgkQf8x9RqzM -TPhuCQ//QAjRSAOCQ02qmUAikT+mTB6baOAakkYq6uHbEO7qPZkv4E/M+HPIJ4wd -nBNeSQjfvdNcZBA/x0hr5EMcBneKKPDj4hJ0panOIRQmNSTThQw9OU351gm3YQct -AMPRUu1fTJAL/AuZUQf9ESmhyVtWNlH/56HBfYjE4iVeaRkkNLJyX3vkWdJSMwC/ -LO3Lw/0M3R8itDsm74F8w4xOdSQ52nSRFRh7PunFtREl+QzQ3EA/WB4AIj3VohIG -kWDfPFCzV3cyZQiEnjAe9gG5pHsXHUWQsDFZ12t784JgkGyO5wT26pzTiuApWM3k -/9V+o3HJSgH5hn7wuTi3TelEFwP1fNzI5iUUtZdtxbFOfWMnZAypEhaLmXNkg4zD -kH44r0ss9fR0DAgUav1a25UnbOn4PgIEQy2fgHKHwRpCy20d6oCSlmgyWsR40EPP -YvtGq49A2aK6ibXmdvvFT+Ts8Z+q2SkFpoYFX20mR2nsF0fbt1lfH65P64dukxeR -GteWIeNakDD40bAAOH8+OaoTGVBJ2ACJfLVNM53PEoftavAwUYMrR910qvwYfd/4 -6rh46g1Frr9SFMKYE9uvIJIgDsQB3QBp71houU4H55M5GD8XURYs+bfiQpJG1p7e -B8e5jZx1SagNWc4XwL2FzQ9svrkbg1Y+359buUiP7T6QXX2zY++JAj0EEwEIACcC -GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlEqbZUFCQg2wEEACgkQf8x9RqzM -TPhFMQ//WxAfKMdpSIA9oIC/yPD/dJpY/+DyouOljpE6MucMy/ArBECjFTBwi/j9 -NYM4ynAk34IkhuNexc1i9/05f5RM6+riLCLgAOsADDbHD4miZzoSxiVr6GQ3YXMb -OGld9kV9Sy6mGNjcUov7iFcf5Hy5w3AjPfKuR9zXswyfzIU1YXObiiZT38l55pp/ -BSgvGVQsvbNjsff5CbEKXS7q3xW+WzN0QWF6YsfNVhFjRGj8hKtHvwKcA02wwjLe -LXVTm6915ZUKhZXUFc0vM4Pj4EgNswH8Ojw9AJaKWJIZmLyW+aP+wpu6YwVCicxB -Y59CzBO2pPJDfKFQzUtrErk9irXeuCCLesDyirxJhv8o0JAvmnMAKOLhNFUrSQ2m -+3EnF7zhfz70gHW+EG8X8mL/EN3/dUM09j6TVrjtw43RLxBzwMDeariFF9yC+5bL -tnGgxjsB9Ik6GV5v34/NEEGf1qBiAzFmDVFRZlrNDkq6gmpvGnA5hUWNr+y0i01L -jGyaLSWHYjgw2UEQOqcUtTFK9MNzbZze4mVaHMEz9/aMfX25R6qbiNqCChveIm8m -Yr5Ds2zdZx+G5bAKdzX7nx2IUAxFQJEE94VLSp3npAaTWv3sHr7dR8tSyUJ9poDw -gw4W9BIcnAM7zvFYbLF5FNggg/26njHCCN70sHt8zGxKQINMc6SJAj0EEwEIACcC -GwMFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AFAlLpFRkFCQ6EJy0ACgkQf8x9RqzM -TPjOZA//Zp0e25pcvle7cLc0YuFr9pBv2JIkLzPm83nkcwKmxaWayUIG4Sv6pH6h -m8+S/CHQij/yFCX+o3ngMw2J9HBUvafZ4bnbI0RGJ70GsAwraQ0VlkIfg7GUw3Tz -voGYO42rZTru9S0K/6nFP6D1HUu+U+AsJONLeb6oypQgInfXQExPZyliUnHdipei -4WR1YFW6sjSkZT/5C3J1wkAvPl5lvOVthI9Zs6bZlJLZwusKxU0UM4Btgu1Sf3nn -JcHmzisixwS9PMHE+AgPWIGSec/N27a0KmTTvImV6K6nEjXJey0K2+EYJuIBsYUN -orOGBwDFIhfRk9qGlpgt0KRyguV+AP5qvgry95IrYtrOuE7307SidEbSnvO5ezNe -mE7gT9Z1tM7IMPfmoKph4BfpNoH7aXiQh1Wo+ChdP92hZUtQrY2Nm13cmkxYjQ4Z -gMWfYMC+DA/GooSgZM5i6hYqyyfAuUD9kwRN6BqTbuAUAp+hCWYeN4D88sLYpFh3 -paDYNKJ+Gf7Yyi6gThcV956RUFDH3ys5Dk0vDL9NiWwdebWfRFbzoRM3dyGP889a -OyLzS3mh6nHzZrNGhW73kslSQek8tjKrB+56hXOnb4HaElTZGDvD5wmrrhN94kby -Gtz3cydIohvNO9d90+29h0eGEDYti7j7maHkBKUAwlcPvMg5m3Y= -=DA1T ------END PGP PUBLIC KEY BLOCK----- diff --git a/spatial/docker-compose.yml b/spatial/docker-compose.yml index 71d61a1..5fa1a80 100644 --- a/spatial/docker-compose.yml +++ b/spatial/docker-compose.yml @@ -14,7 +14,7 @@ version: "3.8" services: postgis: - build: ./db # postgis/postgis:16-3.4 + postgresql-16-pgvector (db/Dockerfile) + image: postgis/postgis:16-3.4 container_name: trips-postgis environment: POSTGRES_DB: trips @@ -26,7 +26,6 @@ services: - pgdata:/var/lib/postgresql/data # initdb scripts run in lexicographic order, once, on an empty volume - ./schema.sql:/docker-entrypoint-initdb.d/10-schema.sql:ro - - ./20-vector.sql:/docker-entrypoint-initdb.d/20-vector.sql:ro restart: unless-stopped # One-shot importer (also usable as: docker-compose run --rm importer @@ -50,8 +49,6 @@ services: container_name: trips-spatiald environment: SPATIAL_DSN: "host=postgis port=5432 user=trips password=trips dbname=trips sslmode=disable" - EMBED_BASE: ${EMBED_BASE:-http://192.168.3.7:1234/v1} - EMBED_MODEL: ${EMBED_MODEL:-zembed-1-Q4_K_M} ports: - "5005:5005" depends_on: diff --git a/spatial/embed/main.go b/spatial/embed/main.go deleted file mode 100644 index 2831ab9..0000000 --- a/spatial/embed/main.go +++ /dev/null @@ -1,211 +0,0 @@ -// embedpoi — build the semantic POI index (poi_vec) from the poi table. -// -// For every named POI (all families except place=*) it embeds -// "name — kind" with the local zembed model (llama.cpp /embeddings) in -// batches and upserts into poi_vec. Resumable: existing keys are skipped. -// After the run it creates the HNSW cosine index. -// -// go run ./embed -dsn "host=localhost ... dbname=trips sslmode=disable" -package main - -import ( - "bytes" - "database/sql" - "encoding/json" - "flag" - "fmt" - "io" - "log" - "net/http" - "os" - "strconv" - "strings" - "time" - - _ "github.com/lib/pq" -) - -func getenvDefault(k, def string) string { - if v := os.Getenv(k); v != "" { - return v - } - return def -} - -func must(err error) { - if err != nil { - log.Fatal(err) - } -} - -func exec(db *sql.DB, q string, args ...any) { - if _, e := db.Exec(q, args...); e != nil { - log.Fatal(e) - } -} - -// vecToSQL renders a []float32 as the body of a vector literal: 0.1,-0.2,… -func vecToSQL(v []float32) string { - parts := make([]string, len(v)) - for i, f := range v { - parts[i] = strconv.FormatFloat(float64(f), 'g', 6, 32) - } - return strings.Join(parts, ",") -} - -func main() { - dsnFlag := flag.String("dsn", getenvDefault("SPATIAL_DSN", - "host=localhost port=5432 user=trips password=trips dbname=trips sslmode=disable"), "DSN") - baseFlag := flag.String("embed-base", getenvDefault("EMBED_BASE", "http://192.168.3.7:1234/v1"), "llama.cpp /v1 base") - modelFlag := flag.String("embed-model", getenvDefault("EMBED_MODEL", "zembed-1-Q4_K_M"), "embedding model") - batchFlag := flag.Int("batch", 64, "embeddings per request") - extractFlag := flag.String("extract", "", "restrict to one extract (default: all)") - qpsFlag := flag.Int("qps", 4, "requests per second (politeness toward the shared model server)") - flag.Parse() - - db, err := sql.Open("postgres", *dsnFlag) - must(err) - db.SetMaxOpenConns(4) - must(db.Ping()) - client := &http.Client{Timeout: 120 * time.Second} - - embed := func(texts []string) ([][]float32, error) { - body, _ := json.Marshal(map[string]any{"model": *modelFlag, "input": texts}) - req, err := http.NewRequest("POST", *baseFlag+"/embeddings", bytes.NewReader(body)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - res, err := client.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != 200 { - b, _ := io.ReadAll(io.LimitReader(res.Body, 300)) - return nil, fmt.Errorf("embed http %d: %s", res.StatusCode, b) - } - var j struct { - Data []struct { - Embedding []float32 `json:"embedding"` - } `json:"data"` - } - if err := json.NewDecoder(res.Body).Decode(&j); err != nil { - return nil, err - } - var out [][]float32 - for _, d := range j.Data { - out = append(out, d.Embedding) - } - return out, nil - } - - // ---- collect rows (everything semantic-searchable: named POIs of the - // POI families; place=* localities are not "places to go") - type row struct { - Extract, OsmType, Name, Kind string - OsmID int64 - } - q := `SELECT extract, osm_type, osm_id, name, kind FROM poi - WHERE kind NOT LIKE 'place=%'` - var args []any - if *extractFlag != "" { - q += " AND extract = $1" - args = append(args, *extractFlag) - } - q += " ORDER BY extract, osm_id" - rows, err := db.Query(q, args...) - must(err) - var all []row - for rows.Next() { - var r row - must(rows.Scan(&r.Extract, &r.OsmType, &r.OsmID, &r.Name, &r.Kind)) - all = append(all, r) - } - rows.Close() - log.Printf("poi rows to consider: %d", len(all)) - - // resume: keys already embedded - have := map[string]bool{} - hrows, err := db.Query(`SELECT extract, osm_type, osm_id FROM poi_vec`) - if err != nil { - have = nil // table doesn't exist yet — everything is to do - } else { - for hrows.Next() { - var e, t string - var id int64 - hrows.Scan(&e, &t, &id) - have[e+"|"+t+"|"+strconv.FormatInt(id, 10)] = true - } - hrows.Close() - } - var todo []row - for _, r := range all { - if have == nil || !have[r.Extract+"|"+r.OsmType+"|"+strconv.FormatInt(r.OsmID, 10)] { - todo = append(todo, r) - } - } - log.Printf("already embedded: %d, to do: %d", len(all)-len(todo), len(todo)) - if len(todo) == 0 { - return - } - - var dim int - total := len(todo) - for i := 0; i < total; i += *batchFlag { - j := i + *batchFlag - if j > total { - j = total - } - chunk := todo[i:j] - texts := make([]string, len(chunk)) - for k, r := range chunk { - texts[k] = r.Name + " — " + r.Kind - } - // the shared llama.cpp host evicts zembed when another model loads - // (500 "proxy error") — that clears on the next request, so retry - // patiently rather than dying - var vecs [][]float32 - for attempt := 0; ; attempt++ { - var err error - vecs, err = embed(texts) - if err == nil { - break - } - if attempt >= 20 { - must(err) - } - log.Printf("batch at %d failed (%v) — retry %d/20", i, err, attempt+1) - time.Sleep(time.Duration(5+attempt) * time.Second) - } - if dim == 0 { - dim = len(vecs[0]) - // derived table: safe to rebuild if the model's dim changed - exec(db, `DROP TABLE IF EXISTS poi_vec`) - stmt := fmt.Sprintf(`CREATE TABLE poi_vec ( - extract text NOT NULL, osm_type char NOT NULL, osm_id bigint NOT NULL, - name text NOT NULL, kind text NOT NULL, - vec vector(%d) NOT NULL, - PRIMARY KEY (extract, osm_type, osm_id))`, dim) - log.Printf("embedding dim: %d — creating poi_vec", dim) - exec(db, stmt) - } - for k, r := range chunk { - _, err := db.Exec(`INSERT INTO poi_vec (extract, osm_type, osm_id, name, kind, vec) - VALUES ($1,$2,$3,$4,$5,$6) - ON CONFLICT (extract, osm_type, osm_id) DO UPDATE SET vec = EXCLUDED.vec`, - r.Extract, r.OsmType, r.OsmID, r.Name, r.Kind, - "["+vecToSQL(vecs[k])+"]") - must(err) - } - if i == 0 || i/(*batchFlag*25) > (i-*batchFlag)/(*batchFlag*25) { - log.Printf("%d/%d embedded (%.1f%%)", j, total, 100.0*float64(j)/float64(total)) - } - time.Sleep(time.Second / time.Duration(*qpsFlag)) - } - - log.Printf("creating HNSW index (vector_cosine_ops) on poi_vec…") - exec(db, `CREATE INDEX IF NOT EXISTS poi_vec_hnsw ON poi_vec - USING hnsw (vec vector_cosine_ops)`) - log.Printf("done: %d vectors in poi_vec", len(all)) -} diff --git a/spatial/main.go b/spatial/main.go index 3ffb5e8..2c07782 100644 --- a/spatial/main.go +++ b/spatial/main.go @@ -8,7 +8,6 @@ // GET /corridor?points=&r= corridor buffering along a route // GET /search?kinds=&terms=&lat=&lng=&r= targeted tag+name search // GET /kinds?q=&extract= the OSM tag vocabulary of the extracts -// GET /semantic?q= (opt-in; needs the poi_vec index, see embed/) // // Optional filters on the spatial three: // @@ -27,12 +26,10 @@ package main import ( - "bytes" "database/sql" "encoding/json" "flag" "fmt" - "io" "log" "net/http" "os" @@ -46,130 +43,8 @@ import ( var ( dsn *sql.DB addr string - // /semantic: embed the query with the local zembed model, cosine-search poi_vec - embedBase = getenvDefault("EMBED_BASE", "http://192.168.3.7:1234/v1") - embedModel = getenvDefault("EMBED_MODEL", "zembed-1-Q4_K_M") ) -func getenvDefault(k, def string) string { - if v := os.Getenv(k); v != "" { - return v - } - return def -} - -// embedOne asks the llama.cpp OpenAI-compatible /embeddings endpoint for a -// single text. Zembed uses a "query: " prompt prefix for search queries. -func embedOne(text string) ([]float32, error) { - body, _ := json.Marshal(map[string]any{"model": embedModel, "input": []string{"query: " + text}}) - req, err := http.NewRequest("POST", embedBase+"/embeddings", bytes.NewReader(body)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - client := &http.Client{Timeout: 60 * time.Second} - res, err := client.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != 200 { - b, _ := io.ReadAll(io.LimitReader(res.Body, 300)) - return nil, fmt.Errorf("embed http %d: %s", res.StatusCode, b) - } - var j struct { - Data []struct { - Embedding []float32 `json:"embedding"` - } `json:"data"` - } - if err := json.NewDecoder(res.Body).Decode(&j); err != nil { - return nil, err - } - if len(j.Data) == 0 || len(j.Data[0].Embedding) == 0 { - return nil, fmt.Errorf("no embedding in response") - } - return j.Data[0].Embedding, nil -} - -// vecLiteral renders [0.1, -0.2] as the pgvector text literal [0.1,-0.2] -func vecLiteral(v []float32) string { - parts := make([]string, len(v)) - for i, f := range v { - parts[i] = strconv.FormatFloat(float64(f), 'g', 6, 32) - } - return "[" + strings.Join(parts, ",") + "]" -} - -func handleSemantic(w http.ResponseWriter, r *http.Request) { - q := r.URL.Query() - text := strings.TrimSpace(first(q, "q")) - if text == "" { - jerr(w, 400, "q is required") - return - } - vec, err := embedOne(text) - if err != nil { - jerr(w, 502, "embedding failed: "+err.Error()) - return - } - // lib/pq has no []float32 → vector codec: send the literal "[f1,f2,…]" - // as text and cast with ::vector - where, args := filters(q) // extract/kind/name filters still apply - args = append(args, vecLiteral(vec)) - vecTok := "@@" + strconv.Itoa(len(args)) + "@@" - args = append(args, limitOf(q)) - limTok := "@@" + strconv.Itoa(len(args)) + "@@" - // poi_vec carries (extract, osm_type, osm_id, name, kind, vec); the JOIN - // back to poi yields the geometry (for map pins) and the tag facts. - query := `SELECT p.extract, p.name, p.kind, - ST_Y(p.geom) AS lat, ST_X(p.geom) AS lng, - p.opening_hours, p.fee, p.website, - 1 - (v.vec <=> ` + vecTok + `::vector)::float8 AS score - FROM poi_vec v - JOIN poi p ON p.extract = v.extract AND p.osm_type = v.osm_type AND p.osm_id = v.osm_id` + where + ` - ORDER BY v.vec <=> ` + vecTok + `::vector - LIMIT ` + limTok - query = renumber(query, 0) // no body placeholders here; just resolve tokens - rows, err := dsn.Query(query, args...) - if err != nil { - jerr(w, 502, "query: "+err.Error()) - return - } - var out []map[string]any - for rows.Next() { - var extract, name, kind string - var lat, lng float64 - var oh, fee, web *string - var score float64 - if err := rows.Scan(&extract, &name, &kind, &lat, &lng, &oh, &fee, &web, &score); err != nil { - rows.Close() - jerr(w, 502, err.Error()) - return - } - m := map[string]any{"extract": extract, "name": name, "kind": kind, - "lat": lat, "lng": lng, "score": float64(int64(score*10000)) / 10000} - if oh != nil { - m["opening_hours"] = *oh - } - if fee != nil { - m["fee"] = *fee - } - if web != nil { - m["website"] = *web - } - out = append(out, m) - } - rows.Close() - if out == nil { - out = []map[string]any{} - } - cors(w) - b, _ := json.Marshal(out) - fmt.Fprintf(w, `{"count":%d,"results":`, len(out)) - w.Write(b) - w.Write([]byte("}")) -} - type poi struct { Extract string `json:"extract"` OsmID int64 `json:"osm_id"` @@ -426,7 +301,6 @@ func main() { mux.HandleFunc("/corridor", handleCorridor) mux.HandleFunc("/search", handleSearch) mux.HandleFunc("/kinds", handleKinds) - mux.HandleFunc("/semantic", handleSemantic) // opt-in; needs poi_vec (see embed/) log.Printf("spatiald: listening on %s (dsn: %s)", addr, *dsnFlag) srv := &http.Server{Addr: addr, Handler: mux, ReadHeaderTimeout: 5 * time.Second} if err = srv.ListenAndServe(); err != nil {