From f03e7b5dbcf69bdd2f98781876a5b90018794f4c Mon Sep 17 00:00:00 2001 From: Greg Pomerantz Date: Wed, 9 Sep 2026 14:16:36 -0400 Subject: [PATCH] mock: robust booking-doc extraction + documents persist across turns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An 118 KB HTML flight confirmation exposed two problems: 1. Extraction — the old <[^>]+> tag-strip broke on a ">" inside a quoted attribute (inline SVG data-URIs are common in mail markup) and the text was silently cut at 20 000 chars, so the model saw a mangled, truncated document ("it was truncated"). /parse-doc now uses a quote-aware state-machine extractor (skips comments / MSO conditional comments / style / script / head blocks; block tags → newlines; table cells → " | "; single-pass entity decode), decodes .eml / MIME mail (base64 + quoted-printable, html part preferred, one level of nested multipart), collapses 10×+ repeated boilerplate lines, and caps at 60 000 chars with an explicit `truncated` flag (server logs it too). File limit raised to 16 MB; Outlook .msg gets a clear "forward as HTML" error. A 130 KB bloated fixture now extracts to 901 clean chars with all flight details intact. 2. Memory — uploaded documents now stay attached to the trip: persisted in tripStore with the plan, shown as chips in the chat, and included in the model's context on EVERY turn, so follow-up questions ("what's my confirmation code?") are answered without a re-upload and documents survive reloads. 📎 now works for any reference doc (flight bookings still trigger the re-anchor flow; the model asks to confirm when the booked dates don't match the plan). Also: askLLM refuses to start a second agent loop while one is in flight — a question typed during a long tool loop used to fork a parallel loop that corrupted the first. Co-Authored-By: Claude --- mock/app.js | 45 ++++++++++++--- mock/index.html | 5 +- mock/server.js | 145 ++++++++++++++++++++++++++++++++++++++++++++---- mock/styles.css | 4 ++ 4 files changed, 176 insertions(+), 23 deletions(-) diff --git a/mock/app.js b/mock/app.js index ed39726..ca85a50 100644 --- a/mock/app.js +++ b/mock/app.js @@ -429,7 +429,7 @@ function llmSystemPrompt() { '- Before a big rework, consider review_plan() and mention any issues it reports.\n' + '- Be efficient: call each tool at most once unless you need to, and never re-fetch facts you already have. A fix-op alone is enough — do not re-search or re-fetch the stop you are moving/editing.\n' + '- When a real flight/booking document is attached, read out the arrival and departure (date, local time, airport, city), then call apply_flight_anchors exactly once. If the document is ambiguous or its dates don’t match the plan’s, ask me to confirm before applying.\n\n' + - 'Current plan:\n' + llmContext()); + 'Current plan:\n' + llmContext() + docSection()); } // Parse a {...} block from the model output, if present. @@ -446,6 +446,7 @@ function toolLine(name, args) { } async function askLLM(v) { + if (scriptBusy) return msg('ai', 'Hold on — I’m still working on the last request.'); scriptBusy = true; let t = typing(); const messages = [{ role: 'system', content: llmSystemPrompt() }, { role: 'user', content: v }]; @@ -1528,6 +1529,7 @@ $('#l3-close').onclick = closeL3; let version = 0; let history = []; // [{v, label, at, days, stays}] let hIdx = -1; // pointer into history +let tripDocs = {}; // name → {name,kind,chars,text,truncated,at} — attached documents, persisted per trip const commit = (label) => { history = history.slice(0, hIdx + 1); // drop any redo branch history.push({ v: ++version, label, at: Date.now(), days: deep(days), stays: deep(stays), bookings: deep(M.trip.bookings), title: M.trip.title }); @@ -1820,17 +1822,40 @@ async function ingestDoc(file) { reconfigureFromDoc(j); } catch { note.remove(); msg('ai', 'The document reader is unreachable right now.'); } } +// attached documents stay in the model's context on EVERY turn, so follow-up +// questions ("what’s my confirmation code?", "when’s the flight out?") work +// without a re-upload +const DOC_CTX_CAP = 20000; // per-document budget inside the system prompt +function docSection() { + const ds = Object.values(tripDocs); + if (!ds.length) return ''; + return '\n\nAttached documents (the user’s own files, kept for reference on any question). For questions about their real bookings (confirmation codes, seats, prices, flight times), answer from these documents, not from the plan’s placeholder bookings:\n' + + ds.map(d => 'DOCUMENT “' + d.name + '”' + (d.truncated ? ' (extract truncated)' : '') + '\n' + (d.text || '').slice(0, DOC_CTX_CAP)).join('\n\n'); +} +function renderDocChips() { + const el = $('#doc-chips'); if (!el) return; + const names = Object.keys(tripDocs); + if (!names.length) { el.classList.add('hidden'); el.innerHTML = ''; return; } + el.classList.remove('hidden'); + el.innerHTML = names.map(n => + '📎 ' + esc(n) + ' ' + (Math.round((tripDocs[n].chars || 0) / 100) / 10) + ' KB').join(' '); +} function reconfigureFromDoc(j) { - msg('user', '📎 ' + esc(j.name) + ' ' + (j.kind || 'doc') + ' · ' + (j.chars || 0) + ' chars — reconfigure the trip around my flights'); - if (!llmOn) return msg('ai', 'I read the booking (' + (j.chars || 0) + ' chars), but the model is offline so I can’t reconfigure around it yet. I’ll do it as soon as it’s back.'); + // keep the document on file: it persists with the trip and rides in the + // model's context on every turn + tripDocs[j.name] = { name: j.name, kind: j.kind, chars: j.chars, text: j.text, truncated: !!j.truncated, at: Date.now() }; + persistTrip(); renderDocChips(); + msg('user', '📎 ' + esc(j.name) + ' ' + (j.kind || 'doc') + ' · ' + (j.chars || 0) + ' chars — attached' + (j.truncated ? ' (truncated)' : '') + ''); + if (!llmOn) return msg('ai', 'I read the document (' + (j.chars || 0) + ' chars) and kept it attached, but the model is offline so I can’t act on it yet. I’ll do it as soon as it’s back.'); const prompt = - 'I’ve attached my real flight booking for this trip (extracted text below).\n\n' + - 'DOCUMENT “' + (j.name || 'booking') + '”\n' + + 'I’ve attached a document for this trip (extracted text below). It stays on file for later questions, so no need to re-list its contents in your reply.' + (j.truncated ? ' NOTE: the extraction was truncated at the end — if a needed detail is missing, say so and ask me to paste it.' : '') + '\n\n' + + 'DOCUMENT “' + (j.name || 'document') + '”\n' + '\n' + (j.text || '') + '\n\n\n' + - 'Reconfigure this trip around it:\n' + + 'If this is a flight booking (or contains actual flight times), reconfigure the trip around it:\n' + '1. Read the ARRIVAL (date, local time, airport, city) and the DEPARTURE (date, local time, airport, city). With connections, the arrival is where the trip starts (first city) and the departure where it ends (last city).\n' + '2. Call apply_flight_anchors exactly once with those two anchors.\n' + - '3. Confirm what changed — the new first and last day with their start/end times, and any stops that had to go. Keep it to a couple of sentences.'; + '3. Confirm what changed — the new first and last day with their start/end times, and any stops that had to go, in a couple of sentences.\n' + + 'If it is NOT a flight booking, do not change the plan — just confirm it’s attached and summarize what it contains in 1–2 sentences.'; askLLM(prompt); } // drag & drop a file anywhere on the chat panel @@ -1895,7 +1920,7 @@ let persistT = 0; function persistTrip() { clearTimeout(persistT); const hist = history.slice(0, hIdx + 1).slice(-25); // drop the redo tail, cap at 25 - persistT = setTimeout(() => tripStore.save(tripId, { days, stays, version, hIdx: hist.length - 1, history: hist, bookings: M.trip.bookings, title: M.trip.title }), 250); + persistT = setTimeout(() => tripStore.save(tripId, { days, stays, version, hIdx: hist.length - 1, history: hist, bookings: M.trip.bookings, title: M.trip.title, docs: tripDocs }), 250); } function renderTripMenu() { $('#crumb').innerHTML = `🧳 ${M.trip.title} ▾`; @@ -1927,7 +1952,8 @@ function enterTrip(id) { days = deep(saved.days); stays = deep(saved.stays); version = saved.version; history = saved.history; hIdx = saved.hIdx; if (saved.bookings) M.trip.bookings = deep(saved.bookings); if (saved.title != null) M.trip.title = saved.title; - } + tripDocs = saved.docs ? deep(saved.docs) : {}; + } else tripDocs = {}; // reset per-trip UI state compareItems = []; closeDiscover(); closeL3(); @@ -1944,6 +1970,7 @@ function enterTrip(id) { renderTripMenu(); buildDayTabs(); renderHotelMks(); renderBaseChip(); + renderDocChips(); // this trip may live on a different OSRM extract — re-check before routing Promise.all([checkRouter(), checkLLM()]).then(() => { const b = $('#offline-badge'); diff --git a/mock/index.html b/mock/index.html index 3191a70..680ebda 100644 --- a/mock/index.html +++ b/mock/index.html @@ -111,9 +111,10 @@