mock: robust booking-doc extraction + documents persist across turns

An 118 KB HTML flight confirmation exposed two problems:

1. Extraction — the old <[^>]+> tag-strip broke on a ">" inside a quoted
   attribute (inline SVG data-URIs are common in mail markup) and the
   text was silently cut at 20 000 chars, so the model saw a mangled,
   truncated document ("it was truncated"). /parse-doc now uses a
   quote-aware state-machine extractor (skips comments / MSO conditional
   comments / style / script / head blocks; block tags → newlines; table
   cells → " | "; single-pass entity decode), decodes .eml / MIME mail
   (base64 + quoted-printable, html part preferred, one level of nested
   multipart), collapses 10×+ repeated boilerplate lines, and caps at
   60 000 chars with an explicit `truncated` flag (server logs it too).
   File limit raised to 16 MB; Outlook .msg gets a clear "forward as
   HTML" error. A 130 KB bloated fixture now extracts to 901 clean chars
   with all flight details intact.

2. Memory — uploaded documents now stay attached to the trip: persisted
   in tripStore with the plan, shown as chips in the chat, and included
   in the model's context on EVERY turn, so follow-up questions
   ("what's my confirmation code?") are answered without a re-upload and
   documents survive reloads. 📎 now works for any reference doc (flight
   bookings still trigger the re-anchor flow; the model asks to confirm
   when the booked dates don't match the plan).

Also: askLLM refuses to start a second agent loop while one is in
flight — a question typed during a long tool loop used to fork a
parallel loop that corrupted the first.

Co-Authored-By: Claude <noreply@anthropic.com>

mock/app.js

@@ -429,7 +429,7 @@ function llmSystemPrompt() {
     '- Before a big rework, consider review_plan() and mention any issues it reports.\n' +
     '- Be efficient: call each tool at most once unless you need to, and never re-fetch facts you already have. A fix-op alone is enough — do not re-search or re-fetch the stop you are moving/editing.\n' +
     '- When a real flight/booking document is attached, read out the arrival and departure (date, local time, airport, city), then call apply_flight_anchors exactly once. If the document is ambiguous or its dates don’t match the plan’s, ask me to confirm before applying.\n\n' +
-    'Current plan:\n' + llmContext());
+    'Current plan:\n' + llmContext() + docSection());
 }
 
 // Parse a <tool>{...}</tool> block from the model output, if present.
@@ -446,6 +446,7 @@ function toolLine(name, args) {
 }
 
 async function askLLM(v) {
+  if (scriptBusy) return msg('ai', 'Hold on — I’m still working on the last request.');
   scriptBusy = true;
   let t = typing();
   const messages = [{ role: 'system', content: llmSystemPrompt() }, { role: 'user', content: v }];
@@ -1528,6 +1529,7 @@ $('#l3-close').onclick = closeL3;
 let version = 0;
 let history = [];      // [{v, label, at, days, stays}]
 let hIdx = -1;         // pointer into history
+let tripDocs = {};     // name → {name,kind,chars,text,truncated,at} — attached documents, persisted per trip
 const commit = (label) => {
   history = history.slice(0, hIdx + 1);      // drop any redo branch
   history.push({ v: ++version, label, at: Date.now(), days: deep(days), stays: deep(stays), bookings: deep(M.trip.bookings), title: M.trip.title });
@@ -1820,17 +1822,40 @@ async function ingestDoc(file) {
     reconfigureFromDoc(j);
   } catch { note.remove(); msg('ai', 'The document reader is unreachable right now.'); }
 }
+// attached documents stay in the model's context on EVERY turn, so follow-up
+// questions ("what’s my confirmation code?", "when’s the flight out?") work
+// without a re-upload
+const DOC_CTX_CAP = 20000;   // per-document budget inside the system prompt
+function docSection() {
+  const ds = Object.values(tripDocs);
+  if (!ds.length) return '';
+  return '\n\nAttached documents (the user’s own files, kept for reference on any question). For questions about their real bookings (confirmation codes, seats, prices, flight times), answer from these documents, not from the plan’s placeholder bookings:\n' +
+    ds.map(d => 'DOCUMENT “' + d.name + '”' + (d.truncated ? ' (extract truncated)' : '') + '\n' + (d.text || '').slice(0, DOC_CTX_CAP)).join('\n\n');
+}
+function renderDocChips() {
+  const el = $('#doc-chips'); if (!el) return;
+  const names = Object.keys(tripDocs);
+  if (!names.length) { el.classList.add('hidden'); el.innerHTML = ''; return; }
+  el.classList.remove('hidden');
+  el.innerHTML = names.map(n =>
+    '<span class="dc" title="' + esc(n) + ' — ' + (tripDocs[n].chars || 0) + ' chars, attached ' + new Date(tripDocs[n].at || 0).toLocaleDateString() + '">📎 ' + esc(n) + ' <span class="dc-k">' + (Math.round((tripDocs[n].chars || 0) / 100) / 10) + ' KB</span></span>').join(' ');
+}
 function reconfigureFromDoc(j) {
-  msg('user', '📎 ' + esc(j.name) + ' <span class="src">' + (j.kind || 'doc') + ' · ' + (j.chars || 0) + ' chars — reconfigure the trip around my flights</span>');
+  // keep the document on file: it persists with the trip and rides in the
-  if (!llmOn) return msg('ai', 'I read the booking (' + (j.chars || 0) + ' chars), but the model is offline so I can’t reconfigure around it yet. I’ll do it as soon as it’s back.');
+  // model's context on every turn
+  tripDocs[j.name] = { name: j.name, kind: j.kind, chars: j.chars, text: j.text, truncated: !!j.truncated, at: Date.now() };
+  persistTrip(); renderDocChips();
+  msg('user', '📎 ' + esc(j.name) + ' <span class="src">' + (j.kind || 'doc') + ' · ' + (j.chars || 0) + ' chars — attached' + (j.truncated ? ' (truncated)' : '') + '</span>');
+  if (!llmOn) return msg('ai', 'I read the document (' + (j.chars || 0) + ' chars) and kept it attached, but the model is offline so I can’t act on it yet. I’ll do it as soon as it’s back.');
   const prompt =
-    'I’ve attached my real flight booking for this trip (extracted text below).\n\n' +
+    'I’ve attached a document for this trip (extracted text below). It stays on file for later questions, so no need to re-list its contents in your reply.' + (j.truncated ? ' NOTE: the extraction was truncated at the end — if a needed detail is missing, say so and ask me to paste it.' : '') + '\n\n' +
-    'DOCUMENT “' + (j.name || 'booking') + '”\n' +
+    'DOCUMENT “' + (j.name || 'document') + '”\n' +
     '<document>\n' + (j.text || '') + '\n</document>\n\n' +
-    'Reconfigure this trip around it:\n' +
+    'If this is a flight booking (or contains actual flight times), reconfigure the trip around it:\n' +
     '1. Read the ARRIVAL (date, local time, airport, city) and the DEPARTURE (date, local time, airport, city). With connections, the arrival is where the trip starts (first city) and the departure where it ends (last city).\n' +
     '2. Call apply_flight_anchors exactly once with those two anchors.\n' +
-    '3. Confirm what changed — the new first and last day with their start/end times, and any stops that had to go. Keep it to a couple of sentences.';
+    '3. Confirm what changed — the new first and last day with their start/end times, and any stops that had to go, in a couple of sentences.\n' +
+    'If it is NOT a flight booking, do not change the plan — just confirm it’s attached and summarize what it contains in 1–2 sentences.';
   askLLM(prompt);
 }
 // drag & drop a file anywhere on the chat panel
@@ -1895,7 +1920,7 @@ let persistT = 0;
 function persistTrip() {
   clearTimeout(persistT);
   const hist = history.slice(0, hIdx + 1).slice(-25);   // drop the redo tail, cap at 25
-  persistT = setTimeout(() => tripStore.save(tripId, { days, stays, version, hIdx: hist.length - 1, history: hist, bookings: M.trip.bookings, title: M.trip.title }), 250);
+  persistT = setTimeout(() => tripStore.save(tripId, { days, stays, version, hIdx: hist.length - 1, history: hist, bookings: M.trip.bookings, title: M.trip.title, docs: tripDocs }), 250);
 }
 function renderTripMenu() {
   $('#crumb').innerHTML = `🧳 <span class="t-name">${M.trip.title}</span> <span class="caret">▾</span>`;
@@ -1927,7 +1952,8 @@ function enterTrip(id) {
     days = deep(saved.days); stays = deep(saved.stays); version = saved.version; history = saved.history; hIdx = saved.hIdx;
     if (saved.bookings) M.trip.bookings = deep(saved.bookings);
     if (saved.title != null) M.trip.title = saved.title;
-  }
+    tripDocs = saved.docs ? deep(saved.docs) : {};
+  } else tripDocs = {};
   // reset per-trip UI state
   compareItems = [];
   closeDiscover(); closeL3();
@@ -1944,6 +1970,7 @@ function enterTrip(id) {
   renderTripMenu();
   buildDayTabs();
   renderHotelMks(); renderBaseChip();
+  renderDocChips();
   // this trip may live on a different OSRM extract — re-check before routing
   Promise.all([checkRouter(), checkLLM()]).then(() => {
     const b = $('#offline-badge');

mock/index.html

@@ -111,9 +111,10 @@
   </div>
   <aside id="chat">
     <div id="chat-body"></div>
+    <div id="doc-chips" class="hidden"></div>
     <div id="chat-input-row">
-      <input id="doc-file" type="file" accept=".pdf,.txt,.ics,.html,.htm,.csv,.json" hidden />
+      <input id="doc-file" type="file" accept=".pdf,.txt,.ics,.html,.htm,.eml,.csv,.json" hidden />
-      <button class="btn small" id="doc-attach" title="Drop or attach your flight booking (PDF or text) to reconfigure the trip around it">📎</button>
+      <button class="btn small" id="doc-attach" title="Attach a booking or confirmation (PDF, HTML email, .eml, text) — flight bookings reconfigure the trip, and everything stays attached for reference">📎</button>
       <input id="chat-input" type="text" placeholder="Ask, direct — or drop a booking…" />
       <button class="btn primary small" id="chat-send">→</button>
     </div>

mock/server.js

@@ -42,6 +42,105 @@ const mime = {
   '.json': 'application/json', '.svg': 'image/svg+xml', '.png': 'image/png',
 };
 
+// ---- document text extraction (used by /parse-doc) --------------------
+// Real email HTML defeats the naive <[^>]+> strip (a ">" inside a quoted
+// attribute — e.g. an inline SVG data-URI — truncates the tag), so scan
+// char-by-char instead: a tag only ends at an UNQUOTED ">"; comments
+// (including MSO conditional comments) and whole style/script/head blocks
+// are skipped; block-level tags become line breaks; table cells get " | ".
+const ENT = { nbsp: ' ', amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", mdash: '—', ndash: '–', hellip: '…', rsquo: '’', lsquo: '‘', rdquo: '”', ldquo: '“', laquo: '«', raquo: '»', copy: '©', reg: '®', trade: '™', times: '×', deg: '°' };
+function decodeEntities(s) {
+  return String(s).replace(/&(#x?[0-9a-f]+|[a-z][a-z0-9]*);/gi, (m, e) => {
+    if (e[0] === '#') {
+      try { return String.fromCodePoint(e[1].toLowerCase() === 'x' ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10)); }
+      catch { return ' '; }
+    }
+    return Object.prototype.hasOwnProperty.call(ENT, e.toLowerCase()) ? ENT[e.toLowerCase()] : m;
+  });
+}
+function htmlToText(html) {
+  const BLOCK_END = /^<\/(p|div|tr|table|h[1-6]|li|ul|ol|section|header|footer|blockquote|pre|article|main|form|dl|dd|figure)\s*>$/i;
+  let out = '', i = 0, n = html.length;
+  while (i < n) {
+    const c = html[i];
+    if (c !== '<') { out += c; i++; continue; }
+    if (html.startsWith('<!--', i)) {                       // comment (incl. <!--[if gte mso 9]> … <![endif]-->)
+      const end = html.indexOf('-->', i + 4);
+      i = end < 0 ? n : end + 3; continue;
+    }
+    if (html.startsWith('<!', i) || html.startsWith('<[', i)) {   // doctype / stray conditional tag
+      const end = html.indexOf('>', i); i = end < 0 ? n : end + 1; continue;
+    }
+    const skip = /^<(script|style|head|title)\b/i.exec(html.slice(i, i + 40));
+    if (skip) {                                             // skip the whole block
+      const cm = new RegExp('</' + skip[1] + '\\s*>', 'i').exec(html.slice(i));
+      i = cm ? i + cm.index + cm[0].length : n; continue;
+    }
+    let j = i + 1, inQ = null;                              // find tag end, quote-aware
+    while (j < n) {
+      const d = html[j];
+      if (inQ) { if (d === inQ) inQ = null; }
+      else if (d === '"' || d === "'") inQ = d;
+      else if (d === '>') break;
+      j++;
+    }
+    const tag = html.slice(i, j < n ? j + 1 : n);
+    if (/^<br\b/i.test(tag)) out += '\n';
+    else if (BLOCK_END.test(tag)) out += '\n';
+    else if (/^<t[dh]\b/i.test(tag)) out += ' | ';
+    else out += ' ';
+    i = j < n ? j + 1 : n;
+  }
+  return decodeEntities(out);
+}
+// ---- email (MIME) decoding for .eml / raw mail -------------------------
+function looksLikeMime(t) {
+  const h = t.slice(0, 6000);
+  return /MIME-Version:\s*1\.0/im.test(h) || /Content-Transfer-Encoding:\s*(base64|quoted-printable)/im.test(h);
+}
+const decodeQP = (s) => String(s).replace(/=\r?\n/g, '').replace(/=([0-9a-fA-F]{2})/g, (_, hx) => String.fromCharCode(parseInt(hx, 16)));
+function decodeMime(raw) {
+  // best text part (html preferred, then longest); handles one level of
+  // nested multipart; returns null if nothing decodable was found
+  const head = raw.slice(0, 20000);
+  const bm = head.match(/boundary="?([^";\r\n]+)"?/i);
+  const parts = [];
+  if (bm) {
+    for (const chunk of raw.split(bm[1])) {
+      const nm = chunk.match(/\r?\n([\s\S]*?)\r?\n\r?\n([\s\S]*)/);
+      if (!nm) continue;
+      const h = nm[1];
+      const ct = ((h.match(/Content-Type:\s*([^\r\n;]+)/i) || [])[1] || '').trim().toLowerCase();
+      const ce = ((h.match(/Content-Transfer-Encoding:\s*([^\r\n]+)/i) || [])[1] || '').trim().toLowerCase();
+      const body = nm[2].replace(/\n--\s*$/, '');
+      if (body.trim()) parts.push({ ct, ce, body });
+    }
+  } else {
+    const nm = raw.match(/\r?\n([\s\S]*?)\r?\n\r?\n([\s\S]*)/);
+    if (nm) {
+      const h = nm[1];
+      parts.push({ ct: ((h.match(/Content-Type:\s*([^\r\n;]+)/i) || [])[1] || '').trim().toLowerCase(), ce: ((h.match(/Content-Transfer-Encoding:\s*([^\r\n]+)/i) || [])[1] || '').trim().toLowerCase(), body: nm[2] });
+    }
+  }
+  const dec = [];
+  for (const p of parts) {
+    if (/^multipart\//.test(p.ct)) {                        // nested multipart — recurse
+      const inner = decodeMime(p.body);
+      if (inner) dec.push({ html: /<\s*(html|body|div|table|p)\b/i.test(inner.slice(0, 2000)), len: inner.length, body: inner });
+      continue;
+    }
+    if (!/text\/(html|plain)/.test(p.ct)) continue;
+    let body = p.body;
+    try {
+      if (p.ce === 'base64') body = Buffer.from(body.replace(/\s+/g, ''), 'base64').toString('utf8');
+      else if (p.ce === 'quoted-printable') body = decodeQP(body);
+    } catch { continue; }
+    if (body.trim()) dec.push({ html: p.ct.includes('html'), len: body.length, body });
+  }
+  dec.sort((a, b) => (b.html - a.html) || (b.len - a.len));
+  return dec.length ? dec[0].body : null;
+}
+
 http.createServer((req, res) => {
   const u = new URL(req.url, 'http://x');
 
@@ -165,28 +264,48 @@ http.createServer((req, res) => {
 
   // ---- document ingestion (flight/booking → text) --------------------
   // POST /parse-doc — body { filename, mime, b64 }. Returns { ok, kind,
-  // name, text, chars }. PDFs go through pdftotext; text/ics/csv/json read
+  // name, text, chars, truncated }. PDFs go through pdftotext; emails (.eml
-  // directly; html tags are stripped. Images are rejected (no OCR here).
+  // or MIME-looking text) have their text parts decoded; HTML is reduced to
+  // clean text with htmlToText(); txt/ics/csv/json read directly. Images
+  // are rejected (no OCR on this box).
   if (u.pathname === '/parse-doc' && req.method === 'POST') {
     let body = '';
-    req.on('data', c => { body += c; if (body.length > 8e6) req.destroy(); });
+    req.on('data', c => { body += c; if (body.length > 24e6) req.destroy(); });
     req.on('end', () => {
       let in0; try { in0 = JSON.parse(body); } catch { res.writeHead(400); return res.end('bad json'); }
       const send = j => { res.setHeader('Content-Type', 'application/json'); res.writeHead(200); res.end(JSON.stringify(j)); };
       const name = String(in0.filename || 'document');
       const ext = path.extname(name).toLowerCase();
-      const mime = String(in0.mime || '');
+      const ctype = String(in0.mime || '');
-      if (/^image\//.test(mime) || ['.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp'].includes(ext))
+      if (/^image\//.test(ctype) || ['.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp'].includes(ext))
         return send({ ok: false, error: 'Screenshots and images aren’t supported yet (no OCR on this box). Save the booking as a PDF, or copy its text and paste it.' });
+      if (ext === '.msg')
+        return send({ ok: false, error: 'Outlook .msg files can’t be read here — in your mail app use “Forward as HTML” (or Save As → .eml) and drop that instead.' });
       let buf; try { buf = Buffer.from(String(in0.b64 || ''), 'base64'); } catch { return send({ ok: false, error: 'couldn’t read the file data' }); }
       if (!buf.length) return send({ ok: false, error: 'that file looks empty' });
-      if (buf.length > 4e6) return send({ ok: false, error: 'file too large (max ~4 MB)' });
+      if (buf.length > 16e6) return send({ ok: false, error: 'file too large (max ~16 MB)' });
+      const CAP = 60000;   // well under the 131k-token context; flag if we had to cut
       const finish = (text) => {
         text = String(text || '').replace(/\r\n/g, '\n').replace(/\t/g, '  ').replace(/[ \u00a0]{2,}/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
         if (!text) return send({ ok: false, error: 'no readable text found in that file' });
-        send({ ok: true, kind: ext.replace('.', '') || 'text', name, text: text.slice(0, 20000), chars: text.length });
+        // collapse heavy repetition (boilerplate fare rules etc.): keep the first
+        // two copies of any line that shows up 10+ times, then a single note
+        const lines = text.split('\n'), count = {};
+        lines.forEach(l => { const k = l.trim(); if (k) count[k] = (count[k] || 0) + 1; });
+        const seen = {}, out = [];
+        for (const l of lines) {
+          const k = l.trim();
+          if (k && count[k] >= 10) {
+            seen[k] = (seen[k] || 0) + 1;
+            if (seen[k] > 2) { if (out[out.length - 1] !== '__DUP__' + k) out.push('__DUP__' + k); continue; }
+          }
+          out.push(l);
+        }
+        text = out.map(l => l.startsWith('__DUP__') ? '  [… ' + count[l.slice(7)] + '× this line repeats — shown above]' : l).join('\n');
+        console.log(`[parse-doc] ${name} ${buf.length}B → ${text.length} chars${text.length > CAP ? ' (TRUNCATED at ' + CAP + ')' : ''}`);
+        send({ ok: true, kind: ext.replace('.', '') || 'text', name, text: text.slice(0, CAP), chars: text.length, truncated: text.length > CAP });
       };
-      if (ext === '.pdf' || mime.includes('pdf')) {
+      if (ext === '.pdf' || ctype.includes('pdf')) {
         const tmp = path.join(os.tmpdir(), 'parse-doc-' + Date.now() + '-' + Math.floor(Math.random() * 1e5) + '.pdf');
         fs.writeFile(tmp, buf, (werr) => {
           if (werr) return send({ ok: false, error: 'couldn’t write temp file' });
@@ -198,10 +317,12 @@ http.createServer((req, res) => {
         return;
       }
       let text = buf.toString('utf8');
-      if (ext === '.html' || ext === '.htm' || mime.includes('html'))
+      if (ext === '.eml' || looksLikeMime(text)) {
-        text = text.replace(/<script[\s\S]*?<\/script>/gi, ' ').replace(/<style[\s\S]*?<\/style>/gi, ' ')
+        const mimeBody = decodeMime(text);
-          .replace(/<br[^>]*>/gi, '\n').replace(/<\/?p[^>]*>/gi, '\n').replace(/<[^>]+>/g, ' ')
+        if (mimeBody != null) text = mimeBody;   // not really MIME → treat as plain text
-          .replace(/&nbsp;/g, ' ').replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&#39;|&apos;/g, "'").replace(/&quot;/g, '"');
+      }
+      if (ext === '.html' || ext === '.htm' || ctype.includes('html') || /<\s*(html|!doctype|head|body)\b/i.test(text.slice(0, 4000)))
+        text = htmlToText(text);
       finish(text);
     });
     return;

mock/styles.css

@@ -82,6 +82,10 @@ button { font: inherit; }
 #chat-body { flex: 1; overflow-y: auto; padding: 14px; display: flex; flex-direction: column; gap: 10px; }
 #chat-input-row { display: flex; gap: 8px; padding: 12px; border-top: 1px solid var(--line); align-items: center; }
 #doc-attach { cursor: pointer; line-height: 1; }
+#doc-chips { display: flex; flex-wrap: wrap; gap: 4px; padding: 6px 12px 0; max-height: 54px; overflow-y: auto; }
+#doc-chips.hidden { display: none; }
+.dc { font-size: 11px; background: #eef0f4; border: 1px solid var(--line); color: var(--ink); border-radius: 10px; padding: 1px 8px; white-space: nowrap; max-width: 100%; overflow: hidden; text-overflow: ellipsis; }
+.dc-k { opacity: .55; font-family: ui-monospace, Menlo, Consolas, monospace; font-size: 10px; }
 #chat { position: relative; }
 #chat.dropping::after {
   content: 'Drop the flight booking to reconfigure the trip';