Greg Pomerantz
91e93ec265
The real-email case exposed two gaps:
1. Confirmation friction: when the booked dates didn't match the plan,
the model asked to confirm instead of anchoring. The guideline now
says to apply directly (the user's ticket is the truth, and the edit
is undo-able) and report the date shift — verified: a dropped booking
now re-anchors in one round, and a redundant "yes" afterwards is
handled gracefully ("already done").
2. Amnesia on the follow-up: the model has no conversation memory, so a
confirming "yes" round had to re-derive the dates from the document
in context — and that copy was clipped at 20k chars. Now the recent
turns ride in the model's context (chatLog, ~12k char budget, upload
messages excluded since docSection carries the document), and the doc
is no longer clipped in-context at all (60k server cap ≪ 131k-token
window). A new guideline tells the model to apply a confirmed change
from its own earlier message rather than claiming it can't see info
it already quoted.
Extraction hardening: broader HTML detection (20k-char scan + tag
density), and a brute-force base64-block fallback in decodeMime for
mail clients with quirky MIME structure. The server now logs an
extraction preview and saves the exact model-facing text to
/tmp/parse-doc-last.txt for debugging real files.
Co-Authored-By: Claude <noreply@anthropic.com>
356 lines
18 KiB
JavaScript
356 lines
18 KiB
JavaScript
// Mock server: static UI + local tile backend (proxy -> OSM, disk-cached)
|
||
// + routing backend (proxy -> local OSRM instances; per-trip extracts).
|
||
// The UI only ever talks to localhost; swap the upstream for a real
|
||
// tileserver-gl / OSRM later without touching the frontend.
|
||
const http = require('http');
|
||
const fs = require('fs');
|
||
const path = require('path');
|
||
const os = require('os');
|
||
const { execFile } = require('child_process');
|
||
|
||
const PORT = 8077;
|
||
const ROOT = __dirname;
|
||
// One upstream OSRM per dataset. `probe` is a short routable pair inside the
|
||
// extract, used by /router-status (health checks return Ok for 0-distance
|
||
// queries only when the graph has edges there).
|
||
const ROUTERS = {
|
||
northeast: {
|
||
url: process.env.OSRM_NORTHEAST || 'http://localhost:5000',
|
||
probe: '-71.06,42.35;-71.07,42.36', // Boston, car extract
|
||
},
|
||
colombia: {
|
||
url: process.env.OSRM_COLOMBIA || 'http://localhost:5003',
|
||
probe: '-75.5478,10.3954;-75.5400,10.4020', // Cartagena, foot extract
|
||
},
|
||
};
|
||
// LLM backend: local llama.cpp OpenAI-compatible server (see ~/.pi/agent/
|
||
// models.json). Model + budget are pinned server-side; the client only ever
|
||
// sends messages. Qwen3.8-27B-UD3-Q5-dual: --parallel 2, --ctx-size 131072.
|
||
const LLM = {
|
||
base: process.env.LLM_BASE || 'http://192.168.3.7:1234',
|
||
model: process.env.LLM_MODEL || 'Qwen3.8-27B-UD3-Q5-dual',
|
||
ctx: 131072,
|
||
maxTokens: +(process.env.LLM_MAX_TOKENS || 8192), // thinking + answer share this budget
|
||
};
|
||
const routerFor = (u) => ROUTERS[u.searchParams.get('router') || 'northeast'] || ROUTERS.northeast;
|
||
const PROFILE = { foot: 'walking', walk: 'walking', bike: 'cycling', cycle: 'cycling', car: 'driving', drive: 'driving', transit: 'driving' };
|
||
const CACHE = path.join(__dirname, '.tilecache');
|
||
fs.mkdirSync(CACHE, { recursive: true });
|
||
|
||
const mime = {
|
||
'.html': 'text/html', '.js': 'text/javascript', '.css': 'text/css',
|
||
'.json': 'application/json', '.svg': 'image/svg+xml', '.png': 'image/png',
|
||
};
|
||
|
||
// ---- document text extraction (used by /parse-doc) --------------------
|
||
// Real email HTML defeats the naive <[^>]+> strip (a ">" inside a quoted
|
||
// attribute — e.g. an inline SVG data-URI — truncates the tag), so scan
|
||
// char-by-char instead: a tag only ends at an UNQUOTED ">"; comments
|
||
// (including MSO conditional comments) and whole style/script/head blocks
|
||
// are skipped; block-level tags become line breaks; table cells get " | ".
|
||
const ENT = { nbsp: ' ', amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", mdash: '—', ndash: '–', hellip: '…', rsquo: '’', lsquo: '‘', rdquo: '”', ldquo: '“', laquo: '«', raquo: '»', copy: '©', reg: '®', trade: '™', times: '×', deg: '°' };
|
||
function decodeEntities(s) {
|
||
return String(s).replace(/&(#x?[0-9a-f]+|[a-z][a-z0-9]*);/gi, (m, e) => {
|
||
if (e[0] === '#') {
|
||
try { return String.fromCodePoint(e[1].toLowerCase() === 'x' ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10)); }
|
||
catch { return ' '; }
|
||
}
|
||
return Object.prototype.hasOwnProperty.call(ENT, e.toLowerCase()) ? ENT[e.toLowerCase()] : m;
|
||
});
|
||
}
|
||
function htmlToText(html) {
|
||
const BLOCK_END = /^<\/(p|div|tr|table|h[1-6]|li|ul|ol|section|header|footer|blockquote|pre|article|main|form|dl|dd|figure)\s*>$/i;
|
||
let out = '', i = 0, n = html.length;
|
||
while (i < n) {
|
||
const c = html[i];
|
||
if (c !== '<') { out += c; i++; continue; }
|
||
if (html.startsWith('<!--', i)) { // comment (incl. <!--[if gte mso 9]> … <![endif]-->)
|
||
const end = html.indexOf('-->', i + 4);
|
||
i = end < 0 ? n : end + 3; continue;
|
||
}
|
||
if (html.startsWith('<!', i) || html.startsWith('<[', i)) { // doctype / stray conditional tag
|
||
const end = html.indexOf('>', i); i = end < 0 ? n : end + 1; continue;
|
||
}
|
||
const skip = /^<(script|style|head|title)\b/i.exec(html.slice(i, i + 40));
|
||
if (skip) { // skip the whole block
|
||
const cm = new RegExp('</' + skip[1] + '\\s*>', 'i').exec(html.slice(i));
|
||
i = cm ? i + cm.index + cm[0].length : n; continue;
|
||
}
|
||
let j = i + 1, inQ = null; // find tag end, quote-aware
|
||
while (j < n) {
|
||
const d = html[j];
|
||
if (inQ) { if (d === inQ) inQ = null; }
|
||
else if (d === '"' || d === "'") inQ = d;
|
||
else if (d === '>') break;
|
||
j++;
|
||
}
|
||
const tag = html.slice(i, j < n ? j + 1 : n);
|
||
if (/^<br\b/i.test(tag)) out += '\n';
|
||
else if (BLOCK_END.test(tag)) out += '\n';
|
||
else if (/^<t[dh]\b/i.test(tag)) out += ' | ';
|
||
else out += ' ';
|
||
i = j < n ? j + 1 : n;
|
||
}
|
||
return decodeEntities(out);
|
||
}
|
||
// ---- email (MIME) decoding for .eml / raw mail -------------------------
|
||
function looksLikeMime(t) {
|
||
const h = t.slice(0, 6000);
|
||
return /MIME-Version:\s*1\.0/im.test(h) || /Content-Transfer-Encoding:\s*(base64|quoted-printable)/im.test(h);
|
||
}
|
||
const decodeQP = (s) => String(s).replace(/=\r?\n/g, '').replace(/=([0-9a-fA-F]{2})/g, (_, hx) => String.fromCharCode(parseInt(hx, 16)));
|
||
function decodeMime(raw) {
|
||
// best text part (html preferred, then longest); handles one level of
|
||
// nested multipart; returns null if nothing decodable was found
|
||
const head = raw.slice(0, 20000);
|
||
const bm = head.match(/boundary="?([^";\r\n]+)"?/i);
|
||
const parts = [];
|
||
if (bm) {
|
||
for (const chunk of raw.split(bm[1])) {
|
||
const nm = chunk.match(/\r?\n([\s\S]*?)\r?\n\r?\n([\s\S]*)/);
|
||
if (!nm) continue;
|
||
const h = nm[1];
|
||
const ct = ((h.match(/Content-Type:\s*([^\r\n;]+)/i) || [])[1] || '').trim().toLowerCase();
|
||
const ce = ((h.match(/Content-Transfer-Encoding:\s*([^\r\n]+)/i) || [])[1] || '').trim().toLowerCase();
|
||
const body = nm[2].replace(/\n--\s*$/, '');
|
||
if (body.trim()) parts.push({ ct, ce, body });
|
||
}
|
||
} else {
|
||
const nm = raw.match(/\r?\n([\s\S]*?)\r?\n\r?\n([\s\S]*)/);
|
||
if (nm) {
|
||
const h = nm[1];
|
||
parts.push({ ct: ((h.match(/Content-Type:\s*([^\r\n;]+)/i) || [])[1] || '').trim().toLowerCase(), ce: ((h.match(/Content-Transfer-Encoding:\s*([^\r\n]+)/i) || [])[1] || '').trim().toLowerCase(), body: nm[2] });
|
||
}
|
||
}
|
||
const dec = [];
|
||
for (const p of parts) {
|
||
if (/^multipart\//.test(p.ct)) { // nested multipart — recurse
|
||
const inner = decodeMime(p.body);
|
||
if (inner) dec.push({ html: /<\s*(html|body|div|table|p)\b/i.test(inner.slice(0, 2000)), len: inner.length, body: inner });
|
||
continue;
|
||
}
|
||
if (!/text\/(html|plain)/.test(p.ct)) continue;
|
||
let body = p.body;
|
||
try {
|
||
if (p.ce === 'base64') body = Buffer.from(body.replace(/\s+/g, ''), 'base64').toString('utf8');
|
||
else if (p.ce === 'quoted-printable') body = decodeQP(body);
|
||
} catch { continue; }
|
||
if (body.trim()) dec.push({ html: p.ct.includes('html'), len: body.length, body });
|
||
}
|
||
dec.sort((a, b) => (b.html - a.html) || (b.len - a.len));
|
||
if (dec.length) return dec[0].body;
|
||
// last resort: brute-force the biggest base64 block (mail clients vary in
|
||
// boundary quoting, charset headers, etc.)
|
||
const blocks = raw.match(/[A-Za-z0-9+\/\n=]{400,}/g) || [];
|
||
for (const b of blocks.slice(0, 12)) {
|
||
try {
|
||
const d = Buffer.from(b.replace(/\s+/g, ''), 'base64').toString('utf8');
|
||
if (d.length > 200 && /[a-z]{4,}/i.test(d))
|
||
dec.push({ html: /<\s*(html|body|div|table|p)\b/i.test(d.slice(0, 2000)), len: d.length, body: d });
|
||
} catch {}
|
||
}
|
||
dec.sort((a, b) => (b.html - a.html) || (b.len - a.len));
|
||
return dec.length ? dec[0].body : null;
|
||
}
|
||
|
||
http.createServer((req, res) => {
|
||
const u = new URL(req.url, 'http://x');
|
||
|
||
// ---- routing backend (proxy -> local OSRM) ------------------------
|
||
// /route?mode=foot&points=lng,lat;lng,lat[;...]
|
||
// /table?mode=foot&points=lng,lat;lng,lat[;...]
|
||
if (u.pathname === '/route' || u.pathname === '/table') {
|
||
const kind = u.pathname.slice(1);
|
||
const mode = u.searchParams.get('mode') || 'foot';
|
||
const points = u.searchParams.get('points') || '';
|
||
const profile = PROFILE[mode] || 'driving';
|
||
let osrmPath = `/${kind}/v1/${profile}/${encodeURIComponent(points)}`;
|
||
if (kind === 'route') osrmPath += '?overview=full&alternatives=false&steps=false';
|
||
else {
|
||
// table: durations from the first point (the anchor) to every point
|
||
const n = points.split(';').filter(Boolean).length;
|
||
const targets = Array.from({ length: n }, (_, i) => i).join(',');
|
||
osrmPath += `?annotations=duration,distance&sources=0&targets=${targets}`;
|
||
}
|
||
const up = routerFor(u).url;
|
||
const fetchUp = fetch(up + osrmPath).then(r => r.text()).catch(() => null);
|
||
fetchUp.then(body => {
|
||
res.setHeader('Access-Control-Allow-Origin', '*');
|
||
res.setHeader('Content-Type', 'application/json');
|
||
if (body == null) { res.writeHead(502); return res.end(JSON.stringify({ code: 'ProxyError' })); }
|
||
res.writeHead(200); res.end(body);
|
||
});
|
||
return;
|
||
}
|
||
// routing availability probe (so the UI can label router vs estimate)
|
||
if (u.pathname === '/router-status') {
|
||
const rt = routerFor(u);
|
||
fetch(rt.url + '/route/v1/walking/' + rt.probe + '?overview=false').then(r => r.text())
|
||
.then(t => { let ok = false; try { ok = JSON.parse(t).code === 'Ok' && JSON.parse(t).routes?.[0]?.distance > 0; } catch (e) {}
|
||
res.setHeader('Content-Type', 'application/json'); res.writeHead(200); res.end(JSON.stringify({ router: ok, osrm: rt.url, key: rt === ROUTERS.northeast ? 'northeast' : 'colombia' })); })
|
||
.catch(() => { res.setHeader('Content-Type', 'application/json'); res.writeHead(200); res.end(JSON.stringify({ router: false, osrm: rt.url })); });
|
||
return;
|
||
}
|
||
|
||
// ---- tile backend -------------------------------------------------
|
||
if (u.pathname.startsWith('/tiles/')) {
|
||
const parts = u.pathname.split('/');
|
||
const [z, x, y] = [parts[2], parts[3], (parts[4] || '').replace(/\.png$/, '')];
|
||
if (!/^\d+$/.test(z) || !/^\d+$/.test(x) || !/^\d+$/.test(y)) { res.writeHead(400); return res.end(); }
|
||
const file = path.join(CACHE, `${z}/${x}`, `${y}.png`);
|
||
const upstream = `https://tile.openstreetmap.org/${z}/${x}/${y}.png`;
|
||
res.setHeader('Access-Control-Allow-Origin', '*');
|
||
res.setHeader('Content-Type', 'image/png');
|
||
res.setHeader('Cache-Control', 'public, max-age=86400');
|
||
if (fs.existsSync(file)) { res.writeHead(200); fs.createReadStream(file).pipe(res); return; }
|
||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||
const fail = (code) => { if (!res.headersSent) { res.writeHead(code); res.end(); } else res.destroy(); };
|
||
// buffer the whole tile (≤100 KB), write the cache file, then reply —
|
||
// no stream-splitting races with a client that reads fast
|
||
fetch(upstream, { headers: { 'User-Agent': 'mapmock-dev/0.1 (local tile proxy)', 'Referer': `http://localhost:${PORT}/` } })
|
||
.then(async r => {
|
||
if (!r.ok) { return fail(r.status); }
|
||
const buf = Buffer.from(await r.arrayBuffer());
|
||
if (buf.length < 20) { return fail(502); } // empty/garbage tile
|
||
try { fs.writeFileSync(file, buf); } catch (e) {}
|
||
if (res.writableEnded) return;
|
||
res.writeHead(200); res.end(buf);
|
||
})
|
||
.catch(() => fail(502));
|
||
return;
|
||
}
|
||
|
||
// ---- LLM backend (proxy -> local llama.cpp server) ----------------
|
||
// GET /llm-status — availability probe (mirrors /router-status)
|
||
if (u.pathname === '/llm-status') {
|
||
fetch(LLM.base + '/v1/models').then(r => r.json())
|
||
.then(j => {
|
||
const m = (j.data || []).find(x => x.id === LLM.model);
|
||
res.setHeader('Access-Control-Allow-Origin', '*');
|
||
res.setHeader('Content-Type', 'application/json');
|
||
res.writeHead(200);
|
||
res.end(JSON.stringify({
|
||
llm: !!(m && m.status && m.status.value === 'loaded'),
|
||
model: LLM.model, base: LLM.base, ctx: LLM.ctx,
|
||
status: (m && m.status && m.status.value) || 'unknown',
|
||
}));
|
||
})
|
||
.catch(() => {
|
||
res.setHeader('Access-Control-Allow-Origin', '*');
|
||
res.setHeader('Content-Type', 'application/json');
|
||
res.writeHead(200); res.end(JSON.stringify({ llm: false, model: LLM.model, base: LLM.base, status: 'unreachable' }));
|
||
});
|
||
return;
|
||
}
|
||
// POST /llm-chat — body { messages, max_tokens?, temperature? }.
|
||
// Non-streaming; the model is a reasoning model, so the server-side budget
|
||
// must cover hidden thinking plus the visible answer.
|
||
if (u.pathname === '/llm-chat' && req.method === 'POST') {
|
||
let body = '';
|
||
req.on('data', c => { body += c; if (body.length > 2e6) req.destroy(); });
|
||
req.on('end', () => {
|
||
let in0;
|
||
try { in0 = JSON.parse(body); } catch { res.writeHead(400); return res.end('bad json'); }
|
||
const payload = {
|
||
model: LLM.model,
|
||
messages: Array.isArray(in0.messages) ? in0.messages : [],
|
||
max_tokens: in0.max_tokens || LLM.maxTokens,
|
||
temperature: in0.temperature ?? 0.7,
|
||
stream: false,
|
||
};
|
||
res.setHeader('Access-Control-Allow-Origin', '*');
|
||
res.setHeader('Content-Type', 'application/json');
|
||
fetch(LLM.base + '/v1/chat/completions', {
|
||
method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload),
|
||
}).then(async r => {
|
||
const t = await r.text();
|
||
res.writeHead(r.ok ? 200 : 502);
|
||
res.end(t);
|
||
}).catch(() => {
|
||
res.writeHead(502);
|
||
res.end(JSON.stringify({ error: 'LLM server unreachable at ' + LLM.base }));
|
||
});
|
||
});
|
||
return;
|
||
}
|
||
|
||
// ---- document ingestion (flight/booking → text) --------------------
|
||
// POST /parse-doc — body { filename, mime, b64 }. Returns { ok, kind,
|
||
// name, text, chars, truncated }. PDFs go through pdftotext; emails (.eml
|
||
// or MIME-looking text) have their text parts decoded; HTML is reduced to
|
||
// clean text with htmlToText(); txt/ics/csv/json read directly. Images
|
||
// are rejected (no OCR on this box).
|
||
if (u.pathname === '/parse-doc' && req.method === 'POST') {
|
||
let body = '';
|
||
req.on('data', c => { body += c; if (body.length > 24e6) req.destroy(); });
|
||
req.on('end', () => {
|
||
let in0; try { in0 = JSON.parse(body); } catch { res.writeHead(400); return res.end('bad json'); }
|
||
const send = j => { res.setHeader('Content-Type', 'application/json'); res.writeHead(200); res.end(JSON.stringify(j)); };
|
||
const name = String(in0.filename || 'document');
|
||
const ext = path.extname(name).toLowerCase();
|
||
const ctype = String(in0.mime || '');
|
||
if (/^image\//.test(ctype) || ['.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp'].includes(ext))
|
||
return send({ ok: false, error: 'Screenshots and images aren’t supported yet (no OCR on this box). Save the booking as a PDF, or copy its text and paste it.' });
|
||
if (ext === '.msg')
|
||
return send({ ok: false, error: 'Outlook .msg files can’t be read here — in your mail app use “Forward as HTML” (or Save As → .eml) and drop that instead.' });
|
||
let buf; try { buf = Buffer.from(String(in0.b64 || ''), 'base64'); } catch { return send({ ok: false, error: 'couldn’t read the file data' }); }
|
||
if (!buf.length) return send({ ok: false, error: 'that file looks empty' });
|
||
if (buf.length > 16e6) return send({ ok: false, error: 'file too large (max ~16 MB)' });
|
||
const CAP = 60000; // well under the 131k-token context; flag if we had to cut
|
||
const finish = (text) => {
|
||
text = String(text || '').replace(/\r\n/g, '\n').replace(/\t/g, ' ').replace(/[ \u00a0]{2,}/g, ' ').replace(/\n{3,}/g, '\n\n').trim();
|
||
if (!text) return send({ ok: false, error: 'no readable text found in that file' });
|
||
// collapse heavy repetition (boilerplate fare rules etc.): keep the first
|
||
// two copies of any line that shows up 10+ times, then a single note
|
||
const lines = text.split('\n'), count = {};
|
||
lines.forEach(l => { const k = l.trim(); if (k) count[k] = (count[k] || 0) + 1; });
|
||
const seen = {}, out = [];
|
||
for (const l of lines) {
|
||
const k = l.trim();
|
||
if (k && count[k] >= 10) {
|
||
seen[k] = (seen[k] || 0) + 1;
|
||
if (seen[k] > 2) { if (out[out.length - 1] !== '__DUP__' + k) out.push('__DUP__' + k); continue; }
|
||
}
|
||
out.push(l);
|
||
}
|
||
text = out.map(l => l.startsWith('__DUP__') ? ' [… ' + count[l.slice(7)] + '× this line repeats — shown above]' : l).join('\n');
|
||
try { fs.writeFileSync('/tmp/parse-doc-last.txt', 'file: ' + name + ' (' + buf.length + ' bytes) → ' + text.length + ' chars\n' + '=== extracted text (exactly what the model receives) ===\n' + text); } catch {}
|
||
console.log(`[parse-doc] ${name} ${buf.length}B → ${text.length} chars${text.length > CAP ? ' (TRUNCATED at ' + CAP + ')' : ''} | preview: ${text.slice(0, 200).replace(/\n/g, ' ⏎ ')}`);
|
||
send({ ok: true, kind: ext.replace('.', '') || 'text', name, text: text.slice(0, CAP), chars: text.length, truncated: text.length > CAP });
|
||
};
|
||
if (ext === '.pdf' || ctype.includes('pdf')) {
|
||
const tmp = path.join(os.tmpdir(), 'parse-doc-' + Date.now() + '-' + Math.floor(Math.random() * 1e5) + '.pdf');
|
||
fs.writeFile(tmp, buf, (werr) => {
|
||
if (werr) return send({ ok: false, error: 'couldn’t write temp file' });
|
||
execFile('pdftotext', ['-layout', tmp, '-'], (err, stdout) => {
|
||
try { fs.unlinkSync(tmp); } catch {}
|
||
finish(err ? '' : stdout);
|
||
});
|
||
});
|
||
return;
|
||
}
|
||
let text = buf.toString('utf8');
|
||
if (ext === '.eml' || looksLikeMime(text)) {
|
||
const mimeBody = decodeMime(text);
|
||
if (mimeBody != null) text = mimeBody; // not really MIME → treat as plain text
|
||
}
|
||
const head = text.slice(0, 20000);
|
||
const tagDensity = (head.match(/<[a-z][\s\/a-z>]/gi) || []).length;
|
||
if (ext === '.html' || ext === '.htm' || ctype.includes('html') || /<\s*(html|!doctype|head|body)\b/i.test(head) || tagDensity > 30)
|
||
text = htmlToText(text);
|
||
finish(text);
|
||
});
|
||
return;
|
||
}
|
||
|
||
// ---- static -------------------------------------------------------
|
||
let p = u.pathname === '/' ? '/index.html' : u.pathname;
|
||
const file = path.join(ROOT, p);
|
||
if (!file.startsWith(ROOT)) { res.writeHead(403); return res.end(); }
|
||
fs.readFile(file, (e, d) => {
|
||
if (e) { res.writeHead(404); return res.end('not found'); }
|
||
res.writeHead(200, { 'Content-Type': mime[path.extname(file)] || 'application/octet-stream' });
|
||
res.end(d);
|
||
});
|
||
}).listen(PORT, '0.0.0.0', () => console.log(`map mock: http://localhost:${PORT} (tiles: /tiles/{z}/{x}/{y}.png · llm: ${LLM.base} ${LLM.model})`));
|