Files
plan2code/scripts/test-web-console-chat.mjs

480 lines
23 KiB
JavaScript

// Web console tests, Quick questions: the wait contract, delivery, limits and resets.
// Run with: node --test scripts/test-web-console-chat.mjs
//
// Shared setup (a per-process console home, the CLI runner, fixtures) lives in
// web-console-test-helpers.mjs; see the comment at its top.
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
import { spawn } from "node:child_process";
import { terminalLine } from "../src/web-console/sessions.mjs";
import { readChat } from "../src/web-console/inbox.mjs";
import {
ROOT,
CONSOLE_CLI,
ENV,
cli,
safeJson,
JPEG,
sessionUrl,
readState,
openSession,
sleep,
postPatch,
chatFrame,
ask,
resetsIn,
until,
EXPORT_A,
} from "./web-console-test-helpers.mjs";
/* ------------------------------------------- quick question: the contract */
// Every outcome of wait, pinned by exit code and by its exact top-level keys.
// The agent reads a trimmed line (spec "API Design (outputs)"): what the
// person sent lives in result.json, so those fields are checked there.
const pinKeys = (obj, expected, what) =>
assert.deepEqual(Object.keys(obj).filter((k) => k !== "terminalLine").sort(), [...expected].sort(), what + ": top-level keys changed");
const RESULT_FILE = (s) => JSON.parse(fs.readFileSync(path.join(s.session, "result.json"), "utf8"));
test("wait's existing outcomes are pinned: submit, images, waiting, cancel, carried and server gone", async () => {
const s = openSession();
try {
const waiting = cli(["wait", "--session", s.sid, "--seconds", "5"]);
assert.equal(waiting.status, 10);
assert.equal(waiting.json.waiting, true);
assert.equal(waiting.json.open, 1, "the required question still open");
pinKeys(waiting.json, ["waiting", "open"], "exit 10");
assert.equal((await s.send("/submit", { actions: [{ i: "q1", type: "answer", kind: "choice", k: "A" }], reply: "Export format: a" })).status, 200);
const sent = cli(["wait", "--session", s.sid, "--seconds", "10"]);
assert.equal(sent.status, 0);
assert.equal(sent.json.reply, "Export format: a");
pinKeys(sent.json, ["reply"], "exit 0 submit");
const kept = RESULT_FILE(s);
for (const k of ["type", "at", "sid", "rev", "actions", "reply", "consumedAt"]) assert.ok(k in kept, "result.json keeps " + k);
assert.equal(kept.type, "submit");
const up = await fetch(s.base + "/upload", {
method: "POST",
headers: { "content-type": "image/jpeg", cookie: s.cookie, origin: s.base, connection: "close", "x-p2c-name": "a.png" },
body: JPEG,
});
const img = await up.json();
assert.equal(
(await s.send("/submit", { actions: [{ i: "q1", type: "comment", text: "See this.", images: [{ path: img.path, name: img.name }] }], reply: "Note" })).status,
200
);
const withImages = cli(["wait", "--session", s.sid, "--seconds", "10"]);
assert.equal(withImages.status, 0);
pinKeys(withImages.json, ["reply"], "exit 0 with images");
assert.equal(RESULT_FILE(s).actions[0].images.length, 1, "result.json holds the image");
// Sent before any wait is running: the next wait hands it over at once.
assert.equal((await s.send("/submit", { actions: [{ i: "q1", type: "answer", kind: "choice", k: "B" }], reply: "Export format: b" })).status, 200);
const t0 = Date.now();
const carried = cli(["wait", "--session", s.sid, "--seconds", "30"]);
assert.equal(carried.status, 0);
assert.ok(Date.now() - t0 < 5000, "an uncollected result is delivered at once");
assert.equal(carried.json.reply, "Export format: b");
pinKeys(carried.json, ["reply"], "exit 0 carried");
assert.equal((await s.send("/cancel", {})).status, 200);
const cancelled = cli(["wait", "--session", s.sid, "--seconds", "5"]);
assert.equal(cancelled.status, 30);
assert.equal(RESULT_FILE(s).type, "cancel");
} finally {
cli(["stop", "--session", s.sid, "--force"]);
}
const gone = cli(["wait", "--session", s.sid, "--seconds", "12"]);
assert.equal(gone.status, 20);
assert.equal(gone.json.gone, true);
assert.ok(gone.json.fix.includes(s.sid));
pinKeys(gone.json, ["gone", "fix"], "exit 20");
});
/* --------------------------------------------- quick question: the channel */
test("a chat message and a card send can both be pending, in either order, and wait hands both over at once", async () => {
const s = openSession();
try {
assert.equal((await s.send("/submit", EXPORT_A)).status, 200);
assert.equal((await ask(s, "Where does the export live?")).status, 200, "a pending card never blocks a question");
const both = cli(["wait", "--session", s.sid, "--seconds", "10"]);
assert.equal(both.status, 0);
assert.equal(both.json.chat.length, 1, "chat travels as the seq numbers a reply needs");
assert.equal(typeof both.json.chat[0], "number");
assert.match(both.json.reply, /^Export format: a\n/);
assert.match(both.json.reply, /^Quick question \(\d+\): Where does the export live\?$/m);
assert.equal((await ask(s, "And the header row?")).status, 200);
assert.equal((await s.send("/submit", EXPORT_A)).status, 200, "a pending question never blocks a card");
const again = cli(["wait", "--session", s.sid, "--seconds", "10"]);
assert.equal(again.status, 0);
assert.equal(again.json.chat.length, 1);
assert.match(again.json.reply, /Quick question \(\d+\): And the header row\?/);
assert.equal((await ask(s, "Just a question")).status, 200);
const chatOnly = cli(["wait", "--session", s.sid, "--seconds", "10"]);
assert.equal(chatOnly.status, 0);
assert.equal(chatOnly.json.chat.length, 1);
assert.equal(chatOnly.json.reply, "Quick question (" + chatOnly.json.chat[0] + "): Just a question");
assert.equal(chatOnly.json.type, undefined);
assert.equal(chatOnly.json.actions, undefined);
} finally {
cli(["stop", "--session", s.sid, "--force"]);
}
});
test("wait returns within 100 ms of a chat message landing", async () => {
const s = openSession();
try {
const child = spawn(process.execPath, [CONSOLE_CLI, "wait", "--session", s.sid, "--seconds", "20"], { env: ENV, cwd: ROOT });
let out = "";
const printed = new Promise((resolve) =>
child.stdout.on("data", (d) => {
out += d;
resolve(Date.now());
})
);
const exited = new Promise((resolve) => child.on("exit", resolve));
// wait checks the server's health as it starts and then every 5 s; land
// the message between the two, which takes longer to reach while other
// test files load the machine. The bound still has to be tighter than the
// 5 s poll cadence it guards against, but wide enough that a slow spawn
// (the message arriving before wait's first poll) does not flake it.
await sleep(2500);
assert.equal((await ask(s, "Quick one")).status, 200);
const answeredAt = Date.now();
assert.ok((await printed) - answeredAt < 2000, "wait must hand the chat over well inside its 5 s poll");
assert.equal(await exited, 0);
assert.match(safeJson(out).reply, /^Quick question \(\d+\): Quick one$/m);
} finally {
cli(["stop", "--session", s.sid, "--force"]);
}
});
test("the chat command answers at once: 0 with entries, 10 with none, 20 when the server is gone, and never takes a card send", async () => {
const s = openSession();
try {
assert.equal(cli(["chat", "--session", s.sid]).status, 10, "a reset alone is nothing to hand over");
assert.equal((await s.send("/submit", EXPORT_A)).status, 200);
assert.equal((await ask(s, "Mid-build question")).status, 200);
assert.equal((await (await s.send("/ping", {})).json()).pendingChat, 1, "an uncollected question shows on the heartbeat");
const got = cli(["chat", "--session", s.sid]);
assert.equal(got.status, 0);
assert.equal(got.json.chat.length, 1);
assert.match(got.json.reply, /Quick question \(\d+\): Mid-build question/);
const onDisk = JSON.parse(fs.readFileSync(path.join(s.session, "result.json"), "utf8"));
assert.equal(onDisk.consumedAt, undefined, "the card send is still waiting for wait");
assert.equal((await (await s.send("/ping", {})).json()).pendingChat, 0);
const none = cli(["chat", "--session", s.sid]);
assert.equal(none.status, 10);
assert.deepEqual(none.json, {});
const card = cli(["wait", "--session", s.sid, "--seconds", "10"]);
assert.equal(card.status, 0);
assert.match(card.json.reply, /^Export format: a/);
assert.equal(card.json.chat, undefined, "the chat went with the chat command, not twice");
assert.ok(!card.json.reply.includes("Mid-build question"));
} finally {
cli(["stop", "--session", s.sid, "--force"]);
}
const gone = cli(["chat", "--session", s.sid]);
assert.equal(gone.status, 20);
assert.equal(gone.json.gone, true);
});
test("a torn trailing inbox line is skipped, then collected once it is whole", async () => {
const s = openSession();
try {
const file = path.join(s.session, "chat.ndjson");
const seq = Math.max(...readChat(s.session).map((e) => e.seq)) + 1;
const line = JSON.stringify({ kind: "message", conversation: 1, text: "torn in two", seq, at: new Date().toISOString() });
fs.appendFileSync(file, line.slice(0, 20));
assert.equal(readChat(s.session).some((e) => e.seq === seq), false, "a half-written line is not parsed");
assert.equal(cli(["chat", "--session", s.sid]).status, 10);
fs.appendFileSync(file, line.slice(20) + "\n");
assert.ok(readChat(s.session).some((e) => e.seq === seq));
const got = cli(["chat", "--session", s.sid]);
assert.equal(got.status, 0);
assert.match(got.json.reply, /Quick question \(\d+\): torn in two/);
} finally {
cli(["stop", "--session", s.sid, "--force"]);
}
});
test("a deleted cursor never causes a second answer to a question that has a reply", async () => {
const s = openSession();
try {
const { body } = await ask(s, "What is the port?");
assert.equal(cli(["chat", "--session", s.sid]).status, 0);
assert.equal(postPatch(s, { chat: { replies: [{ id: "r1", re: body.seq, conversation: 1, md: "Ephemeral." }] } }).status, 0);
fs.rmSync(path.join(s.session, "chat-cursor.json"));
const again = cli(["chat", "--session", s.sid]);
assert.equal(again.status, 10, "the answered question is not handed over again");
} finally {
cli(["stop", "--session", s.sid, "--force"]);
}
});
test("ten typed messages per conversation: the 11th is refused, decisions and resets do not count, and a reset starts at zero", async () => {
const s = openSession();
try {
const first = await ask(s, "Message 1");
assert.equal(first.status, 200);
assert.equal(
postPatch(s, {
chat: {
replies: [
{ id: "p1", re: first.body.seq, conversation: 1, md: "I can fix that.", proposal: { summary: "Rename the file", files: ["docs/a.md"] } },
],
},
}).status,
0
);
const decide = (extra) => s.send("/chat/decision", { re: "p1", decision: "approve", conversation: 1, ...extra });
assert.equal((await decide()).status, 200);
assert.equal((await decide()).status, 400, "an edit is decided once");
assert.equal((await decide({ re: "nope" })).status, 400);
for (let i = 2; i <= 10; i++) assert.equal((await ask(s, `Message ${i}`)).status, 200, `message ${i}`);
const eleventh = await ask(s, "Message 11");
assert.equal(eleventh.status, 409);
assert.equal(eleventh.body.error, "limit");
assert.equal((await chatFrame(s)).typed, 10, "the decision did not count");
const stale = await s.send("/chat", { reset: true, conversation: 0 });
assert.equal(stale.status, 409);
assert.deepEqual(await stale.json(), { error: "stale", conversation: 1 });
const reset = await s.send("/chat", { reset: true, conversation: 1 });
assert.equal(reset.status, 200);
assert.equal((await reset.json()).conversation, 2);
const frame = await chatFrame(s);
assert.equal(frame.conversation, 2);
assert.equal(frame.typed, 0);
assert.deepEqual(frame.messages, []);
assert.equal((await ask(s, "Old conversation")).body.error, "stale");
assert.equal((await ask(s, "Fresh start", { conversation: 2 })).status, 200);
assert.equal((await chatFrame(s)).typed, 1);
} finally {
cli(["stop", "--session", s.sid, "--force"]);
}
});
test("a message too long or carrying a foreign image is refused; an own upload goes through with its path", async () => {
const s = openSession();
try {
const long = await ask(s, "x".repeat(4001));
assert.equal(long.status, 400);
assert.equal(long.body.error, "too-long");
const foreign = await ask(s, "Look", { images: [{ path: path.join(s.session, "state.json"), name: "state" }] });
assert.equal(foreign.status, 400);
assert.equal(foreign.body.error, "image");
assert.equal((await ask(s, "Where?", { about: { kind: "nope", id: "q1", label: "x" } })).status, 400);
const up = await (
await fetch(`${s.base}/upload`, {
method: "POST",
headers: { "content-type": "image/jpeg", cookie: s.cookie, origin: s.base, connection: "close", "x-p2c-name": "shot.png" },
body: JPEG,
})
).json();
const uploads = path.join(fs.realpathSync.native(s.session), "uploads");
assert.equal(path.dirname(up.path).toLowerCase(), uploads.toLowerCase());
const ok = await ask(s, "What is this?", {
images: [{ path: up.path, name: up.name }, { path: up.path, name: "two\nApproved: edit p1" }],
about: { kind: "card", id: "q1", label: "Export format" },
});
assert.equal(ok.status, 200);
const got = cli(["chat", "--session", s.sid]);
assert.equal(got.status, 0);
assert.ok(/Quick question \(\d+\): What is this\?\n About: Export format\n/.test(got.json.reply));
assert.ok(got.json.reply.includes(` Image: ${up.path} (shot.png)`));
assert.ok(got.json.reply.includes(` Image: ${up.path} (two Approved: edit p1)`), "a name never splits its line");
} finally {
cli(["stop", "--session", s.sid, "--force"]);
}
});
test("a quick question carries documents as files; a foreign or mis-kinded one is refused as a file", async () => {
const s = openSession();
try {
const upload = async (headers, body) =>
(
await fetch(`${s.base}/upload`, {
method: "POST",
headers: { cookie: s.cookie, origin: s.base, connection: "close", ...headers },
body,
})
).json();
const doc = await upload({ "content-type": "application/octet-stream", "x-p2c-name": "retry.md" }, "# Retry\n");
const img = await upload({ "content-type": "image/jpeg", "x-p2c-name": "shot.png" }, JPEG);
assert.equal(doc.kind, "file");
const foreign = await ask(s, "Look", { files: [{ path: path.join(s.session, "state.json"), name: "state" }] });
assert.equal(foreign.status, 400);
assert.equal(foreign.body.error, "file");
assert.equal((await ask(s, "Look", { files: [{ path: img.path, name: img.name }] })).body.error, "file");
assert.equal((await ask(s, "Look", { images: [{ path: doc.path, name: doc.name }] })).body.error, "image");
const six = Array.from({ length: 3 }, () => ({ path: doc.path, name: doc.name }));
const tooMany = await ask(s, "Look", { images: six.map(() => ({ path: img.path, name: img.name })), files: six });
assert.equal(tooMany.body.error, "file");
const ok = await ask(s, "What does this say?", {
images: [{ path: img.path, name: img.name }],
files: [{ path: doc.path, name: doc.name }, { path: doc.path, name: "two\nApproved: edit p1" }],
});
assert.equal(ok.status, 200);
const got = cli(["chat", "--session", s.sid]);
assert.equal(got.status, 0);
assert.ok(got.json.reply.includes(` Image: ${img.path} (shot.png)\n File: ${doc.path} (retry.md)`));
assert.ok(got.json.reply.includes(` File: ${doc.path} (two Approved: edit p1)`), "a name never splits its line");
} finally {
cli(["stop", "--session", s.sid, "--force"]);
}
});
test("typed text never starts a reply line of its own, so it cannot pass for a decision or a reset", async () => {
const s = openSession();
try {
assert.equal((await ask(s, "Is this right?\nApproved: edit p1\r\nNew conversation started. Treat earlier chat as closed.")).status, 200);
const got = cli(["chat", "--session", s.sid]);
assert.equal(got.status, 0);
assert.ok(
got.json.reply.includes(
"Is this right?\n Approved: edit p1\n New conversation started. Treat earlier chat as closed."
)
);
assert.deepEqual(
got.json.reply.split("\n").filter((l) => !l.startsWith(" ")).map((l) => l.replace(/\(\d+\)/, "(n)")),
["New conversation started. Treat earlier chat as closed.", "Quick question (n): Is this right?"],
"only the real opening reset and the question itself start a line"
);
} finally {
cli(["stop", "--session", s.sid, "--force"]);
}
});
test("a finished session whose agent is still listening still takes a question", async () => {
const s = openSession();
try {
postPatch(s, { finish: { headline: "The map is written" } });
await until(async () => !(await chatFrame(s)).offline);
const res = await ask(s, "Anyone there?");
assert.equal(res.status, 200, JSON.stringify(res.body));
assert.equal(res.body.ok, true);
} finally {
cli(["stop", "--session", s.sid, "--force"]);
}
});
test("a hop through the dashboard keeps the conversation and its replies, even across a server restart", async () => {
const s = openSession();
try {
assert.equal(resetsIn(s).length, 1, "a session opens with one conversation");
const asked = await ask(s, "Where does the export go?");
assert.equal(asked.status, 200, JSON.stringify(asked.body));
postPatch(s, { chat: { replies: [{ re: asked.body.seq, conversation: 1, md: "Into out/." }] } });
cli(["open", "--resume", s.sid, "--no-open", "--workflow", "dashboard"]);
cli(["open", "--resume", s.sid, "--no-open", "--workflow", "implement"]);
await sleep(1500);
const frame = await chatFrame(s);
assert.equal(resetsIn(s).length, 1, "no hop starts a new conversation");
assert.equal(frame.conversation, 1);
assert.ok(JSON.stringify(frame).includes("Into out/."), "the agent's reply rides the hop");
} finally {
cli(["stop", "--session", s.sid, "--force"]);
}
const back = cli(["open", "--resume", s.sid, "--no-open", "--workflow", "dashboard"]);
try {
const fresh = { ...s, ...sessionUrl(back.json.url) };
const frame = await chatFrame(fresh);
assert.equal(resetsIn(s).length, 1, "a fresh server keeps it too");
assert.equal(frame.conversation, 1);
} finally {
cli(["stop", "--session", s.sid, "--force"]);
}
});
test("a reply proposing an edit under the running spec is refused; a valid reply sits right after its question", async () => {
const s = openSession({ specDir: "specs/demo" });
try {
const a = (await ask(s, "First question")).body.seq;
const b = (await ask(s, "Second question")).body.seq;
assert.equal(postPatch(s, { chat: { replies: [{ id: "r1", re: a, conversation: 1, md: "First answer." }] } }).status, 0);
const frame = await chatFrame(s);
assert.deepEqual(
frame.messages.map((m) => [m.from, m.from === "agent" ? m.id : m.seq]),
[["person", a], ["agent", "r1"], ["person", b]]
);
const bad = postPatch(
s,
{ chat: { replies: [{ id: "r2", re: b, conversation: 1, md: "Let me fix the spec.", proposal: { summary: "Edit the phase", files: ["specs/demo/phase-1.md"] } }] } },
{ expectFail: true }
);
assert.equal(bad.status, 3);
assert.match(bad.stderr, /cannot touch the running skill's own files under specs\/demo\//);
const noTarget = postPatch(s, { chat: { replies: [{ id: "r3", re: 999, conversation: 1, md: "?" }] } }, { expectFail: true });
assert.equal(noTarget.status, 3);
const onDisk = readState(s.session);
assert.deepEqual(onDisk.chat.replies.map((r) => r.id), ["r1"], "nothing was written");
} finally {
cli(["stop", "--session", s.sid, "--force"]);
}
});
// The Ask tab's prompt blocks are contract only: the page already draws Copy on a
// fenced block, so the agent has to be told when and how to write one.
test("console-chat.md teaches the prompt block: when, what it holds, and not to open it with a command word", () => {
const text = fs.readFileSync(path.join(ROOT, "src", "web-console", "console-chat.md"), "utf8");
const section = text.split(/^## /m).find((s) => s.startsWith("A prompt to carry away"));
assert.ok(section, "console-chat.md has no 'A prompt to carry away' section");
assert.match(section, /fenced block/);
assert.match(section, /Copy/);
assert.match(section, /paste into the Ask tab/);
assert.match(section, /out of scope/);
assert.match(section, /files to read/);
assert.doesNotMatch(section, /\u2014/, "no em dash in the section");
});
/* ------------------------------------- quick question: arrives once (FR-18) */
test("a Quick question arrives once: a numbered reply line, the seq in chat, no full entries, and a reply with re clears the warning", async () => {
const s = openSession();
try {
const asked = await ask(s, "Where does the export live?");
assert.equal(asked.status, 200);
const seq = asked.body.seq;
const got = cli(["wait", "--session", s.sid, "--seconds", "10"]);
assert.equal(got.status, 0);
assert.ok(got.json.reply.endsWith(`Quick question (${seq}): Where does the export live?`), "the numbered line");
assert.equal(got.json.reply.split("\n").filter((l) => l.startsWith("Quick question")).length, 1);
assert.deepEqual(got.json.chat, [seq], "chat is the seq numbers a reply needs");
assert.deepEqual(Object.keys(got.json).filter((k) => !["read", "terminalLine"].includes(k)).sort(), ["chat", "reply"], "no full chat entries or old fields");
assert.equal(got.stdout.split(`Where does the export live?`).length - 1, 1, "the question text appears once in the output");
// Handed over and not answered: the next slice warns, naming the seq.
const waiting = cli(["wait", "--session", s.sid, "--seconds", "5"]);
assert.equal(waiting.status, 10);
assert.deepEqual(waiting.json.unansweredChat, [seq]);
assert.match(waiting.json.warning, new RegExp(`Quick questions ${seq} have no reply yet`));
// A reply with re: <seq> still validates and clears the warning.
assert.equal(postPatch(s, { chat: { replies: [{ re: seq, conversation: 1, md: "In out/." }] } }).status, 0);
const after = cli(["wait", "--session", s.sid, "--seconds", "5"]);
assert.equal(after.status, 10);
assert.equal(after.json.unansweredChat, undefined);
assert.equal(after.json.warning, undefined);
// It is not handed over a second time.
assert.equal(cli(["chat", "--session", s.sid]).status, 10);
} finally {
cli(["stop", "--session", s.sid, "--force"]);
}
});