Files
plan2code/scripts/test-web-console-chat.mjs
T

434 lines
20 KiB
JavaScript

// Web console tests, Quick questions: the wait contract, delivery, limits and resets.
// Run with: node --test scripts/test-web-console-chat.mjs
//
// Shared setup (a per-process console home, the CLI runner, fixtures) lives in
// web-console-test-helpers.mjs; see the comment at its top.
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
import { spawn } from "node:child_process";
import { readChat, terminalLine } from "../src/web-console/lib.mjs";
import {
ROOT,
CONSOLE_CLI,
ENV,
cli,
safeJson,
JPEG,
sessionUrl,
readState,
openSession,
sleep,
postPatch,
chatFrame,
ask,
resetsIn,
until,
EXPORT_A,
} from "./web-console-test-helpers.mjs";
/* ------------------------------------------- quick question: the contract */
// Every outcome `wait` had before the chat channel existed, pinned by exit
// code and by its exact top-level keys. The only change allowed later is the
// chat keys appearing beside them; anything else breaks an agent somewhere.
const CHAT_KEYS = new Set(["chat", "pendingChat"]);
// The workspace channel, allowed beside them the same way.
const WORKSPACE_KEYS = new Set(["workspace", "pendingWorkspace"]);
const pinKeys = (obj, expected, what) =>
assert.deepEqual(
Object.keys(obj).filter((k) => !CHAT_KEYS.has(k) && !WORKSPACE_KEYS.has(k)).sort(),
[...expected].sort(),
`${what}: top-level keys changed`
);
const RESULT_KEYS = ["type", "at", "sid", "rev", "actions", "reply", "consumedAt", "terminalLine"];
const WAITING_KEYS = ["status", "sid", "elapsed", "answered", "total", "staged", "url", "message", "next", "terminalLine"];
test("wait's existing outcomes are pinned: submit, images, waiting, cancel, carried and server gone", async () => {
const s = openSession();
try {
const waiting = cli(["wait", "--session", s.sid, "--seconds", "5"]);
assert.equal(waiting.status, 10);
assert.equal(waiting.json.status, "waiting");
pinKeys(waiting.json, WAITING_KEYS, "exit 10");
assert.equal((await s.send("/submit", { actions: [{ i: "q1", type: "answer", kind: "choice", k: "A" }], reply: "Export format: a" })).status, 200);
const sent = cli(["wait", "--session", s.sid, "--seconds", "10"]);
assert.equal(sent.status, 0);
assert.equal(sent.json.type, "submit");
pinKeys(sent.json, RESULT_KEYS, "exit 0 submit");
const up = await fetch(`${s.base}/upload`, {
method: "POST",
headers: { "content-type": "image/jpeg", cookie: s.cookie, origin: s.base, connection: "close", "x-p2c-name": "a.png" },
body: JPEG,
});
const img = await up.json();
assert.equal(
(await s.send("/submit", { actions: [{ i: "q1", type: "comment", text: "See this.", images: [{ path: img.path, name: img.name }] }], reply: "Note" })).status,
200
);
const withImages = cli(["wait", "--session", s.sid, "--seconds", "10"]);
assert.equal(withImages.status, 0);
assert.equal(withImages.json.actions[0].images.length, 1);
pinKeys(withImages.json, RESULT_KEYS, "exit 0 with images");
// Sent before any wait is running: the next wait hands it over at once.
assert.equal((await s.send("/submit", { actions: [{ i: "q1", type: "answer", kind: "choice", k: "B" }], reply: "Export format: b" })).status, 200);
const t0 = Date.now();
const carried = cli(["wait", "--session", s.sid, "--seconds", "30"]);
assert.equal(carried.status, 0);
assert.ok(Date.now() - t0 < 5000, "an uncollected result is delivered at once");
pinKeys(carried.json, RESULT_KEYS, "exit 0 carried");
assert.equal((await s.send("/cancel", {})).status, 200);
const cancelled = cli(["wait", "--session", s.sid, "--seconds", "5"]);
assert.equal(cancelled.status, 30);
assert.equal(cancelled.json.type, "cancel");
pinKeys(cancelled.json, RESULT_KEYS, "exit 30");
} finally {
cli(["stop", "--session", s.sid]);
}
const gone = cli(["wait", "--session", s.sid, "--seconds", "12"]);
assert.equal(gone.status, 20);
assert.equal(gone.json.status, "server-gone");
pinKeys(gone.json, ["status", "sid", "message"], "exit 20");
});
/* --------------------------------------------- quick question: the channel */
test("a chat message and a card send can both be pending, in either order, and wait hands both over at once", async () => {
const s = openSession();
try {
assert.equal((await s.send("/submit", EXPORT_A)).status, 200);
assert.equal((await ask(s, "Where does the export live?")).status, 200, "a pending card never blocks a question");
const both = cli(["wait", "--session", s.sid, "--seconds", "10"]);
assert.equal(both.status, 0);
assert.equal(both.json.type, "submit");
assert.equal(both.json.actions[0].k, "A");
assert.ok(both.json.chat.some((e) => e.kind === "message" && e.text === "Where does the export live?"));
assert.match(both.json.reply, /^Export format: a\n/);
assert.match(both.json.reply, /^Quick question: Where does the export live\?$/m);
assert.equal((await ask(s, "And the header row?")).status, 200);
assert.equal((await s.send("/submit", EXPORT_A)).status, 200, "a pending question never blocks a card");
const again = cli(["wait", "--session", s.sid, "--seconds", "10"]);
assert.equal(again.status, 0);
assert.equal(again.json.type, "submit");
assert.deepEqual(again.json.chat.map((e) => e.text), ["And the header row?"]);
assert.equal((await ask(s, "Just a question")).status, 200);
const chatOnly = cli(["wait", "--session", s.sid, "--seconds", "10"]);
assert.equal(chatOnly.status, 0);
assert.equal(chatOnly.json.type, "chat");
assert.deepEqual(chatOnly.json.actions, []);
assert.equal(chatOnly.json.reply, "Quick question: Just a question");
assert.ok(chatOnly.json.terminalLine);
} finally {
cli(["stop", "--session", s.sid]);
}
});
test("wait returns within 100 ms of a chat message landing", async () => {
const s = openSession();
try {
const child = spawn(process.execPath, [CONSOLE_CLI, "wait", "--session", s.sid, "--seconds", "20"], { env: ENV, cwd: ROOT });
let out = "";
const printed = new Promise((resolve) =>
child.stdout.on("data", (d) => {
out += d;
resolve(Date.now());
})
);
const exited = new Promise((resolve) => child.on("exit", resolve));
// wait checks the server's health as it starts and then every 5 s; land
// the message between the two, which takes longer to reach while other
// test files load the machine. The bound still has to be tighter than the
// 5 s poll cadence it guards against, but wide enough that a slow spawn
// (the message arriving before wait's first poll) does not flake it.
await sleep(2500);
assert.equal((await ask(s, "Quick one")).status, 200);
const answeredAt = Date.now();
assert.ok((await printed) - answeredAt < 2000, "wait must hand the chat over well inside its 5 s poll");
assert.equal(await exited, 0);
assert.equal(safeJson(out).type, "chat");
} finally {
cli(["stop", "--session", s.sid]);
}
});
test("the chat command answers at once: 0 with entries, 10 with none, 20 when the server is gone, and never takes a card send", async () => {
const s = openSession();
try {
assert.equal(cli(["chat", "--session", s.sid]).status, 10, "a reset alone is nothing to hand over");
assert.equal((await s.send("/submit", EXPORT_A)).status, 200);
assert.equal((await ask(s, "Mid-build question")).status, 200);
assert.equal((await (await s.send("/ping", {})).json()).pendingChat, 1, "an uncollected question shows on the heartbeat");
const got = cli(["chat", "--session", s.sid]);
assert.equal(got.status, 0);
assert.ok(got.json.chat.some((e) => e.text === "Mid-build question"));
assert.match(got.json.reply, /Quick question: Mid-build question/);
assert.ok(got.json.terminalLine);
const onDisk = JSON.parse(fs.readFileSync(path.join(s.session, "result.json"), "utf8"));
assert.equal(onDisk.consumedAt, undefined, "the card send is still waiting for wait");
assert.equal((await (await s.send("/ping", {})).json()).pendingChat, 0);
const none = cli(["chat", "--session", s.sid]);
assert.equal(none.status, 10);
assert.deepEqual(none.json, { chat: [] });
const card = cli(["wait", "--session", s.sid, "--seconds", "10"]);
assert.equal(card.status, 0);
assert.equal(card.json.type, "submit");
assert.deepEqual(card.json.chat, [], "the chat went with the chat command, not twice");
} finally {
cli(["stop", "--session", s.sid]);
}
const gone = cli(["chat", "--session", s.sid]);
assert.equal(gone.status, 20);
assert.equal(gone.json.status, "server-gone");
});
test("a torn trailing inbox line is skipped, then collected once it is whole", async () => {
const s = openSession();
try {
const file = path.join(s.session, "chat.ndjson");
const seq = Math.max(...readChat(s.session).map((e) => e.seq)) + 1;
const line = JSON.stringify({ kind: "message", conversation: 1, text: "torn in two", seq, at: new Date().toISOString() });
fs.appendFileSync(file, line.slice(0, 20));
assert.equal(readChat(s.session).some((e) => e.seq === seq), false, "a half-written line is not parsed");
assert.equal(cli(["chat", "--session", s.sid]).status, 10);
fs.appendFileSync(file, line.slice(20) + "\n");
assert.ok(readChat(s.session).some((e) => e.seq === seq));
const got = cli(["chat", "--session", s.sid]);
assert.equal(got.status, 0);
assert.ok(got.json.chat.some((e) => e.text === "torn in two"));
} finally {
cli(["stop", "--session", s.sid]);
}
});
test("a deleted cursor never causes a second answer to a question that has a reply", async () => {
const s = openSession();
try {
const { body } = await ask(s, "What is the port?");
assert.equal(cli(["chat", "--session", s.sid]).status, 0);
assert.equal(postPatch(s, { chat: { replies: [{ id: "r1", re: body.seq, conversation: 1, md: "Ephemeral." }] } }).status, 0);
fs.rmSync(path.join(s.session, "chat-cursor.json"));
const again = cli(["chat", "--session", s.sid]);
assert.equal(again.status, 10, "the answered question is not handed over again");
} finally {
cli(["stop", "--session", s.sid]);
}
});
test("ten typed messages per conversation: the 11th is refused, decisions and resets do not count, and a reset starts at zero", async () => {
const s = openSession();
try {
const first = await ask(s, "Message 1");
assert.equal(first.status, 200);
assert.equal(
postPatch(s, {
chat: {
replies: [
{ id: "p1", re: first.body.seq, conversation: 1, md: "I can fix that.", proposal: { summary: "Rename the file", files: ["docs/a.md"] } },
],
},
}).status,
0
);
const decide = (extra) => s.send("/chat/decision", { re: "p1", decision: "approve", conversation: 1, ...extra });
assert.equal((await decide()).status, 200);
assert.equal((await decide()).status, 400, "an edit is decided once");
assert.equal((await decide({ re: "nope" })).status, 400);
for (let i = 2; i <= 10; i++) assert.equal((await ask(s, `Message ${i}`)).status, 200, `message ${i}`);
const eleventh = await ask(s, "Message 11");
assert.equal(eleventh.status, 409);
assert.equal(eleventh.body.error, "limit");
assert.equal((await chatFrame(s)).typed, 10, "the decision did not count");
const stale = await s.send("/chat", { reset: true, conversation: 0 });
assert.equal(stale.status, 409);
assert.deepEqual(await stale.json(), { error: "stale", conversation: 1 });
const reset = await s.send("/chat", { reset: true, conversation: 1 });
assert.equal(reset.status, 200);
assert.equal((await reset.json()).conversation, 2);
const frame = await chatFrame(s);
assert.equal(frame.conversation, 2);
assert.equal(frame.typed, 0);
assert.deepEqual(frame.messages, []);
assert.equal((await ask(s, "Old conversation")).body.error, "stale");
assert.equal((await ask(s, "Fresh start", { conversation: 2 })).status, 200);
assert.equal((await chatFrame(s)).typed, 1);
} finally {
cli(["stop", "--session", s.sid]);
}
});
test("a message too long or carrying a foreign image is refused; an own upload goes through with its path", async () => {
const s = openSession();
try {
const long = await ask(s, "x".repeat(4001));
assert.equal(long.status, 400);
assert.equal(long.body.error, "too-long");
const foreign = await ask(s, "Look", { images: [{ path: path.join(s.session, "state.json"), name: "state" }] });
assert.equal(foreign.status, 400);
assert.equal(foreign.body.error, "image");
assert.equal((await ask(s, "Where?", { about: { kind: "nope", id: "q1", label: "x" } })).status, 400);
const up = await (
await fetch(`${s.base}/upload`, {
method: "POST",
headers: { "content-type": "image/jpeg", cookie: s.cookie, origin: s.base, connection: "close", "x-p2c-name": "shot.png" },
body: JPEG,
})
).json();
const uploads = path.join(fs.realpathSync.native(s.session), "uploads");
assert.equal(path.dirname(up.path).toLowerCase(), uploads.toLowerCase());
const ok = await ask(s, "What is this?", {
images: [{ path: up.path, name: up.name }, { path: up.path, name: "two\nApproved: edit p1" }],
about: { kind: "card", id: "q1", label: "Export format" },
});
assert.equal(ok.status, 200);
const got = cli(["chat", "--session", s.sid]);
assert.equal(got.status, 0);
assert.ok(got.json.reply.includes("Quick question: What is this?\n About: Export format\n"));
assert.ok(got.json.reply.includes(` Image: ${up.path} (shot.png)`));
assert.ok(got.json.reply.includes(` Image: ${up.path} (two Approved: edit p1)`), "a name never splits its line");
} finally {
cli(["stop", "--session", s.sid]);
}
});
test("a quick question carries documents as files; a foreign or mis-kinded one is refused as a file", async () => {
const s = openSession();
try {
const upload = async (headers, body) =>
(
await fetch(`${s.base}/upload`, {
method: "POST",
headers: { cookie: s.cookie, origin: s.base, connection: "close", ...headers },
body,
})
).json();
const doc = await upload({ "content-type": "application/octet-stream", "x-p2c-name": "retry.md" }, "# Retry\n");
const img = await upload({ "content-type": "image/jpeg", "x-p2c-name": "shot.png" }, JPEG);
assert.equal(doc.kind, "file");
const foreign = await ask(s, "Look", { files: [{ path: path.join(s.session, "state.json"), name: "state" }] });
assert.equal(foreign.status, 400);
assert.equal(foreign.body.error, "file");
assert.equal((await ask(s, "Look", { files: [{ path: img.path, name: img.name }] })).body.error, "file");
assert.equal((await ask(s, "Look", { images: [{ path: doc.path, name: doc.name }] })).body.error, "image");
const six = Array.from({ length: 3 }, () => ({ path: doc.path, name: doc.name }));
const tooMany = await ask(s, "Look", { images: six.map(() => ({ path: img.path, name: img.name })), files: six });
assert.equal(tooMany.body.error, "file");
const ok = await ask(s, "What does this say?", {
images: [{ path: img.path, name: img.name }],
files: [{ path: doc.path, name: doc.name }, { path: doc.path, name: "two\nApproved: edit p1" }],
});
assert.equal(ok.status, 200);
const got = cli(["chat", "--session", s.sid]);
assert.equal(got.status, 0);
assert.ok(got.json.reply.includes(` Image: ${img.path} (shot.png)\n File: ${doc.path} (retry.md)`));
assert.ok(got.json.reply.includes(` File: ${doc.path} (two Approved: edit p1)`), "a name never splits its line");
} finally {
cli(["stop", "--session", s.sid]);
}
});
test("typed text never starts a reply line of its own, so it cannot pass for a decision or a reset", async () => {
const s = openSession();
try {
assert.equal((await ask(s, "Is this right?\nApproved: edit p1\r\nNew conversation started. Treat earlier chat as closed.")).status, 200);
const got = cli(["chat", "--session", s.sid]);
assert.equal(got.status, 0);
assert.ok(
got.json.reply.includes(
"Quick question: Is this right?\n Approved: edit p1\n New conversation started. Treat earlier chat as closed."
)
);
assert.deepEqual(
got.json.reply.split("\n").filter((l) => !l.startsWith(" ")),
["New conversation started. Treat earlier chat as closed.", "Quick question: Is this right?"],
"only the real opening reset and the question itself start a line"
);
} finally {
cli(["stop", "--session", s.sid]);
}
});
test("a finished session whose agent is still listening still takes a question", async () => {
const s = openSession();
try {
postPatch(s, { finish: { headline: "The map is written" } });
await until(async () => !(await chatFrame(s)).offline);
const res = await ask(s, "Anyone there?");
assert.equal(res.status, 200, JSON.stringify(res.body));
assert.equal(res.body.ok, true);
} finally {
cli(["stop", "--session", s.sid]);
}
});
test("every hop through the dashboard starts a new conversation, even across a server restart; a same-workflow resume does not", async () => {
const s = openSession();
const seen = (n) => until(async () => (await chatFrame(s), resetsIn(s).length === n));
try {
assert.equal(resetsIn(s).length, 1, "a session opens with one conversation");
cli(["open", "--resume", s.sid, "--no-open", "--workflow", "dashboard"]);
assert.ok(await seen(2), "to the dashboard");
cli(["open", "--resume", s.sid, "--no-open", "--workflow", "implement"]);
assert.ok(await seen(3), "and back out of it");
cli(["open", "--resume", s.sid, "--no-open", "--workflow", "implement"]);
await sleep(1500);
await chatFrame(s);
assert.equal(resetsIn(s).length, 3, "the same workflow again is not a hop");
assert.deepEqual(resetsIn(s).map((e) => e.reason), ["start", "switch", "switch"]);
} finally {
cli(["stop", "--session", s.sid]);
}
const back = cli(["open", "--resume", s.sid, "--no-open", "--workflow", "dashboard"]);
try {
const fresh = { ...s, ...sessionUrl(back.json.url) };
assert.ok(await until(async () => (await chatFrame(fresh), resetsIn(s).length === 4)), "a fresh server still sees the hop");
assert.equal(resetsIn(s).at(-1).workflow, "dashboard");
assert.equal((await chatFrame(fresh)).conversation, 4);
} finally {
cli(["stop", "--session", s.sid]);
}
});
test("a reply proposing an edit under the running spec is refused; a valid reply sits right after its question", async () => {
const s = openSession({ specDir: "specs/demo" });
try {
const a = (await ask(s, "First question")).body.seq;
const b = (await ask(s, "Second question")).body.seq;
assert.equal(postPatch(s, { chat: { replies: [{ id: "r1", re: a, conversation: 1, md: "First answer." }] } }).status, 0);
const frame = await chatFrame(s);
assert.deepEqual(
frame.messages.map((m) => [m.from, m.from === "agent" ? m.id : m.seq]),
[["person", a], ["agent", "r1"], ["person", b]]
);
const bad = postPatch(
s,
{ chat: { replies: [{ id: "r2", re: b, conversation: 1, md: "Let me fix the spec.", proposal: { summary: "Edit the phase", files: ["specs/demo/phase-1.md"] } }] } },
{ expectFail: true }
);
assert.equal(bad.status, 3);
assert.match(bad.stderr, /cannot touch the running skill's own files under specs\/demo\//);
const noTarget = postPatch(s, { chat: { replies: [{ id: "r3", re: 999, conversation: 1, md: "?" }] } }, { expectFail: true });
assert.equal(noTarget.status, 3);
const onDisk = readState(s.session);
assert.deepEqual(onDisk.chat.replies.map((r) => r.id), ["r1"], "nothing was written");
} finally {
cli(["stop", "--session", s.sid]);
}
});