mirror of
https://github.com/jparkerweb/plan2code.git
synced 2026-11-16 04:45:33 -08:00
Add console wire measurement tooling and per-run byte budgets
AI Assisted
This commit is contained in:
1 parent
8e34bf1005
commit
9c07ee5260
9 files changed
+1527
No files matched your search
@@ -0,0 +1,279 @@
|
||||
#!/usr/bin/env node
|
||||
// Plays the person in a scripted console session, for the live runs (run.mjs).
|
||||
//
|
||||
// node scripts/console-wire-runs/answer.mjs --session <sid> --home <console home> --project <dir> --log <answers.ndjson>
|
||||
//
|
||||
// It follows the page's state from the session's state.json and answers through
|
||||
// the same HTTP routes the page uses, with the session's cookie. The script is
|
||||
// fixed: pick Quick Task on the dashboard, answer its cards, ask one Quick
|
||||
// question once a doc is up, press the finish's dashboard button, launch
|
||||
// Pathfinder, answer one question turn, press Stop, and exit 0 once that stop's
|
||||
// finish is up. Every send is logged with a timestamp.
|
||||
//
|
||||
// The idea box (`templates: "idea"`) gets the fixed text below rather than its
|
||||
// placeholder: an agent writes that placeholder, and the runs have to build the
|
||||
// same small thing to compare.
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const QUICK_IDEA = "Add an optional second argument to greet: when it is { shout: true }, return the greeting in upper case. Add a test for it.";
|
||||
const PATHFINDER_IDEA = "A command line wrapper for greet, so people can run it from a terminal and pipe names into it.";
|
||||
const FALLBACK_TEXT = "Keep it as small as you can.";
|
||||
const QUICK_QUESTION = "Which file will this change the most?";
|
||||
const MAX_SENDS = 40;
|
||||
const TICK_MS = 400;
|
||||
// A question posted while the agent still says it is working: the page would
|
||||
// show it as not the person's turn, so give the agent this long to hand over.
|
||||
const WORKING_GRACE_MS = 60_000;
|
||||
const CHAT_REPLY_WAIT_MS = 120_000;
|
||||
|
||||
function parseArgs(argv) {
|
||||
const opts = {};
|
||||
for (let i = 0; i < argv.length; i += 2) {
|
||||
if (!argv[i].startsWith("--")) throw new Error(`unexpected ${argv[i]}`);
|
||||
opts[argv[i].slice(2)] = argv[i + 1];
|
||||
}
|
||||
for (const key of ["session", "home", "project", "log"]) {
|
||||
if (!opts[key]) throw new Error(`--${key} is required`);
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
const opts = parseArgs(process.argv.slice(2));
|
||||
// Before the imports below: the console reads its home once, as it loads.
|
||||
process.env.PLAN2CODE_CONSOLE_HOME = opts.home;
|
||||
const consoleDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "src", "web-console");
|
||||
const load = (file) => import(new URL(`file:///${path.join(consoleDir, file).replace(/\\/g, "/")}`).href);
|
||||
const { listHandles, repoRoots } = await load("sessions.mjs");
|
||||
const { isOpen } = await load("patches.mjs");
|
||||
const { currentConversation, readChat } = await load("inbox.mjs");
|
||||
const { launchReply, DASHBOARD_REPLY, SKILL_CATALOG, STOP_REPLY, stopAction } = await load("public/answers.js");
|
||||
|
||||
const sessionDir = path.join(opts.home, "sessions", opts.session);
|
||||
const project = repoRoots(opts.project).project;
|
||||
let sends = 0;
|
||||
|
||||
function readState() {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(path.join(sessionDir, "state.json"), "utf8"));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function resultPending() {
|
||||
try {
|
||||
return !JSON.parse(fs.readFileSync(path.join(sessionDir, "result.json"), "utf8")).consumedAt;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function logSend(entry) {
|
||||
fs.appendFileSync(opts.log, JSON.stringify({ at: new Date().toISOString(), ...entry }) + "\n");
|
||||
}
|
||||
|
||||
async function request(route, body) {
|
||||
const handle = listHandles(project).find((h) => h.sid === opts.session);
|
||||
if (!handle || !handle.url) return { status: 0, body: {} };
|
||||
const url = new URL(handle.url);
|
||||
const token = url.pathname.split("/")[2];
|
||||
const res = await fetch(`${url.origin}${route}`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", cookie: `p2c_console_${url.port}=${token}`, origin: url.origin, connection: "close" },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
return { status: res.status, body: await res.json().catch(() => ({})) };
|
||||
}
|
||||
|
||||
// One send, as the page's Send button makes it. True once the server took it.
|
||||
async function send(kind, actions, reply) {
|
||||
if (++sends > MAX_SENDS) throw new Error(`more than ${MAX_SENDS} sends: the script is stuck`);
|
||||
const res = await request("/submit", { actions, reply });
|
||||
if (res.status === 409) {
|
||||
sends--;
|
||||
return false;
|
||||
}
|
||||
if (res.status !== 200) throw new Error(`/submit said ${res.status}: ${JSON.stringify(res.body)}`);
|
||||
logSend({ kind, actions, reply });
|
||||
return true;
|
||||
}
|
||||
|
||||
function catalogEntry(workflow) {
|
||||
const entry = SKILL_CATALOG.find((s) => s.workflow === workflow);
|
||||
if (!entry) throw new Error(`no ${workflow} card in the catalog`);
|
||||
return entry;
|
||||
}
|
||||
|
||||
function launch(workflow) {
|
||||
const entry = catalogEntry(workflow);
|
||||
answered.clear();
|
||||
const action = { i: "__launch", type: "launch", skill: entry.skill, workflow: entry.workflow, title: entry.title };
|
||||
return send(`launch ${workflow}`, [action], launchReply(entry, null));
|
||||
}
|
||||
|
||||
function textAnswer(item, idea) {
|
||||
if (item.templates === "idea") return idea;
|
||||
const hint = String(item.placeholder || "").replace(/^\s*(for example|e\.g\.|example)[:,]?\s*/i, "").trim();
|
||||
const shaped = (value) => !item.pattern || new RegExp(item.pattern).test(value);
|
||||
if (hint && shaped(hint)) return hint;
|
||||
return shaped(FALLBACK_TEXT) ? FALLBACK_TEXT : "wire-fixture-change";
|
||||
}
|
||||
|
||||
// The answer a person following the fixed script gives one card, as the page
|
||||
// would stage it, with the word the terminal flow would have received.
|
||||
function answerFor(item, idea) {
|
||||
const options = Array.isArray(item.options) ? item.options : [];
|
||||
const pickOption = () => options.find((o) => o.recommended) || options[0];
|
||||
const tokenFor = (key) => (item.token && item.token[key]) || key;
|
||||
switch (item.kind) {
|
||||
case "choice":
|
||||
case "menu": {
|
||||
const option = pickOption();
|
||||
if (option) return { value: { k: option.k }, word: tokenFor(option.k) };
|
||||
return { value: { text: FALLBACK_TEXT }, word: FALLBACK_TEXT };
|
||||
}
|
||||
case "multi": {
|
||||
const option = pickOption();
|
||||
return { value: { ks: option ? [option.k] : [] }, word: option ? tokenFor(option.k) : "skip" };
|
||||
}
|
||||
case "text": {
|
||||
const text = textAnswer(item, idea);
|
||||
return { value: { text }, word: text };
|
||||
}
|
||||
case "confirm":
|
||||
return { value: { yes: true }, word: tokenFor("yes") };
|
||||
case "recap":
|
||||
return { value: { ok: true }, word: tokenFor("ok") };
|
||||
case "review": {
|
||||
const verdicts = Array.isArray(item.verdicts) && item.verdicts.length ? item.verdicts : [{ id: "approve" }];
|
||||
const verdict = verdicts.find((v) => !v.danger) || verdicts[0];
|
||||
return { value: { verdict: verdict.id }, word: tokenFor(verdict.id) };
|
||||
}
|
||||
case "checklist": {
|
||||
const steps = Array.isArray(item.steps) ? item.steps : [];
|
||||
return { value: { done: steps.map((_, n) => n), state: "done" }, word: "done" };
|
||||
}
|
||||
case "list":
|
||||
return { value: { text: "Looks right." }, word: "Looks right." };
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// A card is due once, and again only if the agent reopens it with something new.
|
||||
// Ids are only unique within a skill: a launch clears the page, and the next
|
||||
// skill may use `q1` again, so launch() forgets them.
|
||||
const answered = new Map();
|
||||
const cardKey = (item) => `${item.status}:${Array.isArray(item.thread) ? item.thread.length : 0}`;
|
||||
|
||||
function dueCards(state) {
|
||||
return (state.items || []).filter((item) => isOpen(item) && item.kind !== "notice" && answered.get(item.id) !== cardKey(item));
|
||||
}
|
||||
|
||||
let openSince = 0;
|
||||
function theirTurn(state, cards) {
|
||||
if (!cards.length) {
|
||||
openSince = 0;
|
||||
return false;
|
||||
}
|
||||
if (!openSince) openSince = Date.now();
|
||||
return !state.agent || state.agent.status !== "working" || Date.now() - openSince > WORKING_GRACE_MS;
|
||||
}
|
||||
|
||||
async function answerCards(cards, idea) {
|
||||
const actions = [];
|
||||
const lines = [];
|
||||
for (const item of cards) {
|
||||
const answer = answerFor(item, idea);
|
||||
if (!answer) continue;
|
||||
actions.push({ i: item.id, type: "answer", kind: item.kind, ...answer.value });
|
||||
lines.push(`${item.title}: ${answer.word}`);
|
||||
}
|
||||
if (!actions.length) return false;
|
||||
const taken = await send("answer", actions, lines.join("\n"));
|
||||
if (taken) {
|
||||
for (const item of cards) answered.set(item.id, cardKey(item));
|
||||
openSince = 0;
|
||||
}
|
||||
return taken;
|
||||
}
|
||||
|
||||
let asked = null;
|
||||
async function askQuickQuestion() {
|
||||
const conversation = currentConversation(readChat(sessionDir)).conversation;
|
||||
const res = await request("/chat", { text: QUICK_QUESTION, conversation });
|
||||
if (res.status !== 200) return;
|
||||
asked = { seq: res.body.seq, at: Date.now() };
|
||||
logSend({ kind: "chat", seq: res.body.seq, text: QUICK_QUESTION });
|
||||
}
|
||||
|
||||
const chatStillOwed = (state) =>
|
||||
asked && !(state.chat?.replies || []).some((r) => r && r.re === asked.seq) && Date.now() - asked.at < CHAT_REPLY_WAIT_MS;
|
||||
|
||||
async function step(stage, state) {
|
||||
const workflow = state.workflow;
|
||||
const waiting = state.agent && state.agent.status === "waiting";
|
||||
const cards = dueCards(state);
|
||||
|
||||
switch (stage) {
|
||||
case "launch-quick":
|
||||
if (workflow === "dashboard" && waiting && (await launch("quick-task"))) return "quick";
|
||||
return stage;
|
||||
|
||||
case "quick":
|
||||
if (workflow !== "quick-task") return stage;
|
||||
if (state.finish) {
|
||||
const taken = await send("dashboard", [{ i: "__dashboard", type: "dashboard" }], DASHBOARD_REPLY);
|
||||
return taken ? "launch-pathfinder" : stage;
|
||||
}
|
||||
if (!asked && (state.docs || []).length && theirTurn(state, cards)) {
|
||||
await askQuickQuestion();
|
||||
return stage;
|
||||
}
|
||||
if (chatStillOwed(state)) return stage;
|
||||
if (theirTurn(state, cards)) await answerCards(cards, QUICK_IDEA);
|
||||
return stage;
|
||||
|
||||
case "launch-pathfinder":
|
||||
if (workflow === "dashboard" && !state.finish && waiting && (await launch("pathfinder"))) return "pathfinder";
|
||||
return stage;
|
||||
|
||||
case "pathfinder":
|
||||
if (workflow !== "pathfinder") return stage;
|
||||
if (theirTurn(state, cards) && (await answerCards(cards, PATHFINDER_IDEA))) return "pathfinder-after-turn";
|
||||
return stage;
|
||||
|
||||
case "pathfinder-after-turn":
|
||||
if (theirTurn(state, cards) || (waiting && !cards.length)) {
|
||||
const taken = await send("stop", [stopAction()], STOP_REPLY);
|
||||
return taken ? "stopping" : stage;
|
||||
}
|
||||
return stage;
|
||||
|
||||
case "stopping":
|
||||
return state.finish ? "done" : stage;
|
||||
|
||||
default:
|
||||
throw new Error(`unknown stage ${stage}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
let stage = "launch-quick";
|
||||
while (stage !== "done") {
|
||||
await new Promise((resolve) => setTimeout(resolve, TICK_MS));
|
||||
const state = readState();
|
||||
if (!state || resultPending()) continue;
|
||||
const next = await step(stage, state);
|
||||
if (next !== stage) logSend({ kind: "stage", stage: next });
|
||||
stage = next;
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
process.stderr.write(`answer: ${e.message}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1 @@
|
||||
specs/
|
||||
@@ -0,0 +1,3 @@
|
||||
# AGENTS.md
|
||||
|
||||
A tiny Node project with no dependencies: `src/greet.js` exports `greet(name)`, and `test/greet.test.mjs` tests it with `node:test` (run `npm test`). Use ESM, keep functions small, and keep every change covered by a test.
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"name": "wire-fixture",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "node --test"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export function greet(name) {
|
||||
return `Hello, ${name}!`;
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { greet } from "../src/greet.js";
|
||||
|
||||
test("greet says hello by name", () => {
|
||||
assert.equal(greet("Sam"), "Hello, Sam!");
|
||||
});
|
||||
@@ -0,0 +1,258 @@
|
||||
#!/usr/bin/env node
|
||||
// The live-run driver for the console wire measure. Spends model tokens, so it
|
||||
// is run by hand and never from `npm test`.
|
||||
//
|
||||
// node scripts/console-wire-runs/run.mjs [--models opus,haiku] [--runs 3] [--timeout 20] [--out <dir>]
|
||||
//
|
||||
// Each run copies the fixture to a fresh temp folder, starts headless Claude
|
||||
// Code on `/plan2code --web` with the skills built under skills/, and starts
|
||||
// answer.mjs against the session it opens. Run it from outside the desktop app,
|
||||
// or let this script clear the app's variables, as it does for every run.
|
||||
//
|
||||
// `--setting-sources project` keeps the machine's own skills, hooks and memory
|
||||
// out: a personal skill of the same name would otherwise win over the built one.
|
||||
//
|
||||
// Each run has `--permission-mode bypassPermissions`: a model-driven agent with
|
||||
// unrestricted tools on this machine, working in a throwaway copy of the fixture.
|
||||
//
|
||||
// A run ends when the finish after the Pathfinder stop is up, or at --timeout.
|
||||
// An agent that ends its turn on text with the session open stops the process:
|
||||
// that is a silent slip the measure counts, so the run goes on the way a person
|
||||
// would, by telling it to carry on.
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(HERE, "..", "..");
|
||||
const FIXTURE = path.join(HERE, "fixture");
|
||||
const CONSOLE_CLI = path.join(ROOT, "skills", "plan2code", "references", "web-console", "console.mjs");
|
||||
const PROMPT = "/plan2code --web";
|
||||
const NUDGE = "The console page is still open and waiting on you. Carry on with the session.";
|
||||
const MAX_NUDGES = 5;
|
||||
const AFTER_FINISH_MS = 30_000;
|
||||
|
||||
function parseArgs(argv) {
|
||||
const opts = { models: "opus,haiku", runs: "3", timeout: "20", out: "" };
|
||||
for (let i = 0; i < argv.length; i += 2) {
|
||||
const key = argv[i].replace(/^--/, "");
|
||||
if (!Object.hasOwn(opts, key)) throw new Error(`unknown option ${argv[i]}`);
|
||||
opts[key] = argv[i + 1];
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
// What would tie a run to the session it started from. Other CLAUDE_CODE_*
|
||||
// variables stay: some carry the account or the provider (an OAuth token, Bedrock).
|
||||
const SESSION_VARS = [
|
||||
"CLAUDECODE",
|
||||
"CLAUDE_PID",
|
||||
"CLAUDE_EFFORT",
|
||||
"CLAUDE_CODE_CHILD_SESSION",
|
||||
"CLAUDE_CODE_ENTRYPOINT",
|
||||
"CLAUDE_CODE_EXECPATH",
|
||||
"CLAUDE_CODE_MESSAGING_SOCKET",
|
||||
"CLAUDE_CODE_MESSAGING_TOKEN",
|
||||
"CLAUDE_CODE_SESSION_ATTENDED",
|
||||
"CLAUDE_CODE_SESSION_ID",
|
||||
"CLAUDE_CODE_SSE_PORT",
|
||||
];
|
||||
|
||||
// Everything that would tie a run to the session or the app it started from.
|
||||
function runEnv(home) {
|
||||
const env = { ...process.env };
|
||||
for (const key of Object.keys(env)) {
|
||||
if (SESSION_VARS.includes(key) || key.startsWith("PLAN2CODE_")) delete env[key];
|
||||
}
|
||||
return {
|
||||
...env,
|
||||
PLAN2CODE_CONSOLE_HOME: home,
|
||||
PLAN2CODE_HANDOFFS_DIR: path.join(home, "handoffs"),
|
||||
PLAN2CODE_NOTEPADS_DIR: path.join(home, "notepads"),
|
||||
PLAN2CODE_NO_BROWSER: "1",
|
||||
};
|
||||
}
|
||||
|
||||
function realTemp(prefix) {
|
||||
return fs.realpathSync.native(fs.mkdtempSync(path.join(os.tmpdir(), prefix)));
|
||||
}
|
||||
|
||||
function makeProject() {
|
||||
const project = realTemp("p2c-wire-project-");
|
||||
fs.cpSync(FIXTURE, project, { recursive: true });
|
||||
fs.cpSync(path.join(ROOT, "skills"), path.join(project, ".claude", "skills"), { recursive: true });
|
||||
const init = spawnSync("git", ["init", "-q"], { cwd: project, encoding: "utf8" });
|
||||
if (init.status !== 0) throw new Error(`git init failed: ${init.stderr}`);
|
||||
return project;
|
||||
}
|
||||
|
||||
function killTree(child) {
|
||||
if (!child || child.exitCode !== null || child.signalCode !== null) return;
|
||||
if (process.platform === "win32") spawnSync("taskkill", ["/pid", String(child.pid), "/T", "/F"], { stdio: "ignore" });
|
||||
else child.kill("SIGKILL");
|
||||
}
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
function startClaude({ model, prompt, resume, project, env, streamFile }) {
|
||||
const args = ["-p", prompt, "--model", model, "--output-format", "stream-json", "--verbose", "--permission-mode", "bypassPermissions", "--setting-sources", "project"];
|
||||
if (resume) args.push("--resume", resume);
|
||||
const out = fs.openSync(streamFile, "a");
|
||||
const child = spawn("claude", args, { cwd: project, env, stdio: ["ignore", out, out], windowsHide: true });
|
||||
child.once("close", () => fs.closeSync(out));
|
||||
return child;
|
||||
}
|
||||
|
||||
function exited(child) {
|
||||
return new Promise((resolve) => {
|
||||
if (child.exitCode !== null || child.signalCode !== null) resolve();
|
||||
else child.once("close", () => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
function streamSessionIds(streamFile) {
|
||||
const ids = [];
|
||||
for (const line of fs.readFileSync(streamFile, "utf8").split(/\r?\n/)) {
|
||||
if (!line.includes('"session_id"')) continue;
|
||||
try {
|
||||
const id = JSON.parse(line).session_id;
|
||||
if (id && !ids.includes(id)) ids.push(id);
|
||||
} catch {}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
function transcriptPaths(ids) {
|
||||
const root = path.join(os.homedir(), ".claude", "projects");
|
||||
const found = [];
|
||||
for (const dir of fs.readdirSync(root)) {
|
||||
for (const id of ids) {
|
||||
const file = path.join(root, dir, `${id}.jsonl`);
|
||||
if (fs.existsSync(file)) found.push(file);
|
||||
}
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
function firstSession(home) {
|
||||
try {
|
||||
return fs.readdirSync(path.join(home, "sessions"))[0] || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function sessionState(home, sid) {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(path.join(home, "sessions", sid, "state.json"), "utf8"));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function oneRun({ model, n, minutes, outDir }) {
|
||||
const runDir = path.join(outDir, `${model.replace(/[^a-zA-Z0-9.-]/g, "-")}-${n}`);
|
||||
fs.mkdirSync(runDir, { recursive: true });
|
||||
const streamFile = path.join(runDir, "stream.ndjson");
|
||||
const answersFile = path.join(runDir, "answers.ndjson");
|
||||
const home = realTemp("p2c-wire-home-");
|
||||
const project = makeProject();
|
||||
const env = runEnv(home);
|
||||
const deadline = Date.now() + minutes * 60_000;
|
||||
const started = new Date().toISOString();
|
||||
|
||||
let claude = startClaude({ model, prompt: PROMPT, project, env, streamFile });
|
||||
let claudeDone = false;
|
||||
let answer = null;
|
||||
let answerDone = false;
|
||||
let answerCode = null;
|
||||
let sid = null;
|
||||
let nudges = 0;
|
||||
let failure = null;
|
||||
claude.once("close", () => (claudeDone = true));
|
||||
|
||||
while (Date.now() < deadline && !answerDone) {
|
||||
await sleep(500);
|
||||
if (!sid) {
|
||||
sid = firstSession(home);
|
||||
if (sid) {
|
||||
answer = spawn(process.execPath, [path.join(HERE, "answer.mjs"), "--session", sid, "--home", home, "--project", project, "--log", answersFile], {
|
||||
cwd: project,
|
||||
env,
|
||||
stdio: ["ignore", "ignore", fs.openSync(path.join(runDir, "answer.err"), "a")],
|
||||
windowsHide: true,
|
||||
});
|
||||
answer.once("close", (code) => {
|
||||
answerDone = true;
|
||||
answerCode = code;
|
||||
});
|
||||
}
|
||||
}
|
||||
if (claudeDone && !answerDone) {
|
||||
if (!sid || nudges >= MAX_NUDGES) {
|
||||
failure = sid ? "the agent kept ending its turn" : "the agent ended without opening a session";
|
||||
break;
|
||||
}
|
||||
const ids = streamSessionIds(streamFile);
|
||||
nudges++;
|
||||
claudeDone = false;
|
||||
claude = startClaude({ model, prompt: NUDGE, resume: ids[ids.length - 1], project, env, streamFile });
|
||||
claude.once("close", () => (claudeDone = true));
|
||||
}
|
||||
}
|
||||
|
||||
const finished = answerDone && answerCode === 0;
|
||||
if (!finished && !failure) failure = answerDone ? `answer.mjs exited ${answerCode}` : "timed out";
|
||||
// The finish is up: leave the agent its `stop` before ending the process.
|
||||
if (finished) await Promise.race([exited(claude), sleep(AFTER_FINISH_MS)]);
|
||||
killTree(claude);
|
||||
killTree(answer);
|
||||
if (sid) spawnSync(process.execPath, [CONSOLE_CLI, "stop", "--session", sid], { cwd: project, env, stdio: "ignore" });
|
||||
|
||||
const state = sid ? sessionState(home, sid) : null;
|
||||
const record = {
|
||||
model,
|
||||
run: n,
|
||||
finished,
|
||||
failure,
|
||||
nudges,
|
||||
started,
|
||||
ended: new Date().toISOString(),
|
||||
sid,
|
||||
finalWorkflow: state ? state.workflow : null,
|
||||
transcripts: transcriptPaths(streamSessionIds(streamFile)),
|
||||
project,
|
||||
home,
|
||||
};
|
||||
fs.writeFileSync(path.join(runDir, "run.json"), JSON.stringify(record, null, 2) + "\n");
|
||||
return record;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const opts = parseArgs(process.argv.slice(2));
|
||||
const outDir = opts.out ? path.resolve(opts.out) : realTemp("p2c-wire-runs-");
|
||||
const fromRoot = path.relative(ROOT, outDir);
|
||||
if (!fromRoot || (!fromRoot.startsWith("..") && !path.isAbsolute(fromRoot))) throw new Error("--out must be outside the repository");
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
if (!fs.existsSync(CONSOLE_CLI)) throw new Error("skills/ is not built: run node install.js --build-skills first");
|
||||
const runs = Number(opts.runs);
|
||||
const results = [];
|
||||
for (const model of opts.models.split(",")) {
|
||||
for (let n = 1; n <= runs; n++) {
|
||||
const record = await oneRun({ model, n, minutes: Number(opts.timeout), outDir });
|
||||
results.push(record);
|
||||
process.stdout.write(`${model} run ${n}: ${record.finished ? "finished" : `FAILED (${record.failure})`}, ${record.nudges} nudges\n`);
|
||||
}
|
||||
}
|
||||
const finished = results.filter((r) => r.finished).length;
|
||||
console.log(JSON.stringify({ out: outDir, runs: results.length, finished }));
|
||||
if (finished !== results.length) process.exitCode = 1;
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
process.stderr.write(`run: ${e.message}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,847 @@
|
||||
#!/usr/bin/env node
|
||||
// Count what the web console costs an agent, from Claude Code transcripts.
|
||||
//
|
||||
// node scripts/measure-console-wire.mjs [folder] [--since D] [--until D] [--exclude ID]... [--out rows.ndjson]
|
||||
// node scripts/measure-console-wire.mjs --baseline [--reencode]
|
||||
// node scripts/measure-console-wire.mjs --runs <dir> [--reencode]
|
||||
//
|
||||
// The original research scripts (`research-01`, in a Pathfinder session's
|
||||
// scratchpad) were outside this repository; this is their rebuild, following
|
||||
// the method written down in
|
||||
// specs/console-wire-format/pathfinder/questions/01-baseline-cost-today.md.
|
||||
//
|
||||
// Three buckets per session, each counted with `o200k_base` as a proxy (no
|
||||
// Claude tokenizer runs offline; the same counter before and after is what
|
||||
// matters):
|
||||
//
|
||||
// patches heredoc bodies of `post` and `open`, plus a `--file <path>` patch
|
||||
// when the same transcript wrote that path earlier (`--baseline`
|
||||
// leaves those out, since the recorded figures do)
|
||||
// outputs the CLI's own JSON lines in `open`, `post`, `wait`, `chat` results
|
||||
// docs standalone reads of the contract files, plus the non-JSON text of
|
||||
// a console result whose command also cats one of them
|
||||
//
|
||||
// A resumed or forked transcript repeats earlier entries, and the agent read
|
||||
// those results only once, so each call counts once. The recorded baseline
|
||||
// counted a repeated doc read every time it appeared (about 840k of its 3.98M
|
||||
// doc tokens) and missed a read relative to a `cd` into the console folder, so
|
||||
// `--baseline` does both, to compare like for like, and reports the repeats
|
||||
// beside it as `repeatedDocTokens`. Every other mode counts each read once and
|
||||
// counts the `cd` reads: those are the figures to trust.
|
||||
//
|
||||
// Known limits, the same in the before and the after run: a read the harness
|
||||
// persisted to a file (over about 30 KB) counts only the preview it left in the
|
||||
// transcript, and a doc read beside another file stays out of `docs`, as in the
|
||||
// baseline. A live run reports those beside `docs` as `mixedDocTokens` (the whole
|
||||
// file each names), so batching reads can never pass for reading less: compare
|
||||
// `withMixed` as well as `total`.
|
||||
//
|
||||
// The rewrites (`reencode*`) are pure and exported for Phases 6 and 7. The
|
||||
// tokenizer is loaded only when counting, so importing them costs nothing.
|
||||
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import readline from "node:readline";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
/* ------------------------------------------------------------ the baseline */
|
||||
|
||||
export const RECORDED_BASELINE = {
|
||||
since: "2026-09-29",
|
||||
until: "2026-10-29",
|
||||
exclude: ["01bce17b"],
|
||||
sessions: 328,
|
||||
median: { patches: 2089, outputs: 3707, docs: 10326, total: 18726 },
|
||||
p90: { total: 39870 },
|
||||
};
|
||||
const BASELINE_TOLERANCE = 0.05;
|
||||
|
||||
/* --------------------------------------------------------- reading commands */
|
||||
|
||||
const CONSOLE_VERB = /console\.mjs["']?\s+(open|post|wait|chat|stop|help)\b/g;
|
||||
const COUNTED_VERBS = new Set(["open", "post", "wait", "chat"]);
|
||||
const SID = /\b(\d{8}-\d{6}-[0-9a-f]{4,})\b/;
|
||||
const DOC_FILE = /(?:^|\/)(console\.md|console-[^/]+\.md|building[^/]*\.md|notes\/[^/]+\.md)$/;
|
||||
|
||||
// `NAME=value` assignments in the same command, so `node "$C" post` and
|
||||
// `cat "$D/console.md"` read as what they run. Nothing else of a shell.
|
||||
export function expandVars(command) {
|
||||
const vars = {};
|
||||
for (const m of command.matchAll(/(?:^|[\s;&])([A-Za-z_][A-Za-z0-9_]*)=("[^"\n]*"|'[^'\n]*'|[^\s;&]+)/g)) {
|
||||
vars[m[1]] = m[2].replace(/^["']|["']$/g, "");
|
||||
}
|
||||
let out = command;
|
||||
for (let pass = 0; pass < 3; pass++) {
|
||||
out = out.replace(/\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?/g, (whole, name) => (Object.hasOwn(vars, name) ? vars[name] : whole));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function consoleVerbs(command) {
|
||||
return [...command.matchAll(CONSOLE_VERB)].map((m) => m[1]);
|
||||
}
|
||||
|
||||
// Every heredoc in a command: the line that opens it, and its body.
|
||||
export function heredocs(command) {
|
||||
const lines = command.split("\n");
|
||||
const found = [];
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const m = /<<-?\s*(['"]?)([A-Za-z_][A-Za-z0-9_]*)\1/.exec(lines[i]);
|
||||
if (!m) continue;
|
||||
const end = lines.findIndex((l, n) => n > i && l.trim() === m[2]);
|
||||
if (end < 0) continue;
|
||||
found.push({ head: lines[i], body: lines.slice(i + 1, end).join("\n") });
|
||||
i = end;
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
// A command with its heredoc bodies taken out, so a `;` or a `cat` inside a
|
||||
// patch never reads as shell.
|
||||
function withoutHeredocs(command) {
|
||||
const lines = command.split("\n");
|
||||
const kept = [];
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
kept.push(lines[i]);
|
||||
const m = /<<-?\s*(['"]?)([A-Za-z_][A-Za-z0-9_]*)\1/.exec(lines[i]);
|
||||
if (!m) continue;
|
||||
const end = lines.findIndex((l, n) => n > i && l.trim() === m[2]);
|
||||
if (end >= 0) i = end;
|
||||
}
|
||||
return kept.join("\n");
|
||||
}
|
||||
|
||||
function normalizePath(p) {
|
||||
return String(p)
|
||||
.replace(/^["']|["']$/g, "")
|
||||
.replace(/\\/g, "/")
|
||||
.replace(/^\/([a-zA-Z])\//, "$1:/")
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
// One shell segment into words, honoring quotes.
|
||||
function shellWords(segment) {
|
||||
const words = [];
|
||||
for (const m of segment.matchAll(/"((?:[^"\\]|\\.)*)"|'([^']*)'|(\S+)/g)) {
|
||||
words.push(m[1] ?? m[2] ?? m[3]);
|
||||
}
|
||||
return words;
|
||||
}
|
||||
|
||||
function shellSegments(command) {
|
||||
const segments = [];
|
||||
let current = "";
|
||||
let quote = null;
|
||||
for (let i = 0; i < command.length; i++) {
|
||||
const c = command[i];
|
||||
if (quote) {
|
||||
if (c === quote) quote = null;
|
||||
current += c;
|
||||
} else if (c === '"' || c === "'") {
|
||||
quote = c;
|
||||
current += c;
|
||||
} else if (c === "\n" || c === ";" || (c === "&" && command[i + 1] === "&") || (c === "|" && command[i + 1] === "|")) {
|
||||
if (c !== "\n" && c !== ";") i++;
|
||||
segments.push(current.trim());
|
||||
current = "";
|
||||
} else {
|
||||
current += c;
|
||||
}
|
||||
}
|
||||
segments.push(current.trim());
|
||||
return segments.filter(Boolean);
|
||||
}
|
||||
|
||||
// The contract file a path names, or null. A development checkout's copy under
|
||||
// src/web-console/ is not the installed skill copy, and is skipped.
|
||||
export function docNameOf(p, { insideConsoleDir = false } = {}) {
|
||||
const n = String(p).replace(/\\/g, "/");
|
||||
if (/(?:^|\/)src\/web-console\//.test(n)) return null;
|
||||
const m = DOC_FILE.exec(n);
|
||||
if (!m) return null;
|
||||
return insideConsoleDir || /(?:^|\/)web-console\//.test(n) ? m[1] : null;
|
||||
}
|
||||
|
||||
// What a command reads: `docs` are the contract files it cats, and `only` says
|
||||
// every `cat` and `sed -n` in it names contract files alone. Other verbs beside
|
||||
// them (`ls`, `node`) do not change that, as in the baseline: only another file
|
||||
// read beside a contract file makes a command mixed.
|
||||
export function docReads(command, { cdReads = true } = {}) {
|
||||
const docs = [];
|
||||
let only = true;
|
||||
let insideConsoleDir = false;
|
||||
for (const segment of shellSegments(withoutHeredocs(command))) {
|
||||
// Output plumbing is not a file argument: `cat x.md | head -80`, `2>/dev/null`.
|
||||
const words = shellWords(segment.split(/\s\|\s/)[0]).filter((w) => !/^\d*>/.test(w));
|
||||
if (!words.length) continue;
|
||||
const [verb, ...rest] = words;
|
||||
if (/^[A-Za-z_][A-Za-z0-9_]*=/.test(verb) || verb === "echo" || verb === "printf") continue;
|
||||
if (verb === "cd") {
|
||||
insideConsoleDir = cdReads && /web-console\/?$/.test(String(rest[0] || "").replace(/\\/g, "/"));
|
||||
continue;
|
||||
}
|
||||
let files = null;
|
||||
if (verb === "cat") files = rest.filter((w) => !w.startsWith("-"));
|
||||
else if (verb === "sed" && rest[0] === "-n") files = rest.slice(2);
|
||||
if (!files) continue;
|
||||
const names = files.map((f) => docNameOf(f, { insideConsoleDir }));
|
||||
// A contract file read beside another file counts nowhere as standalone.
|
||||
if (!names.every(Boolean)) only = false;
|
||||
for (const name of names) if (name) docs.push(name);
|
||||
}
|
||||
return { docs, only: only && docs.length > 0 };
|
||||
}
|
||||
|
||||
const HELP_TOPIC_FILES = new Set(["finish", "stop", "chat", "workspace", "subagents", "meter", "attachments", "trouble", "launches", "docs"]);
|
||||
function helpTopicFile(topic) {
|
||||
if (topic === "building" || topic === "building-review") return `${topic}.md`;
|
||||
if (topic === "console") return "console.md";
|
||||
return HELP_TOPIC_FILES.has(topic) ? `console-${topic}.md` : `notes/${topic}.md`;
|
||||
}
|
||||
|
||||
function stripLineNumbers(text) {
|
||||
return text.replace(/^[ \t]*\d+(?:→|\t)/gm, "");
|
||||
}
|
||||
|
||||
function resultText(content) {
|
||||
if (typeof content === "string") return content;
|
||||
if (Array.isArray(content)) return content.map((c) => (c && c.type === "text" ? c.text : "")).join("\n");
|
||||
return "";
|
||||
}
|
||||
|
||||
// The CLI's own JSON lines in a tool result, and everything else.
|
||||
export function splitOutput(text) {
|
||||
const objects = [];
|
||||
const other = [];
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
const t = line.trim();
|
||||
if (t.startsWith("{") && t.endsWith("}")) {
|
||||
try {
|
||||
const parsed = JSON.parse(t);
|
||||
if (parsed && typeof parsed === "object") {
|
||||
objects.push({ raw: t, parsed });
|
||||
continue;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
if (t && !/^Exit code \d+$/.test(t)) other.push(line);
|
||||
}
|
||||
return { objects, other: other.join("\n") };
|
||||
}
|
||||
|
||||
export function postFailure(text) {
|
||||
if (/could not read --file|post needs --file/.test(text)) return "unreadable";
|
||||
if (/console: the payload was rejected/.test(text)) return "refused";
|
||||
if (/^Exit code 3\b/.test(text) && /^console: /m.test(text)) return "refused";
|
||||
return null;
|
||||
}
|
||||
|
||||
/* -------------------------------------------------------- reading transcripts */
|
||||
|
||||
// Every tool call of one transcript, in order, with its result. A resumed or
|
||||
// forked transcript repeats entries, so a call counts once by its id.
|
||||
export async function readCalls(file) {
|
||||
const calls = new Map();
|
||||
const stream = fs.createReadStream(file, { encoding: "utf8" });
|
||||
const lines = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
||||
for await (const line of lines) {
|
||||
if (!line.includes('"tool_use"') && !line.includes('"tool_result"')) continue;
|
||||
let entry;
|
||||
try {
|
||||
entry = JSON.parse(line);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (entry.isSidechain) continue;
|
||||
const content = entry.message && entry.message.content;
|
||||
if (!Array.isArray(content)) continue;
|
||||
for (const block of content) {
|
||||
if (block.type === "tool_use" && !calls.has(block.id)) {
|
||||
calls.set(block.id, { id: block.id, name: block.name, input: block.input || {}, at: entry.timestamp, result: null, resultCopies: 0 });
|
||||
} else if (block.type === "tool_result" && calls.has(block.tool_use_id)) {
|
||||
const call = calls.get(block.tool_use_id);
|
||||
call.resultCopies++;
|
||||
if (call.result === null) call.result = resultText(block.content);
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...calls.values()];
|
||||
}
|
||||
|
||||
export function isConsoleCall(call) {
|
||||
if (call.name !== "Bash" && call.name !== "PowerShell") return false;
|
||||
const command = String(call.input.command || "");
|
||||
return consoleVerbs(expandVars(command)).length > 0;
|
||||
}
|
||||
|
||||
function stampOf(text) {
|
||||
const m = SID.exec(String(text || ""));
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
function dayOf(sid) {
|
||||
return `${sid.slice(0, 4)}-${sid.slice(4, 6)}-${sid.slice(6, 8)}`;
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------- the rewrites */
|
||||
|
||||
const LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
|
||||
|
||||
export function reencodeItem(item) {
|
||||
if (!item || typeof item !== "object") return item;
|
||||
const out = { ...item };
|
||||
if (out.status === "open") delete out.status;
|
||||
if (out.required === true) delete out.required;
|
||||
if (out.allowOther === true) delete out.allowOther;
|
||||
if (Array.isArray(out.thread) && !out.thread.length) delete out.thread;
|
||||
if (Array.isArray(out.options)) {
|
||||
if (!out.options.length) delete out.options;
|
||||
else {
|
||||
out.options = out.options.map((opt, n) => {
|
||||
if (!opt || typeof opt !== "object" || opt.k !== LETTERS[n]) return opt;
|
||||
const { k, ...rest } = opt;
|
||||
return rest;
|
||||
});
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// A patch in the new shape: minified, with the console-filled defaults gone.
|
||||
// Text that is not one JSON object stays as it is (it would be refused anyway).
|
||||
export function reencodePatch(text) {
|
||||
let patch;
|
||||
try {
|
||||
patch = JSON.parse(text);
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
if (!patch || typeof patch !== "object" || Array.isArray(patch)) return text;
|
||||
if (Array.isArray(patch.items)) patch = { ...patch, items: patch.items.map(reencodeItem) };
|
||||
return JSON.stringify(patch);
|
||||
}
|
||||
|
||||
const WAIT_FIELDS = ["reply", "resume", "skillFile", "handoffFile", "handoffSkill", "fallback", "workspace", "subagents", "terminalLine"];
|
||||
const WARNING_FIELDS = ["unansweredChat", "warning"];
|
||||
|
||||
function pick(obj, keys) {
|
||||
const out = {};
|
||||
for (const k of keys) if (obj[k] !== undefined) out[k] = obj[k];
|
||||
return out;
|
||||
}
|
||||
|
||||
// `wait`, exit 0: what the person sent, the session actions and the hop fields.
|
||||
export function reencodeWaitSend(obj) {
|
||||
const out = pick(obj, WAIT_FIELDS);
|
||||
const session = (Array.isArray(obj.actions) ? obj.actions : []).filter((a) => a && typeof a.i === "string" && a.i.startsWith("__"));
|
||||
if (session.length) out.session = session;
|
||||
if (Array.isArray(obj.chat) && obj.chat.length) out.chat = obj.chat.map((c) => (c && typeof c === "object" ? c.seq : c));
|
||||
return out;
|
||||
}
|
||||
|
||||
// `wait`, exit 10.
|
||||
export function reencodeWaitIdle(obj) {
|
||||
return { waiting: true, ...pick(obj, ["open", "terminalLine", ...WARNING_FIELDS]) };
|
||||
}
|
||||
|
||||
// `post`, exit 0.
|
||||
export function reencodePostAck(obj) {
|
||||
return pick(obj, ["ok", "open", "terminalLine", ...WARNING_FIELDS]);
|
||||
}
|
||||
|
||||
function reencodeOutput(obj) {
|
||||
if (obj.status === "waiting") return reencodeWaitIdle(obj);
|
||||
if (obj.type === "submit" || obj.type === "chat" || obj.type === "cancel") return reencodeWaitSend(obj);
|
||||
if (obj.ok === true && typeof obj.items === "number") return reencodePostAck(obj);
|
||||
return obj;
|
||||
}
|
||||
|
||||
// One session's outputs in order: terminalLine stays the first time and when it
|
||||
// differs from the last one kept.
|
||||
export function reencodeOutputs(objects) {
|
||||
let lastLine;
|
||||
let lastOpen;
|
||||
return objects.map((obj) => {
|
||||
// An old still-waiting line has no `open`; the new one prints it, and it is
|
||||
// the count the last post ack gave (checked against real runs in Phase 7).
|
||||
const out = { ...reencodeOutput(obj) };
|
||||
if (obj.ok === true && typeof obj.open === "number") lastOpen = obj.open;
|
||||
if (out.waiting === true && out.open === undefined && lastOpen !== undefined) out.open = lastOpen;
|
||||
if (out.terminalLine !== undefined) {
|
||||
if (out.terminalLine === lastLine) delete out.terminalLine;
|
||||
else lastLine = out.terminalLine;
|
||||
}
|
||||
return out;
|
||||
});
|
||||
}
|
||||
|
||||
/* -------------------------------------------------------------- one session */
|
||||
|
||||
const POINTER_TOPICS = new Set(["console-finish.md", "console-stop.md", "console-docs.md", "console-chat.md", "console-workspace.md", "console-subagents.md", "building-review.md"]);
|
||||
|
||||
export function measureCalls(calls, { count, reencode = false, since, until, filePatches = true, repeatDocs = false, cdReads = true, docText = null } = {}) {
|
||||
const written = new Map();
|
||||
const row = {
|
||||
sid: null,
|
||||
commands: { open: 0, post: 0, wait: 0, chat: 0 },
|
||||
patches: 0,
|
||||
outputs: 0,
|
||||
docs: 0,
|
||||
total: 0,
|
||||
refused: 0,
|
||||
unreadable: 0,
|
||||
patchCount: 0,
|
||||
filePatches: 0,
|
||||
filePatchTokens: 0,
|
||||
repeatedDocTokens: 0,
|
||||
mixedDocTokens: 0,
|
||||
docReads: {},
|
||||
events: [],
|
||||
};
|
||||
const outputObjects = [];
|
||||
let currentSid = null;
|
||||
const inWindow = () => {
|
||||
if (!currentSid || (!since && !until)) return true;
|
||||
const day = dayOf(currentSid);
|
||||
return (!since || day >= since) && (!until || day <= until);
|
||||
};
|
||||
// One standalone doc result: once, or every time the transcript repeats it.
|
||||
const addStandaloneDoc = (call, text) => {
|
||||
const tokens = count(text);
|
||||
const repeated = tokens * Math.max(0, (call.resultCopies || 1) - 1);
|
||||
row.docs += tokens + (repeatDocs ? repeated : 0);
|
||||
row.repeatedDocTokens += repeated;
|
||||
};
|
||||
const noteRead = (name) => {
|
||||
row.docReads[name] = (row.docReads[name] || 0) + 1;
|
||||
row.events.push({ kind: "read", name });
|
||||
};
|
||||
|
||||
for (const call of calls) {
|
||||
if (call.name === "Write" && call.input.file_path) {
|
||||
written.set(normalizePath(call.input.file_path), String(call.input.content ?? ""));
|
||||
continue;
|
||||
}
|
||||
if (call.name === "Read") {
|
||||
const name = docNameOf(call.input.file_path || "");
|
||||
if (name && call.result !== null && inWindow()) {
|
||||
addStandaloneDoc(call, stripLineNumbers(call.result));
|
||||
noteRead(name);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (call.name !== "Bash" && call.name !== "PowerShell") continue;
|
||||
|
||||
const raw = String(call.input.command || "");
|
||||
const command = expandVars(raw);
|
||||
const here = heredocs(raw);
|
||||
for (const h of here) {
|
||||
const target = /\b(?:cat|tee)\s*(?:>+\s*)?("[^"]+"|'[^']+'|\S+)/.exec(expandVars(h.head));
|
||||
if (target && />|\btee\b/.test(h.head) && !/console\.mjs/.test(h.head)) written.set(normalizePath(target[1]), h.body);
|
||||
}
|
||||
const verbs = consoleVerbs(command);
|
||||
const text = call.result ?? "";
|
||||
|
||||
if (!verbs.length) {
|
||||
const reads = docReads(command, { cdReads });
|
||||
if (reads.only && call.result !== null && inWindow()) {
|
||||
addStandaloneDoc(call, text);
|
||||
for (const name of reads.docs) noteRead(name);
|
||||
} else if (call.result !== null && inWindow()) {
|
||||
for (const name of reads.docs) {
|
||||
noteRead(name);
|
||||
if (docText) row.mixedDocTokens += count(docText(name));
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const stamp = stampOf(command) || stampOf(text);
|
||||
if (stamp) {
|
||||
currentSid = stamp;
|
||||
if (!row.sid) row.sid = stamp;
|
||||
}
|
||||
if (!inWindow()) continue;
|
||||
|
||||
for (const verb of verbs) if (COUNTED_VERBS.has(verb)) row.commands[verb]++;
|
||||
|
||||
// A hop (`--no-open`, as every `resume` a send carries has) clears the page,
|
||||
// finish and all; a resume after a dead server keeps it.
|
||||
if (/console\.mjs["']?\s+open\b[^\n]*--resume\b[^\n]*--no-open\b/.test(command)) row.events.push({ kind: "hop" });
|
||||
|
||||
let posted = raw;
|
||||
if (verbs.includes("post") || verbs.includes("open")) {
|
||||
for (const h of here) {
|
||||
if (!/console\.mjs["']?\s+(post|open)\b/.test(expandVars(h.head))) continue;
|
||||
row.patches += count(reencode ? reencodePatch(h.body) : h.body);
|
||||
row.patchCount++;
|
||||
}
|
||||
const file = /--file\s+("[^"]+"|'[^']+'|\S+)/.exec(withoutHeredocs(command));
|
||||
if (file && file[1] !== "-") {
|
||||
const body = written.get(normalizePath(file[1]));
|
||||
if (body !== undefined) {
|
||||
posted += body;
|
||||
const tokens = count(reencode ? reencodePatch(body) : body);
|
||||
row.filePatchTokens += tokens;
|
||||
row.filePatches++;
|
||||
if (filePatches) {
|
||||
row.patches += tokens;
|
||||
row.patchCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (verbs.includes("post")) {
|
||||
const failure = postFailure(text);
|
||||
if (failure === "refused") row.refused++;
|
||||
if (failure === "unreadable") row.unreadable++;
|
||||
row.events.push({ kind: "post", at: call.at, failure, finish: /"finish"\s*:\s*\{/.test(posted), clearsFinish: /"finish"\s*:\s*null/.test(posted) });
|
||||
}
|
||||
if (verbs.includes("stop")) row.events.push({ kind: "stop", at: call.at });
|
||||
if (verbs.includes("help")) {
|
||||
const topic = /console\.mjs["']?\s+help\s+([a-z-]+)/.exec(command);
|
||||
if (topic && call.result !== null) {
|
||||
row.docs += count(text);
|
||||
noteRead(helpTopicFile(topic[1]));
|
||||
}
|
||||
}
|
||||
|
||||
if (verbs.some((v) => COUNTED_VERBS.has(v)) && call.result !== null) {
|
||||
const { objects, other } = splitOutput(text);
|
||||
for (const { parsed } of objects) {
|
||||
outputObjects.push(parsed);
|
||||
const pointer = typeof parsed.read === "string" ? parsed.read : null;
|
||||
if (pointer) row.events.push({ kind: "pointer", name: pointer.replace(/^.*[\\/]/, "") });
|
||||
}
|
||||
const reads = docReads(command, { cdReads });
|
||||
if (reads.docs.length) {
|
||||
row.docs += count(other);
|
||||
for (const name of reads.docs) noteRead(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const final = reencode ? reencodeOutputs(outputObjects) : outputObjects;
|
||||
for (const obj of final) row.outputs += count(JSON.stringify(obj));
|
||||
row.total = row.patches + row.outputs + row.docs;
|
||||
row.withMixed = row.total + row.mixedDocTokens;
|
||||
return row;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ the sums */
|
||||
|
||||
// The baseline's own definition, so the figures compare.
|
||||
export function quantile(values, q) {
|
||||
if (!values.length) return 0;
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
return sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))];
|
||||
}
|
||||
|
||||
const PARTS = ["patches", "outputs", "docs", "total", "withMixed"];
|
||||
|
||||
export function summarize(rows) {
|
||||
const summary = { sessions: rows.length, median: {}, p90: {}, totals: {} };
|
||||
for (const part of PARTS) {
|
||||
const values = rows.map((r) => r[part]);
|
||||
summary.median[part] = quantile(values, 0.5);
|
||||
summary.p90[part] = quantile(values, 0.9);
|
||||
summary.totals[part] = values.reduce((a, b) => a + b, 0);
|
||||
}
|
||||
summary.commands = rows.reduce((sum, r) => sum + Object.values(r.commands).reduce((a, b) => a + b, 0), 0);
|
||||
summary.refused = rows.reduce((sum, r) => sum + r.refused, 0);
|
||||
summary.unreadable = rows.reduce((sum, r) => sum + r.unreadable, 0);
|
||||
summary.repeatedDocTokens = rows.reduce((sum, r) => sum + r.repeatedDocTokens, 0);
|
||||
summary.filePatchTokens = rows.reduce((sum, r) => sum + r.filePatchTokens, 0);
|
||||
return summary;
|
||||
}
|
||||
|
||||
async function loadCounter() {
|
||||
const { encode } = await import("gpt-tokenizer/encoding/o200k_base");
|
||||
const cache = new Map();
|
||||
return (text) => {
|
||||
if (!text) return 0;
|
||||
let n = cache.get(text);
|
||||
if (n === undefined) {
|
||||
// Transcripts may quote a special token; it counts as plain text here.
|
||||
n = encode(text, { disallowedSpecial: new Set() }).length;
|
||||
cache.set(text, n);
|
||||
}
|
||||
return n;
|
||||
};
|
||||
}
|
||||
|
||||
function mainTranscripts(root) {
|
||||
const files = [];
|
||||
for (const project of fs.readdirSync(root, { withFileTypes: true })) {
|
||||
if (!project.isDirectory()) continue;
|
||||
const dir = path.join(root, project.name);
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
if (entry.isFile() && entry.name.endsWith(".jsonl")) files.push(path.join(dir, entry.name));
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
export async function measureFolder(root, { since, until, exclude = [], reencode = false, filePatches = true, repeatDocs = false, cdReads = true, count }) {
|
||||
const rows = [];
|
||||
for (const file of mainTranscripts(root)) {
|
||||
const id = path.basename(file, ".jsonl");
|
||||
if (exclude.some((e) => id.startsWith(e))) continue;
|
||||
const calls = await readCalls(file);
|
||||
if (!calls.some(isConsoleCall)) continue;
|
||||
const row = measureCalls(calls, { count, reencode, since, until, filePatches, repeatDocs, cdReads });
|
||||
if (!row.sid) continue;
|
||||
if (!Object.values(row.commands).some(Boolean)) continue;
|
||||
delete row.events;
|
||||
rows.push({ id, project: path.basename(path.dirname(file)), ...row });
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
// Each recorded figure beside what this run found.
|
||||
export function compareBaseline(summary, recorded = RECORDED_BASELINE) {
|
||||
const pairs = [
|
||||
["sessions", recorded.sessions, summary.sessions],
|
||||
["median patches", recorded.median.patches, summary.median.patches],
|
||||
["median outputs", recorded.median.outputs, summary.median.outputs],
|
||||
["median docs", recorded.median.docs, summary.median.docs],
|
||||
["median total", recorded.median.total, summary.median.total],
|
||||
["p90 total", recorded.p90.total, summary.p90.total],
|
||||
];
|
||||
const checks = pairs.map(([name, was, now]) => {
|
||||
const gap = (now - was) / was;
|
||||
return { name, recorded: was, reproduced: now, gapPercent: Math.round(gap * 1000) / 10, pass: Math.abs(gap) <= BASELINE_TOLERANCE };
|
||||
});
|
||||
return { pass: checks.every((c) => c.pass), differs: checks.filter((c) => !c.pass).map((c) => c.name), checks };
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------------- the live runs */
|
||||
|
||||
// Assistant messages in a stream-json log, grouped by message id, since one
|
||||
// message arrives as several events.
|
||||
export function streamMessages(text) {
|
||||
const byId = new Map();
|
||||
for (const line of text.split(/\r?\n/)) {
|
||||
if (!line.trim()) continue;
|
||||
let event;
|
||||
try {
|
||||
event = JSON.parse(line);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (event.type !== "assistant" || !event.message || event.parent_tool_use_id) continue;
|
||||
const key = event.message.id || `line-${byId.size}`;
|
||||
const blocks = Array.isArray(event.message.content) ? event.message.content : [];
|
||||
if (!byId.has(key)) byId.set(key, []);
|
||||
byId.get(key).push(...blocks);
|
||||
}
|
||||
return [...byId.values()];
|
||||
}
|
||||
|
||||
// Text-only assistant messages while a session is open: a turn that ended
|
||||
// without a wait, stranding the person's next click.
|
||||
export function endedOnText(messages) {
|
||||
let open = false;
|
||||
let slips = 0;
|
||||
for (const blocks of messages) {
|
||||
const uses = blocks.filter((b) => b.type === "tool_use");
|
||||
if (!uses.length) {
|
||||
if (open && blocks.some((b) => b.type === "text" && b.text && b.text.trim())) slips++;
|
||||
continue;
|
||||
}
|
||||
for (const use of uses) {
|
||||
const verbs = consoleVerbs(expandVars(String((use.input && use.input.command) || "")));
|
||||
if (verbs.includes("open")) open = true;
|
||||
if (verbs.includes("stop")) open = false;
|
||||
}
|
||||
}
|
||||
return slips;
|
||||
}
|
||||
|
||||
// What the events of one session say went wrong without anyone being told.
|
||||
export function eventSlips(events) {
|
||||
const slips = { stopWithoutFinish: 0, topicBeforePointer: 0, pointerNeverRead: 0 };
|
||||
let finished = false;
|
||||
const pointed = new Set();
|
||||
const readAfter = new Set();
|
||||
const anyPointer = events.some((e) => e.kind === "pointer");
|
||||
for (const e of events) {
|
||||
if (e.kind === "hop" || (e.kind === "post" && !e.failure && e.clearsFinish)) finished = false;
|
||||
if (e.kind === "post" && !e.failure && e.finish) finished = true;
|
||||
if (e.kind === "stop" && !finished) slips.stopWithoutFinish++;
|
||||
if (e.kind === "pointer") pointed.add(e.name);
|
||||
if (e.kind === "read" && POINTER_TOPICS.has(e.name)) {
|
||||
if (anyPointer && !pointed.has(e.name)) slips.topicBeforePointer++;
|
||||
if (pointed.has(e.name)) readAfter.add(e.name);
|
||||
}
|
||||
}
|
||||
for (const name of pointed) if (!readAfter.has(name)) slips.pointerNeverRead++;
|
||||
return slips;
|
||||
}
|
||||
|
||||
export function firstRetries(events) {
|
||||
const posts = events.filter((e) => e.kind === "post");
|
||||
let failures = 0;
|
||||
let success = 0;
|
||||
posts.forEach((post, n) => {
|
||||
if (!post.failure) return;
|
||||
failures++;
|
||||
if (posts[n + 1] && !posts[n + 1].failure) success++;
|
||||
});
|
||||
return { failures, success };
|
||||
}
|
||||
|
||||
// Seconds from each send to the agent's next post. Information only.
|
||||
export function turnTimes(sends, events) {
|
||||
const posts = events.filter((e) => e.kind === "post" && e.at).map((e) => Date.parse(e.at));
|
||||
const times = [];
|
||||
for (const send of sends) {
|
||||
// answer.mjs logs its own stage changes beside the sends.
|
||||
if (send.kind === "stage") continue;
|
||||
const at = Date.parse(send.at);
|
||||
const next = posts.find((p) => p > at);
|
||||
if (next !== undefined) times.push((next - at) / 1000);
|
||||
}
|
||||
return times;
|
||||
}
|
||||
|
||||
function readLines(file) {
|
||||
try {
|
||||
return fs.readFileSync(file, "utf8");
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
export async function measureRuns(dir, { reencode = false, count }) {
|
||||
const rows = [];
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
const runDir = path.join(dir, entry.name);
|
||||
const run = JSON.parse(readLines(path.join(runDir, "run.json")) || "null");
|
||||
if (!run) continue;
|
||||
// A nudged run has several transcripts, and a resume repeats earlier calls.
|
||||
const byId = new Map();
|
||||
for (const file of run.transcripts || []) {
|
||||
if (!fs.existsSync(file)) continue;
|
||||
for (const call of await readCalls(file)) if (!byId.has(call.id)) byId.set(call.id, call);
|
||||
}
|
||||
const calls = [...byId.values()];
|
||||
// The run's own copy of the contract files, as it read them.
|
||||
const consoleDir = path.join(run.project || "", ".claude", "skills", "plan2code", "references", "web-console");
|
||||
const docText = (name) => readLines(path.join(consoleDir, ...name.split("/")));
|
||||
const row = measureCalls(calls, { count, reencode, docText });
|
||||
const events = row.events;
|
||||
delete row.events;
|
||||
const sends = readLines(path.join(runDir, "answers.ndjson"))
|
||||
.split(/\r?\n/)
|
||||
.filter(Boolean)
|
||||
.map((l) => JSON.parse(l));
|
||||
const slips = eventSlips(events);
|
||||
slips.endedOnText = endedOnText(streamMessages(readLines(path.join(runDir, "stream.ndjson"))));
|
||||
rows.push({
|
||||
id: entry.name,
|
||||
model: run.model,
|
||||
run: run.run,
|
||||
finished: Boolean(run.finished),
|
||||
...row,
|
||||
firstRetry: firstRetries(events),
|
||||
slips,
|
||||
turnSeconds: turnTimes(sends, events),
|
||||
});
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
export function summarizeRuns(rows) {
|
||||
const byModel = {};
|
||||
for (const model of [...new Set(rows.map((r) => r.model))]) {
|
||||
// An unfinished run is a harness bug to fix and rerun, never a figure.
|
||||
const all = rows.filter((r) => r.model === model);
|
||||
const group = all.filter((r) => r.finished);
|
||||
const summary = summarize(group);
|
||||
const docReads = {};
|
||||
const slips = { stopWithoutFinish: 0, endedOnText: 0, topicBeforePointer: 0, pointerNeverRead: 0 };
|
||||
const firstRetry = { failures: 0, success: 0 };
|
||||
const seconds = [];
|
||||
for (const r of group) {
|
||||
for (const [name, n] of Object.entries(r.docReads)) docReads[name] = (docReads[name] || 0) + n;
|
||||
for (const k of Object.keys(slips)) slips[k] += r.slips[k] || 0;
|
||||
firstRetry.failures += r.firstRetry.failures;
|
||||
firstRetry.success += r.firstRetry.success;
|
||||
seconds.push(...r.turnSeconds);
|
||||
}
|
||||
byModel[model] = {
|
||||
...summary,
|
||||
runs: all.length,
|
||||
finished: group.length,
|
||||
firstRetry,
|
||||
slips,
|
||||
docReads,
|
||||
turnSeconds: { median: quantile(seconds, 0.5), p90: quantile(seconds, 0.9) },
|
||||
};
|
||||
}
|
||||
return { runs: rows.length, byModel };
|
||||
}
|
||||
|
||||
/* ---------------------------------------------------------------- the command */
|
||||
|
||||
function parseArgs(argv) {
|
||||
const opts = { exclude: [], positional: [] };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === "--baseline" || a === "--reencode") opts[a.slice(2)] = true;
|
||||
else if (a === "--since" || a === "--until" || a === "--out" || a === "--runs") opts[a.slice(2)] = argv[++i];
|
||||
else if (a === "--exclude") opts.exclude.push(argv[++i]);
|
||||
else if (a.startsWith("--")) throw new Error(`unknown option ${a}`);
|
||||
else opts.positional.push(a);
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
function writeRows(file, rows) {
|
||||
if (file) fs.writeFileSync(file, rows.map((r) => JSON.stringify(r)).join("\n") + "\n");
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const opts = parseArgs(process.argv.slice(2));
|
||||
const count = await loadCounter();
|
||||
const { reencode = false } = opts;
|
||||
|
||||
if (opts.runs) {
|
||||
const rows = await measureRuns(opts.runs, { reencode, count });
|
||||
writeRows(opts.out, rows);
|
||||
console.log(JSON.stringify({ reencode, ...summarizeRuns(rows) }));
|
||||
return;
|
||||
}
|
||||
|
||||
const root = opts.positional[0] || path.join(os.homedir(), ".claude", "projects");
|
||||
// The recorded method, whatever its slips (see the header), so the check
|
||||
// compares like for like.
|
||||
const window = opts.baseline
|
||||
? { since: RECORDED_BASELINE.since, until: RECORDED_BASELINE.until, exclude: RECORDED_BASELINE.exclude, filePatches: false, repeatDocs: true, cdReads: false }
|
||||
: { since: opts.since, until: opts.until, exclude: opts.exclude };
|
||||
const rows = await measureFolder(root, { ...window, reencode, count });
|
||||
writeRows(opts.out, rows);
|
||||
const summary = { reencode, ...window, ...summarize(rows) };
|
||||
|
||||
if (opts.baseline && !reencode) {
|
||||
const baseline = compareBaseline(summary);
|
||||
for (const c of baseline.checks) {
|
||||
process.stderr.write(`${c.pass ? "ok " : "GAP "} ${c.name}: recorded ${c.recorded}, reproduced ${c.reproduced} (${c.gapPercent}%)\n`);
|
||||
}
|
||||
console.log(JSON.stringify({ ...summary, baseline }));
|
||||
if (!baseline.pass) process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
console.log(JSON.stringify(summary));
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
|
||||
main().catch((e) => {
|
||||
process.stderr.write(`measure-console-wire: ${e.message}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
// Byte budgets for what the web console costs an agent.
|
||||
//
|
||||
// The console's wire format was trimmed on purpose (specs/console-wire-format):
|
||||
// a small card (`console.md`), topic files read only when their moment comes,
|
||||
// and CLI outputs that carry only what the agent acts on. This suite keeps it
|
||||
// that way. It counts bytes, never tokens, so it needs no tokenizer and no
|
||||
// network, and it never calls a model.
|
||||
//
|
||||
// Each budget is the after-run size plus 10 percent, rounded up to the next 100
|
||||
// bytes. A failure names the file or output, its size and its budget. Either
|
||||
// trim what grew, or raise the budget here on purpose, in the same change, so
|
||||
// the growth is a decision someone made and reviewed.
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { cli, payload, sessionUrl, BASE, WEB_CONSOLE_DIR, lastLine } from "./web-console-test-helpers.mjs";
|
||||
|
||||
// The desktop app's own variables change what `open` prints. Undefined drops
|
||||
// a variable from the spawned CLI's environment.
|
||||
const OUTSIDE_APP = Object.fromEntries(
|
||||
["PLAN2CODE_IN_APP", "PLAN2CODE_APP_TOKEN", "PLAN2CODE_APP_VERSION", "PLAN2CODE_APP_ANNOUNCE", "PLAN2CODE_EMBED_ORIGIN", "PLAN2CODE_CONSOLE_SID"].map((k) => [k, undefined])
|
||||
);
|
||||
|
||||
export const BUDGETS = {
|
||||
// The card an agent reads once per session.
|
||||
card: { "console.md": 9700 },
|
||||
// Each topic file under notes/, read when its workflow starts.
|
||||
notes: {
|
||||
"changelog.md": 1400,
|
||||
"diagnose.md": 2700,
|
||||
"document.md": 400,
|
||||
"finalize.md": 1600,
|
||||
"git.md": 6500,
|
||||
"handoff.md": 300,
|
||||
"implement.md": 5700,
|
||||
"init.md": 300,
|
||||
"loop.md": 11100,
|
||||
"pathfinder.md": 2400,
|
||||
"plan.md": 700,
|
||||
"quick-task.md": 4000,
|
||||
"review.md": 1000,
|
||||
"revise-plan.md": 1200,
|
||||
"xray.md": 2500,
|
||||
},
|
||||
// One printed line each, for a session with one choice card.
|
||||
outputs: {
|
||||
open: 800,
|
||||
post: 100,
|
||||
waitIdle: 100,
|
||||
waitSend: 100,
|
||||
},
|
||||
};
|
||||
|
||||
const bytes = (s) => Buffer.byteLength(s, "utf8");
|
||||
|
||||
function within(label, size, budget, hint) {
|
||||
assert.ok(
|
||||
size <= budget,
|
||||
`${label} is ${size} bytes, over its budget of ${budget}. Trim it, or raise the budget in scripts/test-console-wire-budgets.mjs on purpose in this same change.${hint ? ` ${hint}` : ""}`
|
||||
);
|
||||
}
|
||||
|
||||
test("the card stays within its byte budget", () => {
|
||||
for (const [file, budget] of Object.entries(BUDGETS.card)) {
|
||||
within(`src/web-console/${file}`, fs.statSync(path.join(WEB_CONSOLE_DIR, file)).size, budget);
|
||||
}
|
||||
});
|
||||
|
||||
test("every notes file has a budget and stays within it", () => {
|
||||
const dir = path.join(WEB_CONSOLE_DIR, "notes");
|
||||
const files = fs.readdirSync(dir).filter((f) => f.endsWith(".md")).sort();
|
||||
for (const file of files) {
|
||||
assert.ok(file in BUDGETS.notes, `src/web-console/notes/${file} has no budget: add one to BUDGETS.notes in scripts/test-console-wire-budgets.mjs`);
|
||||
within(`src/web-console/notes/${file}`, fs.statSync(path.join(dir, file)).size, BUDGETS.notes[file]);
|
||||
}
|
||||
});
|
||||
|
||||
test("a budgeted notes file still exists", () => {
|
||||
const dir = path.join(WEB_CONSOLE_DIR, "notes");
|
||||
for (const file of Object.keys(BUDGETS.notes)) {
|
||||
assert.ok(fs.existsSync(path.join(dir, file)), `BUDGETS.notes lists notes/${file}, which no longer exists: remove its line`);
|
||||
}
|
||||
});
|
||||
|
||||
test("a sample session's outputs stay within their byte budgets", async () => {
|
||||
const open = cli(["open", "--file", payload("budget-open.json", BASE), "--no-open"], { env: OUTSIDE_APP });
|
||||
const sid = open.json.sid;
|
||||
try {
|
||||
assert.equal(open.status, 0, open.stderr);
|
||||
within("`open` output", bytes(lastLine(open.stdout)), BUDGETS.outputs.open);
|
||||
|
||||
const item = { id: "budget-q", kind: "choice", title: "Pick one", body: "Which?", options: [{ text: "CSV" }, { text: "Excel" }] };
|
||||
const post = cli(["post", "--session", sid, "--file", payload("budget-post.json", { items: [item] })], { env: OUTSIDE_APP });
|
||||
assert.equal(post.status, 0, post.stderr);
|
||||
within("`post` output", bytes(lastLine(post.stdout)), BUDGETS.outputs.post);
|
||||
|
||||
const idle = cli(["wait", "--session", sid, "--seconds", "5"], { env: OUTSIDE_APP });
|
||||
assert.equal(idle.status, 10, idle.stderr);
|
||||
within("`wait` output with nobody answered", bytes(lastLine(idle.stdout)), BUDGETS.outputs.waitIdle);
|
||||
|
||||
await sendAndWait(sid, open.json.url);
|
||||
} finally {
|
||||
cli(["stop", "--session", sid, "--force"], { env: OUTSIDE_APP, expectFail: true });
|
||||
}
|
||||
});
|
||||
|
||||
async function sendAndWait(sid, url) {
|
||||
const s = sessionUrl(url);
|
||||
const res = await fetch(`${s.base}/submit`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", cookie: s.cookie, connection: "close" },
|
||||
body: JSON.stringify({ actions: [{ i: "budget-q", type: "answer", kind: "choice", k: "A" }], reply: "Pick one: a" }),
|
||||
});
|
||||
assert.equal(res.status, 200);
|
||||
const wait = cli(["wait", "--session", sid, "--seconds", "10"], { env: OUTSIDE_APP });
|
||||
assert.equal(wait.status, 0, wait.stderr);
|
||||
within("`wait` output with an answer", bytes(lastLine(wait.stdout)), BUDGETS.outputs.waitSend);
|
||||
}
|
||||
Reference in new issue
Block a user