Files

314 lines
14 KiB
JavaScript
Raw Permalink Normal View History

#!/usr/bin/env node
// Start-up benchmark for the web console. Dev-only: lives in scripts/ so it
// never ships inside a skill.
//
// node scripts/bench-web-console.mjs [--runs 7] [--workflow dashboard]
// [--browser <path> | --browser none] [--cwd <repo>] [--json <out.json>]
//
// Every run starts from nothing, the way a person's first `/plan2code` does:
// no server, a fresh browser context (each session gets a new port, so a new
// origin, so the HTTP cache is always cold). It measures, from the moment the
// agent runs `console.mjs open`:
//
// url the CLI has printed the link (what the agent hands over)
// cliExit the CLI process has exited (when the agent's next step can start)
// response the browser has the page's HTML
// fcp first contentful paint: the person sees something
// ready the session is drawn from real state (the menu, or a question)
//
// plus the bytes and request count the page cost. The browser columns need
// playwright-core (from `npm install`) and a local Chrome or Edge, picked by
// scripts/browser.mjs; without a browser it still reports url/cliExit and an
// HTTP-only waterfall.
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { spawn, spawnSync } from "node:child_process";
import { fileURLToPath } from "node:url";
import { findBrowser, launchBrowser } from "./browser.mjs";
const ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
const CONSOLE_CLI = path.join(ROOT, "src", "web-console", "console.mjs");
const args = {};
for (let i = 2; i < process.argv.length; i++) {
const a = process.argv[i];
if (a.startsWith("--")) args[a.slice(2)] = process.argv[i + 1] && !process.argv[i + 1].startsWith("--") ? process.argv[++i] : true;
}
const RUNS = Number(args.runs || 7);
const WORKFLOW = args.workflow || "dashboard";
const CWD = path.resolve(args.cwd || ROOT);
const HOME = fs.mkdtempSync(path.join(os.tmpdir(), "p2c-bench-"));
const ENV = { ...process.env, PLAN2CODE_CONSOLE_HOME: HOME, PLAN2CODE_NO_BROWSER: "1" };
// A fresh cached failure: the boot-time release check stays off the network,
// so a slow GitHub never colors the timings.
fs.writeFileSync(path.join(HOME, "update-check.json"), JSON.stringify({ checkedAt: Date.now(), latest: null }));
// Resolves at the first stdout line carrying a url, and again at exit.
// --launch hook (default): `open` launches "the browser" the way it would for
// a person, through PLAN2CODE_BROWSER, and the hook just writes the link to
// a file; the bench points its ready-made tab there the instant it appears.
// That measures everything but the real browser's own start-up, which is
// about 900ms on Windows and runs alongside the rest.
// --launch url: navigate only once `open` prints the link. For comparing
// against versions of console.mjs that predate the hook.
const LAUNCH = args.launch || "hook";
// --browser-delay <ms>: how long "the browser" takes to show up after it is
// asked. 0 is a browser that is instant; ~900 is Chrome on Windows through
// `start`, measured on this machine, and is what a person actually waits.
const BROWSER_DELAY = Number(args["browser-delay"] || 0);
const OPEN_LOG = path.join(HOME, "opened.txt");
function runOpen() {
fs.rmSync(OPEN_LOG, { force: true });
const hook = LAUNCH === "hook";
const env = hook
? { ...ENV, PLAN2CODE_NO_BROWSER: "", PLAN2CODE_BROWSER: `echo>>"${OPEN_LOG}"` }
: ENV;
const t0 = Date.now();
const child = spawn(
process.execPath,
[CONSOLE_CLI, "open", "--workflow", WORKFLOW, "--title", "Bench", ...(hook ? [] : ["--no-open"])],
{ cwd: CWD, env, stdio: ["ignore", "pipe", "pipe"] }
);
let out = "";
let err = "";
let urlResolve;
const urlP = new Promise((r) => (urlResolve = r));
child.stdout.on("data", (c) => {
out += c;
const m = out.match(/"url":"([^"]+)"/);
if (m) urlResolve({ url: m[1], at: Date.now() });
});
child.stderr.on("data", (c) => (err += c));
const exitP = new Promise((r) =>
child.on("exit", (code) => {
urlResolve({ url: null, at: Date.now() });
r({ code, at: Date.now(), out, err });
})
);
// The moment "the browser" was asked to open, and with what.
const openP = hook
? new Promise((resolve) => {
const poll = setInterval(() => {
let text = "";
try {
text = fs.readFileSync(OPEN_LOG, "utf8");
} catch {}
const m = text.match(/http:\/\/127\.0\.0\.1:\d+\/s\/[\w-]+\//);
if (m) {
clearInterval(poll);
resolve({ url: m[0], at: Date.now() });
}
}, 2);
exitP.then(() => setTimeout(() => (clearInterval(poll), resolve({ url: null, at: Date.now() })), 500));
})
: urlP;
return { t0, urlP, exitP, openP };
}
function stopAll() {
spawnSync(process.execPath, [CONSOLE_CLI, "stop", "--all"], { cwd: CWD, env: ENV });
}
async function httpWaterfall(url) {
// What the page costs without a browser: the redirect, the HTML, every
// asset it names, then the modules those import. Sequential levels, like a
// browser without preload hints would discover them.
const t = Date.now();
const r1 = await fetch(url, { redirect: "manual" });
const cookie = (r1.headers.get("set-cookie") || "").split(";")[0];
const base = new URL(url).origin;
const get = async (p) => {
const res = await fetch(base + p, { headers: { cookie } });
const body = Buffer.from(await res.arrayBuffer());
return { p, status: res.status, bytes: body.length, text: body.toString("utf8") };
};
const html = await get("/");
let bytes = html.bytes;
let requests = 2;
const seen = new Set();
let level = [...html.text.matchAll(/(?:href|src)="(\/[^"#]+\.(?:css|js))"/g)].map((m) => m[1]);
let levels = 1;
while (level.length) {
const fresh = level.filter((p) => !seen.has(p));
fresh.forEach((p) => seen.add(p));
if (!fresh.length) break;
const got = await Promise.all(fresh.map(get));
requests += got.length;
bytes += got.reduce((n, g) => n + g.bytes, 0);
level = got
.filter((g) => g.p.endsWith(".js"))
.flatMap((g) => [...g.text.matchAll(/from\s+"\.\/([^"]+)"/g)].map((m) => "/" + path.posix.join(path.posix.dirname(g.p.slice(1)), m[1])));
levels++;
}
return { ms: Date.now() - t, bytes, requests, levels };
}
// --roundtrip: the two hops of a live session, rather than the start.
// send the person presses Send -> the agent's `wait` returns the answers
// post the agent runs `post` -> the page's stream delivers the new state
async function roundtrip() {
const payload = path.join(HOME, "rt.json");
const item = { id: "q1", kind: "text", title: "Anything", body: "Say something." };
fs.writeFileSync(payload, JSON.stringify({ workflow: "pathfinder", items: [item], agent: { status: "waiting" } }));
const out = spawnSync(process.execPath, [CONSOLE_CLI, "open", "--file", payload, "--no-open"], { cwd: CWD, env: ENV, encoding: "utf8" });
const opened = JSON.parse(out.stdout.trim().split("\n").pop());
const [, base, token] = opened.url.match(/^(http:\/\/127\.0\.0\.1:\d+)\/s\/([^/]+)\//);
const headers = { cookie: `p2c_console_${new URL(base).port}=${token}`, "content-type": "application/json" };
const sends = [];
const posts = [];
for (let i = -1; i < RUNS; i++) {
// send: a waiter already sitting in its slice, then the press.
const waiter = spawn(process.execPath, [CONSOLE_CLI, "wait", "--session", opened.sid, "--seconds", "30"], { cwd: CWD, env: ENV });
const done = new Promise((r) => waiter.on("exit", () => r(Date.now())));
await new Promise((r) => setTimeout(r, 700));
const t = Date.now();
await fetch(base + "/submit", { method: "POST", headers, body: JSON.stringify({ actions: [{ i: "q1", type: "answer", kind: "text", text: "x" + i }], reply: "x" }) });
const sendMs = (await done) - t;
// post: a page listening on the stream, then the agent's update.
const ctrl = new AbortController();
const res = await fetch(base + "/events", { headers, signal: ctrl.signal });
const reader = res.body.getReader();
const dec = new TextDecoder();
await reader.read();
const patch = path.join(HOME, "rt-patch.json");
fs.writeFileSync(patch, JSON.stringify({ items: [{ id: "q1", status: "open", body: "Again " + i }], agent: { status: "waiting" } }));
const tp = Date.now();
spawnSync(process.execPath, [CONSOLE_CLI, "post", "--session", opened.sid, "--file", patch], { cwd: CWD, env: ENV });
const postCli = Date.now() - tp;
let buf = "";
while (!buf.includes("Again " + i)) buf += dec.decode((await reader.read()).value);
const postMs = Date.now() - tp;
ctrl.abort();
if (i >= 0) {
sends.push(sendMs);
posts.push({ total: postMs, cli: postCli });
}
}
spawnSync(process.execPath, [CONSOLE_CLI, "stop", "--all"], { cwd: CWD, env: ENV });
const med = (v) => v.sort((a, b) => a - b)[Math.floor(v.length / 2)];
console.log(`web console round trips, ${RUNS} runs`);
console.log(`send -> wait returns median ${med(sends)} ms (min ${Math.min(...sends)}, max ${Math.max(...sends)})`);
console.log(`post -> page has it median ${med(posts.map((p) => p.total))} ms (of which the post CLI ${med(posts.map((p) => p.cli))} ms)`);
fs.rmSync(HOME, { recursive: true, force: true });
}
async function main() {
if (args.roundtrip) return roundtrip();
// --browser none: no launch at all. --browser <path>: that executable.
// Otherwise the helper's pick; a browser problem is one line, then the
// HTTP-only waterfall.
let browser = null;
let browserName = null;
if (args.browser !== "none") {
const custom = typeof args.browser === "string" ? args.browser : null;
try {
browser = await launchBrowser(custom ? { executablePath: custom } : {});
const found = custom ? null : findBrowser();
browserName = found ? `${found.channel} (${found.path})` : custom;
} catch (e) {
if (!e.browserProblem) throw e;
console.error(e.message);
}
}
const rows = [];
// One throwaway run so disk caches and the Node binary are warm; the person
// running this has run node before too.
for (let i = -1; i < RUNS; i++) {
stopAll();
// The tab exists before the clock starts: a person's browser is already
// running, and creating a context is the harness's cost, not the page's.
const ctx = browser ? await browser.newContext() : null;
const page = ctx ? await ctx.newPage() : null;
const { t0, urlP, exitP, openP } = runOpen();
const o = await openP;
if (!o.url) {
const e = await exitP;
throw new Error(`open failed (${e.code}): ${e.err}`);
}
const row = { open: o.at - t0 };
if (browser) {
let bytes = 0;
let requests = 0;
page.on("response", async (res) => {
requests++;
try {
bytes += (await res.body()).length;
} catch {}
});
if (BROWSER_DELAY) await new Promise((r) => setTimeout(r, Math.max(0, o.at + BROWSER_DELAY - Date.now())));
await page.goto(o.url, { waitUntil: "domcontentloaded" });
const timing = await page.evaluate(async () => {
// "ready" = drawn from real state: a dashboard card, a question card,
// the starting screen or the waiting pane a real state produces.
const readySel = ".dash-card, .card.question, .waiting-pane, .launching.starting, .handoff";
await new Promise((resolve) => {
const check = () => (document.querySelector(readySel) ? resolve() : requestAnimationFrame(check));
check();
});
const readyAt = performance.timeOrigin + performance.now();
// The real screen can be drawn before the first paint happens at all,
// so wait for the paint rather than reading an entry not yet made.
const fcp = await new Promise((resolve) => {
const got = () => performance.getEntriesByName("first-contentful-paint")[0];
if (got()) return resolve(got());
new PerformanceObserver(() => got() && resolve(got())).observe({ type: "paint", buffered: true });
setTimeout(() => resolve(got() || null), 3000);
});
const nav = performance.getEntriesByType("navigation")[0];
return {
origin: performance.timeOrigin,
fcp: fcp ? performance.timeOrigin + fcp.startTime : null,
response: nav ? performance.timeOrigin + nav.responseEnd : null,
ready: readyAt,
};
});
// The origin here is the redirected document, so the 302 is inside
// `response`, not before it.
row.response = Math.round(timing.response - t0);
row.fcp = timing.fcp ? Math.round(timing.fcp - t0) : null;
row.ready = Math.round(timing.ready - t0);
await new Promise((r) => setTimeout(r, 150));
row.requests = requests;
row.kb = Math.round(bytes / 1024);
await ctx.close();
}
const e = await exitP;
row.url = (await urlP).at - t0;
row.cliExit = e.at - t0;
if (!browser) Object.assign(row, await httpWaterfall(o.url));
if (i >= 0) rows.push(row);
}
stopAll();
if (browser) await browser.close();
const cols = Object.keys(rows[0]);
const stat = (k) => {
const v = rows.map((r) => r[k]).filter((x) => typeof x === "number").sort((a, b) => a - b);
if (!v.length) return null;
return { median: v[Math.floor(v.length / 2)], min: v[0], max: v[v.length - 1] };
};
const summary = Object.fromEntries(cols.map((k) => [k, stat(k)]));
console.log(`web console start-up, ${RUNS} runs, workflow=${WORKFLOW}, cwd=${CWD}`);
console.log(`browser: ${browser ? browserName : "none (HTTP-only waterfall)"}`);
console.log("metric".padEnd(10) + "median".padStart(9) + "min".padStart(9) + "max".padStart(9));
for (const k of cols) {
const s = summary[k];
if (!s) continue;
const unit = k === "kb" ? " KB" : k === "requests" || k === "levels" ? "" : " ms";
console.log(k.padEnd(10) + `${s.median}${unit}`.padStart(9) + `${s.min}`.padStart(9) + `${s.max}`.padStart(9));
}
if (args.json) fs.writeFileSync(path.resolve(args.json), JSON.stringify({ runs: rows, summary }, null, 2));
fs.rmSync(HOME, { recursive: true, force: true });
}
main().catch((e) => {
stopAll();
console.error(e);
process.exit(1);
});