#!/usr/bin/env node
// The live-run driver for the console wire measure. Spends model tokens, so it
// is run by hand and never from `npm test`.
//
// node scripts/console-wire-runs/run.mjs [--models opus,haiku] [--runs 3] [--timeout 20] [--out
]
//
// Each run copies the fixture to a fresh temp folder, starts headless Claude
// Code on `/plan2code --web` with the skills built under skills/, and starts
// answer.mjs against the session it opens. Run it from outside the desktop app,
// or let this script clear the app's variables, as it does for every run.
//
// `--setting-sources project` keeps the machine's own skills, hooks and memory
// out: a personal skill of the same name would otherwise win over the built one.
//
// Each run has `--permission-mode bypassPermissions`: a model-driven agent with
// unrestricted tools on this machine, working in a throwaway copy of the fixture.
//
// A run ends when the finish after the Pathfinder stop is up, or at --timeout.
// An agent that ends its turn on text with the session open stops the process:
// that is a silent slip the measure counts, so the run goes on the way a person
// would, by telling it to carry on.
import { spawn, spawnSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
const HERE = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.resolve(HERE, "..", "..");
const FIXTURE = path.join(HERE, "fixture");
const CONSOLE_CLI = path.join(ROOT, "skills", "plan2code", "references", "web-console", "console.mjs");
const PROMPT = "/plan2code --web";
const NUDGE = "The console page is still open and waiting on you. Carry on with the session.";
const MAX_NUDGES = 5;
const AFTER_FINISH_MS = 30_000;
function parseArgs(argv) {
const opts = { models: "opus,haiku", runs: "3", timeout: "20", out: "" };
for (let i = 0; i < argv.length; i += 2) {
const key = argv[i].replace(/^--/, "");
if (!Object.hasOwn(opts, key)) throw new Error(`unknown option ${argv[i]}`);
opts[key] = argv[i + 1];
}
return opts;
}
// What would tie a run to the session it started from. Other CLAUDE_CODE_*
// variables stay: some carry the account or the provider (an OAuth token, Bedrock).
const SESSION_VARS = [
"CLAUDECODE",
"CLAUDE_PID",
"CLAUDE_EFFORT",
"CLAUDE_CODE_CHILD_SESSION",
"CLAUDE_CODE_ENTRYPOINT",
"CLAUDE_CODE_EXECPATH",
"CLAUDE_CODE_MESSAGING_SOCKET",
"CLAUDE_CODE_MESSAGING_TOKEN",
"CLAUDE_CODE_SESSION_ATTENDED",
"CLAUDE_CODE_SESSION_ID",
"CLAUDE_CODE_SSE_PORT",
];
// Everything that would tie a run to the session or the app it started from.
function runEnv(home) {
const env = { ...process.env };
for (const key of Object.keys(env)) {
if (SESSION_VARS.includes(key) || key.startsWith("PLAN2CODE_")) delete env[key];
}
return {
...env,
PLAN2CODE_CONSOLE_HOME: home,
PLAN2CODE_HANDOFFS_DIR: path.join(home, "handoffs"),
PLAN2CODE_NOTEPADS_DIR: path.join(home, "notepads"),
PLAN2CODE_NO_BROWSER: "1",
};
}
function realTemp(prefix) {
return fs.realpathSync.native(fs.mkdtempSync(path.join(os.tmpdir(), prefix)));
}
function makeProject() {
const project = realTemp("p2c-wire-project-");
fs.cpSync(FIXTURE, project, { recursive: true });
fs.cpSync(path.join(ROOT, "skills"), path.join(project, ".claude", "skills"), { recursive: true });
const init = spawnSync("git", ["init", "-q"], { cwd: project, encoding: "utf8" });
if (init.status !== 0) throw new Error(`git init failed: ${init.stderr}`);
return project;
}
function killTree(child) {
if (!child || child.exitCode !== null || child.signalCode !== null) return;
if (process.platform === "win32") spawnSync("taskkill", ["/pid", String(child.pid), "/T", "/F"], { stdio: "ignore" });
else child.kill("SIGKILL");
}
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
function startClaude({ model, prompt, resume, project, env, streamFile }) {
const args = ["-p", prompt, "--model", model, "--output-format", "stream-json", "--verbose", "--permission-mode", "bypassPermissions", "--setting-sources", "project"];
if (resume) args.push("--resume", resume);
const out = fs.openSync(streamFile, "a");
const child = spawn("claude", args, { cwd: project, env, stdio: ["ignore", out, out], windowsHide: true });
child.once("close", () => fs.closeSync(out));
return child;
}
function exited(child) {
return new Promise((resolve) => {
if (child.exitCode !== null || child.signalCode !== null) resolve();
else child.once("close", () => resolve());
});
}
function streamSessionIds(streamFile) {
const ids = [];
for (const line of fs.readFileSync(streamFile, "utf8").split(/\r?\n/)) {
if (!line.includes('"session_id"')) continue;
try {
const id = JSON.parse(line).session_id;
if (id && !ids.includes(id)) ids.push(id);
} catch {}
}
return ids;
}
function transcriptPaths(ids) {
const root = path.join(os.homedir(), ".claude", "projects");
const found = [];
for (const dir of fs.readdirSync(root)) {
for (const id of ids) {
const file = path.join(root, dir, `${id}.jsonl`);
if (fs.existsSync(file)) found.push(file);
}
}
return found;
}
function firstSession(home) {
try {
return fs.readdirSync(path.join(home, "sessions"))[0] || null;
} catch {
return null;
}
}
function sessionState(home, sid) {
try {
return JSON.parse(fs.readFileSync(path.join(home, "sessions", sid, "state.json"), "utf8"));
} catch {
return null;
}
}
async function oneRun({ model, n, minutes, outDir }) {
const runDir = path.join(outDir, `${model.replace(/[^a-zA-Z0-9.-]/g, "-")}-${n}`);
fs.mkdirSync(runDir, { recursive: true });
const streamFile = path.join(runDir, "stream.ndjson");
const answersFile = path.join(runDir, "answers.ndjson");
const home = realTemp("p2c-wire-home-");
const project = makeProject();
const env = runEnv(home);
const deadline = Date.now() + minutes * 60_000;
const started = new Date().toISOString();
let claude = startClaude({ model, prompt: PROMPT, project, env, streamFile });
let claudeDone = false;
let answer = null;
let answerDone = false;
let answerCode = null;
let sid = null;
let nudges = 0;
let failure = null;
claude.once("close", () => (claudeDone = true));
while (Date.now() < deadline && !answerDone) {
await sleep(500);
if (!sid) {
sid = firstSession(home);
if (sid) {
answer = spawn(process.execPath, [path.join(HERE, "answer.mjs"), "--session", sid, "--home", home, "--project", project, "--log", answersFile], {
cwd: project,
env,
stdio: ["ignore", "ignore", fs.openSync(path.join(runDir, "answer.err"), "a")],
windowsHide: true,
});
answer.once("close", (code) => {
answerDone = true;
answerCode = code;
});
}
}
if (claudeDone && !answerDone) {
if (!sid || nudges >= MAX_NUDGES) {
failure = sid ? "the agent kept ending its turn" : "the agent ended without opening a session";
break;
}
const ids = streamSessionIds(streamFile);
nudges++;
claudeDone = false;
claude = startClaude({ model, prompt: NUDGE, resume: ids[ids.length - 1], project, env, streamFile });
claude.once("close", () => (claudeDone = true));
}
}
const finished = answerDone && answerCode === 0;
if (!finished && !failure) failure = answerDone ? `answer.mjs exited ${answerCode}` : "timed out";
// The finish is up: leave the agent its `stop` before ending the process.
if (finished) await Promise.race([exited(claude), sleep(AFTER_FINISH_MS)]);
killTree(claude);
killTree(answer);
if (sid) spawnSync(process.execPath, [CONSOLE_CLI, "stop", "--session", sid], { cwd: project, env, stdio: "ignore" });
const state = sid ? sessionState(home, sid) : null;
const record = {
model,
run: n,
finished,
failure,
nudges,
started,
ended: new Date().toISOString(),
sid,
finalWorkflow: state ? state.workflow : null,
transcripts: transcriptPaths(streamSessionIds(streamFile)),
project,
home,
};
fs.writeFileSync(path.join(runDir, "run.json"), JSON.stringify(record, null, 2) + "\n");
return record;
}
async function main() {
const opts = parseArgs(process.argv.slice(2));
const outDir = opts.out ? path.resolve(opts.out) : realTemp("p2c-wire-runs-");
const fromRoot = path.relative(ROOT, outDir);
if (!fromRoot || (!fromRoot.startsWith("..") && !path.isAbsolute(fromRoot))) throw new Error("--out must be outside the repository");
fs.mkdirSync(outDir, { recursive: true });
if (!fs.existsSync(CONSOLE_CLI)) throw new Error("skills/ is not built: run node install.js --build-skills first");
const runs = Number(opts.runs);
const results = [];
for (const model of opts.models.split(",")) {
for (let n = 1; n <= runs; n++) {
const record = await oneRun({ model, n, minutes: Number(opts.timeout), outDir });
results.push(record);
process.stdout.write(`${model} run ${n}: ${record.finished ? "finished" : `FAILED (${record.failure})`}, ${record.nudges} nudges\n`);
}
}
const finished = results.filter((r) => r.finished).length;
console.log(JSON.stringify({ out: outDir, runs: results.length, finished }));
if (finished !== results.length) process.exitCode = 1;
}
main().catch((e) => {
process.stderr.write(`run: ${e.message}\n`);
process.exitCode = 1;
});