mirror of
https://github.com/jparkerweb/plan2code.git
synced 2026-11-16 20:45:33 -08:00
Add console wire measurement tooling and per-run byte budgets
AI Assisted
This commit is contained in:
1 parent
8e34bf1005
commit
9c07ee5260
9 files changed
+1527
No files matched your search
@@ -0,0 +1,279 @@
|
||||
#!/usr/bin/env node
|
||||
// Plays the person in a scripted console session, for the live runs (run.mjs).
|
||||
//
|
||||
// node scripts/console-wire-runs/answer.mjs --session <sid> --home <console home> --project <dir> --log <answers.ndjson>
|
||||
//
|
||||
// It follows the page's state from the session's state.json and answers through
|
||||
// the same HTTP routes the page uses, with the session's cookie. The script is
|
||||
// fixed: pick Quick Task on the dashboard, answer its cards, ask one Quick
|
||||
// question once a doc is up, press the finish's dashboard button, launch
|
||||
// Pathfinder, answer one question turn, press Stop, and exit 0 once that stop's
|
||||
// finish is up. Every send is logged with a timestamp.
|
||||
//
|
||||
// The idea box (`templates: "idea"`) gets the fixed text below rather than its
|
||||
// placeholder: an agent writes that placeholder, and the runs have to build the
|
||||
// same small thing to compare.
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const QUICK_IDEA = "Add an optional second argument to greet: when it is { shout: true }, return the greeting in upper case. Add a test for it.";
|
||||
const PATHFINDER_IDEA = "A command line wrapper for greet, so people can run it from a terminal and pipe names into it.";
|
||||
const FALLBACK_TEXT = "Keep it as small as you can.";
|
||||
const QUICK_QUESTION = "Which file will this change the most?";
|
||||
const MAX_SENDS = 40;
|
||||
const TICK_MS = 400;
|
||||
// A question posted while the agent still says it is working: the page would
|
||||
// show it as not the person's turn, so give the agent this long to hand over.
|
||||
const WORKING_GRACE_MS = 60_000;
|
||||
const CHAT_REPLY_WAIT_MS = 120_000;
|
||||
|
||||
function parseArgs(argv) {
|
||||
const opts = {};
|
||||
for (let i = 0; i < argv.length; i += 2) {
|
||||
if (!argv[i].startsWith("--")) throw new Error(`unexpected ${argv[i]}`);
|
||||
opts[argv[i].slice(2)] = argv[i + 1];
|
||||
}
|
||||
for (const key of ["session", "home", "project", "log"]) {
|
||||
if (!opts[key]) throw new Error(`--${key} is required`);
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
const opts = parseArgs(process.argv.slice(2));
|
||||
// Before the imports below: the console reads its home once, as it loads.
|
||||
process.env.PLAN2CODE_CONSOLE_HOME = opts.home;
|
||||
const consoleDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "src", "web-console");
|
||||
const load = (file) => import(new URL(`file:///${path.join(consoleDir, file).replace(/\\/g, "/")}`).href);
|
||||
const { listHandles, repoRoots } = await load("sessions.mjs");
|
||||
const { isOpen } = await load("patches.mjs");
|
||||
const { currentConversation, readChat } = await load("inbox.mjs");
|
||||
const { launchReply, DASHBOARD_REPLY, SKILL_CATALOG, STOP_REPLY, stopAction } = await load("public/answers.js");
|
||||
|
||||
const sessionDir = path.join(opts.home, "sessions", opts.session);
|
||||
const project = repoRoots(opts.project).project;
|
||||
let sends = 0;
|
||||
|
||||
function readState() {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(path.join(sessionDir, "state.json"), "utf8"));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function resultPending() {
|
||||
try {
|
||||
return !JSON.parse(fs.readFileSync(path.join(sessionDir, "result.json"), "utf8")).consumedAt;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function logSend(entry) {
|
||||
fs.appendFileSync(opts.log, JSON.stringify({ at: new Date().toISOString(), ...entry }) + "\n");
|
||||
}
|
||||
|
||||
async function request(route, body) {
|
||||
const handle = listHandles(project).find((h) => h.sid === opts.session);
|
||||
if (!handle || !handle.url) return { status: 0, body: {} };
|
||||
const url = new URL(handle.url);
|
||||
const token = url.pathname.split("/")[2];
|
||||
const res = await fetch(`${url.origin}${route}`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", cookie: `p2c_console_${url.port}=${token}`, origin: url.origin, connection: "close" },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
return { status: res.status, body: await res.json().catch(() => ({})) };
|
||||
}
|
||||
|
||||
// One send, as the page's Send button makes it. True once the server took it.
|
||||
async function send(kind, actions, reply) {
|
||||
if (++sends > MAX_SENDS) throw new Error(`more than ${MAX_SENDS} sends: the script is stuck`);
|
||||
const res = await request("/submit", { actions, reply });
|
||||
if (res.status === 409) {
|
||||
sends--;
|
||||
return false;
|
||||
}
|
||||
if (res.status !== 200) throw new Error(`/submit said ${res.status}: ${JSON.stringify(res.body)}`);
|
||||
logSend({ kind, actions, reply });
|
||||
return true;
|
||||
}
|
||||
|
||||
function catalogEntry(workflow) {
|
||||
const entry = SKILL_CATALOG.find((s) => s.workflow === workflow);
|
||||
if (!entry) throw new Error(`no ${workflow} card in the catalog`);
|
||||
return entry;
|
||||
}
|
||||
|
||||
function launch(workflow) {
|
||||
const entry = catalogEntry(workflow);
|
||||
answered.clear();
|
||||
const action = { i: "__launch", type: "launch", skill: entry.skill, workflow: entry.workflow, title: entry.title };
|
||||
return send(`launch ${workflow}`, [action], launchReply(entry, null));
|
||||
}
|
||||
|
||||
function textAnswer(item, idea) {
|
||||
if (item.templates === "idea") return idea;
|
||||
const hint = String(item.placeholder || "").replace(/^\s*(for example|e\.g\.|example)[:,]?\s*/i, "").trim();
|
||||
const shaped = (value) => !item.pattern || new RegExp(item.pattern).test(value);
|
||||
if (hint && shaped(hint)) return hint;
|
||||
return shaped(FALLBACK_TEXT) ? FALLBACK_TEXT : "wire-fixture-change";
|
||||
}
|
||||
|
||||
// The answer a person following the fixed script gives one card, as the page
|
||||
// would stage it, with the word the terminal flow would have received.
|
||||
function answerFor(item, idea) {
|
||||
const options = Array.isArray(item.options) ? item.options : [];
|
||||
const pickOption = () => options.find((o) => o.recommended) || options[0];
|
||||
const tokenFor = (key) => (item.token && item.token[key]) || key;
|
||||
switch (item.kind) {
|
||||
case "choice":
|
||||
case "menu": {
|
||||
const option = pickOption();
|
||||
if (option) return { value: { k: option.k }, word: tokenFor(option.k) };
|
||||
return { value: { text: FALLBACK_TEXT }, word: FALLBACK_TEXT };
|
||||
}
|
||||
case "multi": {
|
||||
const option = pickOption();
|
||||
return { value: { ks: option ? [option.k] : [] }, word: option ? tokenFor(option.k) : "skip" };
|
||||
}
|
||||
case "text": {
|
||||
const text = textAnswer(item, idea);
|
||||
return { value: { text }, word: text };
|
||||
}
|
||||
case "confirm":
|
||||
return { value: { yes: true }, word: tokenFor("yes") };
|
||||
case "recap":
|
||||
return { value: { ok: true }, word: tokenFor("ok") };
|
||||
case "review": {
|
||||
const verdicts = Array.isArray(item.verdicts) && item.verdicts.length ? item.verdicts : [{ id: "approve" }];
|
||||
const verdict = verdicts.find((v) => !v.danger) || verdicts[0];
|
||||
return { value: { verdict: verdict.id }, word: tokenFor(verdict.id) };
|
||||
}
|
||||
case "checklist": {
|
||||
const steps = Array.isArray(item.steps) ? item.steps : [];
|
||||
return { value: { done: steps.map((_, n) => n), state: "done" }, word: "done" };
|
||||
}
|
||||
case "list":
|
||||
return { value: { text: "Looks right." }, word: "Looks right." };
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// A card is due once, and again only if the agent reopens it with something new.
|
||||
// Ids are only unique within a skill: a launch clears the page, and the next
|
||||
// skill may use `q1` again, so launch() forgets them.
|
||||
const answered = new Map();
|
||||
const cardKey = (item) => `${item.status}:${Array.isArray(item.thread) ? item.thread.length : 0}`;
|
||||
|
||||
function dueCards(state) {
|
||||
return (state.items || []).filter((item) => isOpen(item) && item.kind !== "notice" && answered.get(item.id) !== cardKey(item));
|
||||
}
|
||||
|
||||
let openSince = 0;
|
||||
function theirTurn(state, cards) {
|
||||
if (!cards.length) {
|
||||
openSince = 0;
|
||||
return false;
|
||||
}
|
||||
if (!openSince) openSince = Date.now();
|
||||
return !state.agent || state.agent.status !== "working" || Date.now() - openSince > WORKING_GRACE_MS;
|
||||
}
|
||||
|
||||
async function answerCards(cards, idea) {
|
||||
const actions = [];
|
||||
const lines = [];
|
||||
for (const item of cards) {
|
||||
const answer = answerFor(item, idea);
|
||||
if (!answer) continue;
|
||||
actions.push({ i: item.id, type: "answer", kind: item.kind, ...answer.value });
|
||||
lines.push(`${item.title}: ${answer.word}`);
|
||||
}
|
||||
if (!actions.length) return false;
|
||||
const taken = await send("answer", actions, lines.join("\n"));
|
||||
if (taken) {
|
||||
for (const item of cards) answered.set(item.id, cardKey(item));
|
||||
openSince = 0;
|
||||
}
|
||||
return taken;
|
||||
}
|
||||
|
||||
let asked = null;
|
||||
async function askQuickQuestion() {
|
||||
const conversation = currentConversation(readChat(sessionDir)).conversation;
|
||||
const res = await request("/chat", { text: QUICK_QUESTION, conversation });
|
||||
if (res.status !== 200) return;
|
||||
asked = { seq: res.body.seq, at: Date.now() };
|
||||
logSend({ kind: "chat", seq: res.body.seq, text: QUICK_QUESTION });
|
||||
}
|
||||
|
||||
const chatStillOwed = (state) =>
|
||||
asked && !(state.chat?.replies || []).some((r) => r && r.re === asked.seq) && Date.now() - asked.at < CHAT_REPLY_WAIT_MS;
|
||||
|
||||
async function step(stage, state) {
|
||||
const workflow = state.workflow;
|
||||
const waiting = state.agent && state.agent.status === "waiting";
|
||||
const cards = dueCards(state);
|
||||
|
||||
switch (stage) {
|
||||
case "launch-quick":
|
||||
if (workflow === "dashboard" && waiting && (await launch("quick-task"))) return "quick";
|
||||
return stage;
|
||||
|
||||
case "quick":
|
||||
if (workflow !== "quick-task") return stage;
|
||||
if (state.finish) {
|
||||
const taken = await send("dashboard", [{ i: "__dashboard", type: "dashboard" }], DASHBOARD_REPLY);
|
||||
return taken ? "launch-pathfinder" : stage;
|
||||
}
|
||||
if (!asked && (state.docs || []).length && theirTurn(state, cards)) {
|
||||
await askQuickQuestion();
|
||||
return stage;
|
||||
}
|
||||
if (chatStillOwed(state)) return stage;
|
||||
if (theirTurn(state, cards)) await answerCards(cards, QUICK_IDEA);
|
||||
return stage;
|
||||
|
||||
case "launch-pathfinder":
|
||||
if (workflow === "dashboard" && !state.finish && waiting && (await launch("pathfinder"))) return "pathfinder";
|
||||
return stage;
|
||||
|
||||
case "pathfinder":
|
||||
if (workflow !== "pathfinder") return stage;
|
||||
if (theirTurn(state, cards) && (await answerCards(cards, PATHFINDER_IDEA))) return "pathfinder-after-turn";
|
||||
return stage;
|
||||
|
||||
case "pathfinder-after-turn":
|
||||
if (theirTurn(state, cards) || (waiting && !cards.length)) {
|
||||
const taken = await send("stop", [stopAction()], STOP_REPLY);
|
||||
return taken ? "stopping" : stage;
|
||||
}
|
||||
return stage;
|
||||
|
||||
case "stopping":
|
||||
return state.finish ? "done" : stage;
|
||||
|
||||
default:
|
||||
throw new Error(`unknown stage ${stage}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
let stage = "launch-quick";
|
||||
while (stage !== "done") {
|
||||
await new Promise((resolve) => setTimeout(resolve, TICK_MS));
|
||||
const state = readState();
|
||||
if (!state || resultPending()) continue;
|
||||
const next = await step(stage, state);
|
||||
if (next !== stage) logSend({ kind: "stage", stage: next });
|
||||
stage = next;
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
process.stderr.write(`answer: ${e.message}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1 @@
|
||||
specs/
|
||||
@@ -0,0 +1,3 @@
|
||||
# AGENTS.md
|
||||
|
||||
A tiny Node project with no dependencies: `src/greet.js` exports `greet(name)`, and `test/greet.test.mjs` tests it with `node:test` (run `npm test`). Use ESM, keep functions small, and keep every change covered by a test.
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"name": "wire-fixture",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "node --test"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export function greet(name) {
|
||||
return `Hello, ${name}!`;
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { greet } from "../src/greet.js";
|
||||
|
||||
test("greet says hello by name", () => {
|
||||
assert.equal(greet("Sam"), "Hello, Sam!");
|
||||
});
|
||||
@@ -0,0 +1,258 @@
|
||||
#!/usr/bin/env node
|
||||
// The live-run driver for the console wire measure. Spends model tokens, so it
|
||||
// is run by hand and never from `npm test`.
|
||||
//
|
||||
// node scripts/console-wire-runs/run.mjs [--models opus,haiku] [--runs 3] [--timeout 20] [--out <dir>]
|
||||
//
|
||||
// Each run copies the fixture to a fresh temp folder, starts headless Claude
|
||||
// Code on `/plan2code --web` with the skills built under skills/, and starts
|
||||
// answer.mjs against the session it opens. Run it from outside the desktop app,
|
||||
// or let this script clear the app's variables, as it does for every run.
|
||||
//
|
||||
// `--setting-sources project` keeps the machine's own skills, hooks and memory
|
||||
// out: a personal skill of the same name would otherwise win over the built one.
|
||||
//
|
||||
// Each run has `--permission-mode bypassPermissions`: a model-driven agent with
|
||||
// unrestricted tools on this machine, working in a throwaway copy of the fixture.
|
||||
//
|
||||
// A run ends when the finish after the Pathfinder stop is up, or at --timeout.
|
||||
// An agent that ends its turn on text with the session open stops the process:
|
||||
// that is a silent slip the measure counts, so the run goes on the way a person
|
||||
// would, by telling it to carry on.
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(HERE, "..", "..");
|
||||
const FIXTURE = path.join(HERE, "fixture");
|
||||
const CONSOLE_CLI = path.join(ROOT, "skills", "plan2code", "references", "web-console", "console.mjs");
|
||||
const PROMPT = "/plan2code --web";
|
||||
const NUDGE = "The console page is still open and waiting on you. Carry on with the session.";
|
||||
const MAX_NUDGES = 5;
|
||||
const AFTER_FINISH_MS = 30_000;
|
||||
|
||||
function parseArgs(argv) {
|
||||
const opts = { models: "opus,haiku", runs: "3", timeout: "20", out: "" };
|
||||
for (let i = 0; i < argv.length; i += 2) {
|
||||
const key = argv[i].replace(/^--/, "");
|
||||
if (!Object.hasOwn(opts, key)) throw new Error(`unknown option ${argv[i]}`);
|
||||
opts[key] = argv[i + 1];
|
||||
}
|
||||
return opts;
|
||||
}
|
||||
|
||||
// What would tie a run to the session it started from. Other CLAUDE_CODE_*
|
||||
// variables stay: some carry the account or the provider (an OAuth token, Bedrock).
|
||||
const SESSION_VARS = [
|
||||
"CLAUDECODE",
|
||||
"CLAUDE_PID",
|
||||
"CLAUDE_EFFORT",
|
||||
"CLAUDE_CODE_CHILD_SESSION",
|
||||
"CLAUDE_CODE_ENTRYPOINT",
|
||||
"CLAUDE_CODE_EXECPATH",
|
||||
"CLAUDE_CODE_MESSAGING_SOCKET",
|
||||
"CLAUDE_CODE_MESSAGING_TOKEN",
|
||||
"CLAUDE_CODE_SESSION_ATTENDED",
|
||||
"CLAUDE_CODE_SESSION_ID",
|
||||
"CLAUDE_CODE_SSE_PORT",
|
||||
];
|
||||
|
||||
// Everything that would tie a run to the session or the app it started from.
|
||||
function runEnv(home) {
|
||||
const env = { ...process.env };
|
||||
for (const key of Object.keys(env)) {
|
||||
if (SESSION_VARS.includes(key) || key.startsWith("PLAN2CODE_")) delete env[key];
|
||||
}
|
||||
return {
|
||||
...env,
|
||||
PLAN2CODE_CONSOLE_HOME: home,
|
||||
PLAN2CODE_HANDOFFS_DIR: path.join(home, "handoffs"),
|
||||
PLAN2CODE_NOTEPADS_DIR: path.join(home, "notepads"),
|
||||
PLAN2CODE_NO_BROWSER: "1",
|
||||
};
|
||||
}
|
||||
|
||||
function realTemp(prefix) {
|
||||
return fs.realpathSync.native(fs.mkdtempSync(path.join(os.tmpdir(), prefix)));
|
||||
}
|
||||
|
||||
function makeProject() {
|
||||
const project = realTemp("p2c-wire-project-");
|
||||
fs.cpSync(FIXTURE, project, { recursive: true });
|
||||
fs.cpSync(path.join(ROOT, "skills"), path.join(project, ".claude", "skills"), { recursive: true });
|
||||
const init = spawnSync("git", ["init", "-q"], { cwd: project, encoding: "utf8" });
|
||||
if (init.status !== 0) throw new Error(`git init failed: ${init.stderr}`);
|
||||
return project;
|
||||
}
|
||||
|
||||
function killTree(child) {
|
||||
if (!child || child.exitCode !== null || child.signalCode !== null) return;
|
||||
if (process.platform === "win32") spawnSync("taskkill", ["/pid", String(child.pid), "/T", "/F"], { stdio: "ignore" });
|
||||
else child.kill("SIGKILL");
|
||||
}
|
||||
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
function startClaude({ model, prompt, resume, project, env, streamFile }) {
|
||||
const args = ["-p", prompt, "--model", model, "--output-format", "stream-json", "--verbose", "--permission-mode", "bypassPermissions", "--setting-sources", "project"];
|
||||
if (resume) args.push("--resume", resume);
|
||||
const out = fs.openSync(streamFile, "a");
|
||||
const child = spawn("claude", args, { cwd: project, env, stdio: ["ignore", out, out], windowsHide: true });
|
||||
child.once("close", () => fs.closeSync(out));
|
||||
return child;
|
||||
}
|
||||
|
||||
function exited(child) {
|
||||
return new Promise((resolve) => {
|
||||
if (child.exitCode !== null || child.signalCode !== null) resolve();
|
||||
else child.once("close", () => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
function streamSessionIds(streamFile) {
|
||||
const ids = [];
|
||||
for (const line of fs.readFileSync(streamFile, "utf8").split(/\r?\n/)) {
|
||||
if (!line.includes('"session_id"')) continue;
|
||||
try {
|
||||
const id = JSON.parse(line).session_id;
|
||||
if (id && !ids.includes(id)) ids.push(id);
|
||||
} catch {}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
function transcriptPaths(ids) {
|
||||
const root = path.join(os.homedir(), ".claude", "projects");
|
||||
const found = [];
|
||||
for (const dir of fs.readdirSync(root)) {
|
||||
for (const id of ids) {
|
||||
const file = path.join(root, dir, `${id}.jsonl`);
|
||||
if (fs.existsSync(file)) found.push(file);
|
||||
}
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
function firstSession(home) {
|
||||
try {
|
||||
return fs.readdirSync(path.join(home, "sessions"))[0] || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function sessionState(home, sid) {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(path.join(home, "sessions", sid, "state.json"), "utf8"));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function oneRun({ model, n, minutes, outDir }) {
|
||||
const runDir = path.join(outDir, `${model.replace(/[^a-zA-Z0-9.-]/g, "-")}-${n}`);
|
||||
fs.mkdirSync(runDir, { recursive: true });
|
||||
const streamFile = path.join(runDir, "stream.ndjson");
|
||||
const answersFile = path.join(runDir, "answers.ndjson");
|
||||
const home = realTemp("p2c-wire-home-");
|
||||
const project = makeProject();
|
||||
const env = runEnv(home);
|
||||
const deadline = Date.now() + minutes * 60_000;
|
||||
const started = new Date().toISOString();
|
||||
|
||||
let claude = startClaude({ model, prompt: PROMPT, project, env, streamFile });
|
||||
let claudeDone = false;
|
||||
let answer = null;
|
||||
let answerDone = false;
|
||||
let answerCode = null;
|
||||
let sid = null;
|
||||
let nudges = 0;
|
||||
let failure = null;
|
||||
claude.once("close", () => (claudeDone = true));
|
||||
|
||||
while (Date.now() < deadline && !answerDone) {
|
||||
await sleep(500);
|
||||
if (!sid) {
|
||||
sid = firstSession(home);
|
||||
if (sid) {
|
||||
answer = spawn(process.execPath, [path.join(HERE, "answer.mjs"), "--session", sid, "--home", home, "--project", project, "--log", answersFile], {
|
||||
cwd: project,
|
||||
env,
|
||||
stdio: ["ignore", "ignore", fs.openSync(path.join(runDir, "answer.err"), "a")],
|
||||
windowsHide: true,
|
||||
});
|
||||
answer.once("close", (code) => {
|
||||
answerDone = true;
|
||||
answerCode = code;
|
||||
});
|
||||
}
|
||||
}
|
||||
if (claudeDone && !answerDone) {
|
||||
if (!sid || nudges >= MAX_NUDGES) {
|
||||
failure = sid ? "the agent kept ending its turn" : "the agent ended without opening a session";
|
||||
break;
|
||||
}
|
||||
const ids = streamSessionIds(streamFile);
|
||||
nudges++;
|
||||
claudeDone = false;
|
||||
claude = startClaude({ model, prompt: NUDGE, resume: ids[ids.length - 1], project, env, streamFile });
|
||||
claude.once("close", () => (claudeDone = true));
|
||||
}
|
||||
}
|
||||
|
||||
const finished = answerDone && answerCode === 0;
|
||||
if (!finished && !failure) failure = answerDone ? `answer.mjs exited ${answerCode}` : "timed out";
|
||||
// The finish is up: leave the agent its `stop` before ending the process.
|
||||
if (finished) await Promise.race([exited(claude), sleep(AFTER_FINISH_MS)]);
|
||||
killTree(claude);
|
||||
killTree(answer);
|
||||
if (sid) spawnSync(process.execPath, [CONSOLE_CLI, "stop", "--session", sid], { cwd: project, env, stdio: "ignore" });
|
||||
|
||||
const state = sid ? sessionState(home, sid) : null;
|
||||
const record = {
|
||||
model,
|
||||
run: n,
|
||||
finished,
|
||||
failure,
|
||||
nudges,
|
||||
started,
|
||||
ended: new Date().toISOString(),
|
||||
sid,
|
||||
finalWorkflow: state ? state.workflow : null,
|
||||
transcripts: transcriptPaths(streamSessionIds(streamFile)),
|
||||
project,
|
||||
home,
|
||||
};
|
||||
fs.writeFileSync(path.join(runDir, "run.json"), JSON.stringify(record, null, 2) + "\n");
|
||||
return record;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const opts = parseArgs(process.argv.slice(2));
|
||||
const outDir = opts.out ? path.resolve(opts.out) : realTemp("p2c-wire-runs-");
|
||||
const fromRoot = path.relative(ROOT, outDir);
|
||||
if (!fromRoot || (!fromRoot.startsWith("..") && !path.isAbsolute(fromRoot))) throw new Error("--out must be outside the repository");
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
if (!fs.existsSync(CONSOLE_CLI)) throw new Error("skills/ is not built: run node install.js --build-skills first");
|
||||
const runs = Number(opts.runs);
|
||||
const results = [];
|
||||
for (const model of opts.models.split(",")) {
|
||||
for (let n = 1; n <= runs; n++) {
|
||||
const record = await oneRun({ model, n, minutes: Number(opts.timeout), outDir });
|
||||
results.push(record);
|
||||
process.stdout.write(`${model} run ${n}: ${record.finished ? "finished" : `FAILED (${record.failure})`}, ${record.nudges} nudges\n`);
|
||||
}
|
||||
}
|
||||
const finished = results.filter((r) => r.finished).length;
|
||||
console.log(JSON.stringify({ out: outDir, runs: results.length, finished }));
|
||||
if (finished !== results.length) process.exitCode = 1;
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
process.stderr.write(`run: ${e.message}\n`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
Reference in new issue
Block a user