feat: run each loop phase on its own strength (Thrifty, Balanced, Best)

AI Assisted
This commit is contained in:
jparkerweb committed 2026-11-04 19:23:52 -08:00
1 parent e963513a8a
commit 81d14c8e78
41 files changed
+3290 -81

No files matched your search

+163
View File
@@ -1891,6 +1891,169 @@
"args": [
"a $HOME & \"b\""
]
},
{
"name": "OpenCode's shape: a chosen model goes ahead of the prompt",
"entry": {
"id": "opencode",
"name": "OpenCode",
"command": "opencode",
"args": [
"{model}",
"--prompt",
"{prompt}"
],
"prompt": "Use the plan2code skill",
"bypassArgs": [
"--auto"
],
"modelArgs": [
"--model",
"{model}"
]
},
"options": {
"model": "anthropic/sonnet",
"bypass": false
},
"args": [
"--model",
"anthropic/sonnet",
"--prompt",
"Use the plan2code skill"
]
},
{
"name": "OpenCode's shape: no model, no model argument",
"entry": {
"id": "opencode",
"name": "OpenCode",
"command": "opencode",
"args": [
"{model}",
"--prompt",
"{prompt}"
],
"prompt": "Use the plan2code skill",
"bypassArgs": [
"--auto"
],
"modelArgs": [
"--model",
"{model}"
]
},
"options": {
"bypass": false
},
"args": [
"--prompt",
"Use the plan2code skill"
]
},
{
"name": "Gemini CLI's shape: a chosen model goes ahead of the prompt",
"entry": {
"id": "gemini",
"name": "Gemini CLI",
"command": "gemini",
"args": [
"{model}",
"{prompt}"
],
"prompt": "Use the plan2code skill",
"bypassArgs": [
"--approval-mode=yolo"
],
"modelArgs": [
"--model",
"{model}"
]
},
"options": {
"model": "flash",
"bypass": false
},
"args": [
"--model",
"flash",
"Use the plan2code skill"
]
},
{
"name": "Gemini CLI's shape: no model, no model argument",
"entry": {
"id": "gemini",
"name": "Gemini CLI",
"command": "gemini",
"args": [
"{model}",
"{prompt}"
],
"prompt": "Use the plan2code skill",
"bypassArgs": [
"--approval-mode=yolo"
],
"modelArgs": [
"--model",
"{model}"
]
},
"options": {
"bypass": false
},
"args": [
"Use the plan2code skill"
]
},
{
"name": "Pi's shape: a chosen model goes ahead of the prompt",
"entry": {
"id": "pi",
"name": "Pi",
"command": "pi",
"args": [
"{model}",
"{prompt}"
],
"prompt": "/skill:plan2code",
"modelArgs": [
"--model",
"{model}"
]
},
"options": {
"model": "sonnet:high",
"bypass": false
},
"args": [
"--model",
"sonnet:high",
"/skill:plan2code"
]
},
{
"name": "Pi's shape: no model, no model argument",
"entry": {
"id": "pi",
"name": "Pi",
"command": "pi",
"args": [
"{model}",
"{prompt}"
],
"prompt": "/skill:plan2code",
"modelArgs": [
"--model",
"{model}"
]
},
"options": {
"bypass": false
},
"args": [
"/skill:plan2code"
]
}
],
"ids": [
+478
View File
@@ -0,0 +1,478 @@
{
"schema": 1,
"cases": [
{
"name": "claude shipped only",
"agent": "claude",
"shipped": {
"thrifty": {
"model": "sonnet",
"effort": "low"
},
"balanced": {
"model": "sonnet",
"effort": "medium"
},
"best": {
"model": "opus",
"effort": "medium"
}
},
"user": null,
"models": [
"default",
"sonnet",
"opus",
"haiku"
],
"hasEffort": true,
"expect": {
"resolved": {
"thrifty": {
"model": "sonnet",
"effort": "low"
},
"balanced": {
"model": "sonnet",
"effort": "medium"
},
"best": {
"model": "opus",
"effort": "medium"
}
},
"next": {
"thrifty": "balanced",
"balanced": "best",
"best": null
}
}
},
{
"name": "devin shipped only",
"agent": "devin",
"shipped": {
"thrifty": {
"model": "swe-2-medium"
},
"balanced": {
"model": "swe-2-high"
},
"best": {
"model": "claude-opus-5-5-medium"
}
},
"user": null,
"models": [
"adaptive",
"swe-2-medium",
"swe-2-high",
"claude-opus-5-5-medium"
],
"hasEffort": false,
"expect": {
"resolved": {
"thrifty": {
"model": "swe-2-medium"
},
"balanced": {
"model": "swe-2-high"
},
"best": {
"model": "claude-opus-5-5-medium"
}
},
"next": {
"thrifty": "balanced",
"balanced": "best",
"best": null
}
}
},
{
"name": "a user row overrides one shipped row, the others follow shipped",
"agent": "claude",
"shipped": {
"thrifty": {
"model": "sonnet",
"effort": "low"
},
"balanced": {
"model": "sonnet",
"effort": "medium"
},
"best": {
"model": "opus",
"effort": "medium"
}
},
"user": {
"balanced": {
"model": "opus",
"effort": "low"
}
},
"models": [
"default",
"sonnet",
"opus",
"haiku"
],
"hasEffort": true,
"expect": {
"resolved": {
"thrifty": {
"model": "sonnet",
"effort": "low"
},
"balanced": {
"model": "opus",
"effort": "low"
},
"best": {
"model": "opus",
"effort": "medium"
}
},
"next": {
"thrifty": "balanced",
"balanced": "best",
"best": null
}
}
},
{
"name": "user-only mapping for an agent with nothing shipped",
"agent": "codex",
"shipped": null,
"user": {
"thrifty": {
"model": "gpt-a"
},
"balanced": {
"model": "gpt-b"
},
"best": {
"model": "gpt-c"
}
},
"models": [
"gpt-a",
"gpt-b",
"gpt-c"
],
"hasEffort": false,
"expect": {
"resolved": {
"thrifty": {
"model": "gpt-a"
},
"balanced": {
"model": "gpt-b"
},
"best": {
"model": "gpt-c"
}
},
"next": {
"thrifty": "balanced",
"balanced": "best",
"best": null
}
}
},
{
"name": "partly mapped: Balanced only",
"agent": "opencode",
"shipped": null,
"user": {
"balanced": {
"model": "m1"
}
},
"models": [
"m1",
"m2"
],
"hasEffort": false,
"expect": {
"resolved": {
"balanced": {
"model": "m1"
}
},
"next": {
"thrifty": "balanced",
"balanced": null,
"best": null
}
}
},
{
"name": "partly mapped: Thrifty and Best only",
"agent": "opencode",
"shipped": null,
"user": {
"thrifty": {
"model": "m1"
},
"best": {
"model": "m2"
}
},
"models": [
"m1",
"m2"
],
"hasEffort": false,
"expect": {
"resolved": {
"thrifty": {
"model": "m1"
},
"best": {
"model": "m2"
}
},
"next": {
"thrifty": "best",
"balanced": "best",
"best": null
}
}
},
{
"name": "a user row whose model left the list falls back to the shipped row",
"agent": "claude",
"shipped": {
"thrifty": {
"model": "sonnet",
"effort": "low"
},
"balanced": {
"model": "sonnet",
"effort": "medium"
},
"best": {
"model": "opus",
"effort": "medium"
}
},
"user": {
"best": {
"model": "gone",
"effort": "high"
}
},
"models": [
"default",
"sonnet",
"opus",
"haiku"
],
"hasEffort": true,
"expect": {
"resolved": {
"thrifty": {
"model": "sonnet",
"effort": "low"
},
"balanced": {
"model": "sonnet",
"effort": "medium"
},
"best": {
"model": "opus",
"effort": "medium"
}
},
"next": {
"thrifty": "balanced",
"balanced": "best",
"best": null
}
}
},
{
"name": "a shipped row whose model left the list reads unmapped",
"agent": "claude",
"shipped": {
"thrifty": {
"model": "sonnet",
"effort": "low"
},
"balanced": {
"model": "sonnet",
"effort": "medium"
},
"best": {
"model": "opus",
"effort": "medium"
}
},
"user": null,
"models": [
"default",
"opus"
],
"hasEffort": true,
"expect": {
"resolved": {
"best": {
"model": "opus",
"effort": "medium"
}
},
"next": {
"thrifty": "best",
"balanced": "best",
"best": null
}
}
},
{
"name": "an effort on an agent without one is dropped",
"agent": "devin",
"shipped": null,
"user": {
"thrifty": {
"model": "swe-2-medium",
"effort": "low"
},
"best": {
"model": "swe-2-high",
"effort": "high"
}
},
"models": [
"adaptive",
"swe-2-medium",
"swe-2-high",
"claude-opus-5-5-medium"
],
"hasEffort": false,
"expect": {
"resolved": {
"thrifty": {
"model": "swe-2-medium"
},
"best": {
"model": "swe-2-high"
}
},
"next": {
"thrifty": "best",
"balanced": "best",
"best": null
}
}
},
{
"name": "an effort outside low, medium and high is dropped, the row stays",
"agent": "claude",
"shipped": null,
"user": {
"thrifty": {
"model": "haiku",
"effort": "max"
},
"balanced": {
"model": "sonnet",
"effort": "medium"
}
},
"models": [
"default",
"sonnet",
"opus",
"haiku"
],
"hasEffort": true,
"expect": {
"resolved": {
"thrifty": {
"model": "haiku"
},
"balanced": {
"model": "sonnet",
"effort": "medium"
}
},
"next": {
"thrifty": "balanced",
"balanced": null,
"best": null
}
}
},
{
"name": "a bad word or a row that is not an object is ignored",
"agent": "claude",
"shipped": null,
"user": {
"cheap": {
"model": "haiku"
},
"thrifty": "haiku",
"balanced": [
"sonnet"
],
"best": {
"model": 7
},
"extra": null
},
"models": [
"default",
"sonnet",
"opus",
"haiku"
],
"hasEffort": true,
"expect": {
"resolved": {},
"next": {
"thrifty": null,
"balanced": null,
"best": null
}
}
},
{
"name": "nothing anywhere",
"agent": "pi",
"shipped": null,
"user": null,
"models": [],
"hasEffort": false,
"expect": {
"resolved": {},
"next": {
"thrifty": null,
"balanced": null,
"best": null
}
}
},
{
"name": "mappings that are not objects",
"agent": "pi",
"shipped": [
"x"
],
"user": "y",
"models": [
"x"
],
"hasEffort": false,
"expect": {
"resolved": {},
"next": {
"thrifty": null,
"balanced": null,
"best": null
}
}
}
]
}
+13 -1
View File
@@ -11,7 +11,7 @@ import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { attentionSignal, barNextSteps, continueKeys, defaultRightFraction, folderName, pickDefaultAgent, taskbarSwitch, windowTitleFor, deadConsoleAction, loopBanner, loopLocks, loopStopLabels, loopWindowTitle, CONSOLE_ASIDE_BREAKPOINT_PX, NEXT_STEP_LABELS } from "../plan2code-app/ui/meter.js";
import { attentionSignal, barNextSteps, continueKeys, defaultRightFraction, folderName, pickDefaultAgent, taskbarSwitch, windowTitleFor, deadConsoleAction, loopBanner, loopRunText, loopLocks, loopStopLabels, loopWindowTitle, CONSOLE_ASIDE_BREAKPOINT_PX, NEXT_STEP_LABELS } from "../plan2code-app/ui/meter.js";
import { findBrowser, launchBrowser } from "./browser.mjs";
const HERE = path.dirname(fileURLToPath(import.meta.url));
@@ -339,6 +339,18 @@ test("the loop banner says what the loop is doing at every moment", () => {
assert.deepEqual(said({ phase: 0, phases: 0 }).pips, []);
});
test("the loop banner names the strength and model a phase runs on, and nothing when the toggle is off", () => {
const base = { phase: 3, phases: 5, phaseName: "API", step: "Build" };
const said = (extra) => loopBanner({ ...base, ...extra }).detail;
assert.equal(said({ strength: "thrifty", model: "sonnet", effort: "low" }), "Phase 3 of 5: API \u00b7 Build \u00b7 Thrifty (sonnet, low)");
assert.equal(said({ strength: "best", model: "opus" }), "Phase 3 of 5: API \u00b7 Build \u00b7 Best (opus)");
assert.equal(said({ strength: "balanced" }), "Phase 3 of 5: API \u00b7 Build \u00b7 Balanced");
assert.equal(said({ strength: null, model: "opus", effort: "high" }), "Phase 3 of 5: API \u00b7 Build \u00b7 (opus, high)", "Match my session shows the launch model");
assert.equal(said({ strength: null, model: null, effort: null }), "Phase 3 of 5: API \u00b7 Build", "toggle off or no model known: unchanged");
assert.equal(said({}), "Phase 3 of 5: API \u00b7 Build");
assert.equal(loopRunText({}), "");
});
test("Stop now turns the banner to Stopping, ahead of a question or a stop on its way", () => {
const base = { phase: 2, phases: 4, phaseName: "API", step: "Build", stopAfterPhase: false, stopping: true };
for (const extra of [{}, { step: "Waiting on you" }, { stopAfterPhase: true }, { step: "Starting" }]) {
+20
View File
@@ -334,6 +334,26 @@ test("launcher: mergeModels keeps the shipped list first and drops user duplicat
assert.deepEqual(modelEntries({ devin: [{ id: 1 }, null, { id: "ok", label: "Ok" }] }, "devin"), [{ id: "ok", label: "Ok" }]);
});
test("launcher: a strengths key in the models files is never read as a model list, and every shipped strength names a listed model", () => {
const { modelEntries, mergeModels } = require(LAUNCHER);
const shipped = JSON.parse(fs.readFileSync(path.join(path.dirname(LAUNCHER), "models.json"), "utf8"));
assert.ok(shipped.strengths && typeof shipped.strengths === "object");
assert.deepEqual(modelEntries(shipped, "strengths"), []);
const claude = modelEntries(shipped, "claude");
assert.ok(claude.length > 0);
const user = { claude: [{ id: "mine", label: "Mine" }], strengths: { claude: { best: { model: "mine" } } } };
assert.deepEqual(modelEntries(user, "strengths"), []);
assert.deepEqual(mergeModels(claude, modelEntries(user, "claude")), [...claude, { id: "mine", label: "Mine" }]);
assert.deepEqual(mergeModels(claude, modelEntries({ claude: user.claude }, "claude")), mergeModels(claude, modelEntries(user, "claude")));
// Only Claude Code and Devin ship a mapping, and every row names a model in that agent's list.
assert.deepEqual(Object.keys(shipped.strengths).sort(), ["claude", "devin"]);
for (const [agent, rows] of Object.entries(shipped.strengths)) {
assert.deepEqual(Object.keys(rows), ["thrifty", "balanced", "best"], agent);
const ids = new Set(shipped[agent].map((m) => m.id));
for (const row of Object.values(rows)) assert.ok(ids.has(row.model), `${agent}: ${row.model}`);
}
});
test("launcher: the shipped models.json is curated Devin at low/medium/high only", () => {
const shipped = JSON.parse(fs.readFileSync(path.join(path.dirname(LAUNCHER), "models.json"), "utf8"));
assert.ok(shipped.claude.length > 0);
+83 -1
View File
@@ -536,7 +536,7 @@ test("models: the user's additions round-trip, XHigh and Max are refused, and th
assert.equal((await post("/models", { nope: true })).status, 400);
assert.equal((await get()).user.devin.length, 1, "a refused write must not touch what is saved");
// Codex has no curated list, but the launcher reads a user `codex` list all the same.
// The launcher reads a user `codex` list too. The shipped file this test serves has no `codex` key, so that list is empty here.
const codex = await post("/models", { models: { codex: [{ id: " gpt-5-codex ", label: "GPT-5 Codex" }, { id: "gpt-5-codex" }] } });
assert.equal(codex.status, 200);
assert.deepEqual((await get()).user, { claude: [], devin: [], codex: [{ id: "gpt-5-codex", label: "GPT-5 Codex" }] });
@@ -547,6 +547,88 @@ test("models: the user's additions round-trip, XHigh and Max are refused, and th
assert.deepEqual((await get()).user, { claude: [], devin: [], codex: [] });
});
test("models: strengths come back shipped and user, round-trip, survive a models-only save, and bad rows name their field", async () => {
fs.mkdirSync(path.join(MODELS_DIR, "bin"), { recursive: true });
const shippedStrengths = { claude: { thrifty: { model: "sonnet", effort: "low" }, best: { model: "opus" } } };
fs.writeFileSync(
path.join(MODELS_DIR, "bin", "models.json"),
JSON.stringify({ claude: [{ id: "opus", label: "Opus" }, { id: "sonnet", label: "Sonnet" }], strengths: shippedStrengths })
);
const get = async () => (await fetch(baseUrl + "/models", { headers: { cookie: cookieFor(baseUrl, token) } })).json();
const userFile = () => JSON.parse(fs.readFileSync(path.join(MODELS_DIR, "models.json"), "utf8"));
const first = await get();
assert.deepEqual(first.strengths, { shipped: shippedStrengths, user: {} }, "shipped rows come back, no user rows yet");
assert.ok(!("strengths" in first.shipped) && !("strengths" in first.user), "strengths is never read as an agent's model list");
const mapping = { claude: { balanced: { model: " sonnet ", effort: "medium" }, best: { model: "opus" } }, codex: { thrifty: { model: "gpt-5-codex" } } };
const saved = await post("/models", { models: { claude: [{ id: "mine" }] }, strengths: mapping });
assert.equal(saved.status, 200);
const want = { claude: { balanced: { model: "sonnet", effort: "medium" }, best: { model: "opus" } }, codex: { thrifty: { model: "gpt-5-codex" } } };
assert.deepEqual((await get()).strengths.user, want);
assert.deepEqual(userFile().strengths, want, "the launcher's file carries the mapping beside the lists");
assert.equal(userFile().claude.length, 1);
// A save with only a Models list never wipes the mapping.
assert.equal((await post("/models", { models: { claude: [{ id: "mine" }, { id: "other" }] } })).status, 200);
assert.deepEqual((await get()).strengths.user, want);
assert.equal((await get()).user.claude.length, 2);
// The agents' own lists and the reset keep it too; an explicit empty mapping clears it.
assert.equal((await post("/models", { models: {} })).status, 200);
assert.deepEqual((await get()).strengths.user, want);
const refused = [
[{ claude: { thrifty: { model: "bad id!" } } }, /strengths\.claude\.thrifty\.model/],
[{ claude: { cheap: { model: "sonnet" } } }, /strengths\.claude\.cheap/],
[{ claude: { best: { model: "opus", effort: "max" } } }, /strengths\.claude\.best\.effort/],
[{ nobody: { best: { model: "opus" } } }, /strengths\.nobody/],
[{ claude: { best: "opus" } }, /strengths\.claude\.best/],
[["claude"], /strengths/],
];
for (const [body, field] of refused) {
const res = await post("/models", { models: {}, strengths: body });
assert.equal(res.status, 400, JSON.stringify(body));
assert.match((await res.json()).error, field);
}
assert.deepEqual((await get()).strengths.user, want, "a refused write must not touch what is saved");
assert.equal((await post("/models", { models: {}, strengths: {} })).status, 200);
assert.deepEqual((await get()).strengths.user, {});
assert.ok(!("strengths" in userFile()), "an empty mapping leaves no key behind");
});
test("models: a saved mapping for an agent that is gone is skipped, and never hides or wipes the others", async (t) => {
const userFile = path.join(MODELS_DIR, "models.json");
t.after(() => fs.rmSync(userFile, { force: true }));
fs.writeFileSync(
userFile,
JSON.stringify({ claude: [], strengths: { claude: { best: { model: "opus" } }, "deleted-agent": { best: { model: "x" } }, devin: { cheap: { model: "y" } } } })
);
const get = async () => (await fetch(baseUrl + "/models", { headers: { cookie: cookieFor(baseUrl, token) } })).json();
assert.deepEqual((await get()).strengths.user, { claude: { best: { model: "opus" } } });
assert.equal((await post("/models", { models: { claude: [{ id: "mine" }] } })).status, 200);
assert.deepEqual(JSON.parse(fs.readFileSync(userFile, "utf8")).strengths, { claude: { best: { model: "opus" } } }, "the models-only save kept the good rows");
});
test("models: one bad saved row never drops the other rows of its agent, like the app", async (t) => {
const userFile = path.join(MODELS_DIR, "models.json");
t.after(() => fs.rmSync(userFile, { force: true }));
fs.writeFileSync(
userFile,
JSON.stringify({
claude: [],
strengths: { claude: { thrifty: { model: "haiku", effort: "max" }, balanced: { model: "bad id!" }, cheap: { model: "x" }, best: { model: "opus", effort: "high" } } },
})
);
const get = async () => (await fetch(baseUrl + "/models", { headers: { cookie: cookieFor(baseUrl, token) } })).json();
assert.deepEqual(
(await get()).strengths.user,
{ claude: { thrifty: { model: "haiku" }, best: { model: "opus", effort: "high" } } },
"a bad effort is dropped and its row stays; a bad model or word drops only that row"
);
});
test("agents: no file is an empty list, a save is written whole and keeps unknown keys, and a bad save names its entry and field", async (t) => {
const file = path.join(MODELS_DIR, "agents.json");
t.after(() => fs.rmSync(file, { force: true }));
+176 -3
View File
@@ -18,7 +18,7 @@ import path from "node:path";
import { spawnSync } from "node:child_process";
import { pathToFileURL } from "node:url";
import { applyPatch, blankState, validate, switchWorkflow } from "../src/web-console/patches.mjs";
import { ROOT, CONSOLE_CLI, ENV, HOME, cli, readState, sessionUrl, payload } from "./web-console-test-helpers.mjs";
import { ROOT, CONSOLE_CLI, ENV, HOME, MODELS_DIR, cli, readState, sessionUrl, payload } from "./web-console-test-helpers.mjs";
const APP_VARS = ["PLAN2CODE_IN_APP", "PLAN2CODE_EMBED_ORIGIN", "PLAN2CODE_APP_ANNOUNCE", "PLAN2CODE_APP_TOKEN", "PLAN2CODE_CONSOLE_SID"];
@@ -472,10 +472,16 @@ test("loopEnd: a valid stopped with an evidence doc in the same patch is stored"
assert.equal(replaced.next.loopEnd.line, "A later word replaces the earlier");
});
test("loopEnd: ask is accepted in can, so a stopped loop can offer a question before anything runs", () => {
const { problems, next } = tryPatch(loopState(), { loopEnd: goodEnd({ can: ["resume", "ask", "fix", "dashboard"] }) });
assert.deepEqual(problems, []);
assert.deepEqual(next.loopEnd.can, ["resume", "ask", "fix", "dashboard"]);
});
test("loopEnd: an unknown evidence id, a push in can, and the other bad fields are refused, each named", () => {
const cases = [
["an unknown evidence id", goodEnd({ evidence: "nope" }), /loopEnd\.evidence must be the id of a doc on the page/],
["a push in can", goodEnd({ can: ["resume", "push"] }), /loopEnd\.can must be an array with entries from resume, fix, finalize, dashboard/],
["a push in can", goodEnd({ can: ["resume", "push"] }), /loopEnd\.can must be an array with entries from resume, ask, fix, finalize, dashboard/],
["a bad kind", goodEnd({ kind: "paused" }), /loopEnd\.kind must be "stopped" or "finished"/],
["a two-line line", goodEnd({ line: "one\ntwo" }), /loopEnd\.line must be one line/],
["an absolute uncommitted path", goodEnd({ uncommitted: ["/etc/passwd"] }), /loopEnd\.uncommitted must be an array of relative paths/],
@@ -772,7 +778,7 @@ test("loop-setup: GET /loop-setup lists the phases left for a spec and refuses a
}
// Saving through the route, then reading it back in a new request.
const save = await send({ review: true, fixMode: "smart", fix: ["critical"], commit: true, onQuestion: "stop", afterLast: "offer-finalize", quickCheck: "", notifyKind: "query", notifyUrl: "https://example.com/hook", notifyParam: "msg", retryOnce: true, silenceMinutes: 30 });
const save = await send({ review: true, fixMode: "smart", fix: ["critical"], commit: true, onQuestion: "stop", afterLast: "offer-finalize", quickCheck: "", notifyKind: "query", notifyUrl: "https://example.com/hook", notifyParam: "msg", retryOnce: true, silenceMinutes: 30, phaseStrength: false, strengths: {} });
assert.equal(save.status, 200, await save.clone().text());
const after = await get("?spec=specs/demo");
assert.equal(after.body.remembered, true);
@@ -789,3 +795,170 @@ test("loop-setup: GET /loop-setup lists the phases left for a spec and refuses a
cli(["stop", "--session", opened.json.sid, "--force"], { cwd, expectFail: true });
}
});
/* ------------------------------ phase strength: the setup's new fields */
const specWith = (project, name, phases) => {
const dir = path.join(project, "specs", name);
fs.mkdirSync(dir, { recursive: true });
const rows = phases.map(([mark, n, title]) => `- [${mark}] **Phase ${n}:** ${title} - [phase-${n}.md](phase-${n}.md)`);
fs.writeFileSync(path.join(dir, "overview.md"), ["# Demo", "", "## Phase Checklist", "", ...rows, ""].join("\n"));
};
test("loop-setup: the strength fields default off, and their bad values are named", () => {
fs.rmSync(LOOP_SETUP_PATH, { force: true });
const cases = [
["phaseStrength as a string", { phaseStrength: "yes" }, /"phaseStrength" must be true or false/],
["strengths as a list", { strengths: [] }, /"strengths" must be an object/],
["a key outside specs/", { strengths: { "../x": [] } }, /must be a folder directly under specs\//],
["a hidden spec folder", { strengths: { "specs/.hidden": [] } }, /must be a folder directly under specs\//],
["a nested spec key", { strengths: { "specs/a/b": [] } }, /must be a folder directly under specs\//],
["picks as an object", { strengths: { "specs/a": {} } }, /at most 100 phase picks/],
["101 picks", { strengths: { "specs/a": Array.from({ length: 101 }, (_, i) => ({ n: i + 1, name: "p", strength: "best" })) } }, /at most 100 phase picks/],
["n of 0", { strengths: { "specs/a": [{ n: 0, name: "p", strength: "best" }] } }, /positive whole number/],
["n as text", { strengths: { "specs/a": [{ n: "2", name: "p", strength: "best" }] } }, /positive whole number/],
["a repeated n", { strengths: { "specs/a": [{ n: 2, name: "p", strength: "best" }, { n: 2, name: "q", strength: "best" }] } }, /Phase 2 twice/],
["a 201-character name", { strengths: { "specs/a": [{ n: 1, name: "x".repeat(201), strength: "best" }] } }, /at most 200 characters/],
["an unknown word", { strengths: { "specs/a": [{ n: 1, name: "p", strength: "max" }] } }, /strength of thrifty, balanced, best/],
];
const got = inStore(
(m, { project, list }) =>
list.map(([what, change]) => {
try {
m.saveLoopSetup(project, { ...m.LOOP_DEFAULTS, fix: [...m.LOOP_DEFAULTS.fix], ...change });
return { what, threw: false };
} catch (e) {
return { what, threw: true, message: e.message };
}
}),
{ project: projectA, list: cases.map(([what, change]) => [what, change]) }
);
for (const [i, [what, , expected]] of cases.entries()) {
assert.equal(got[i].threw, true, `${what} is refused`);
assert.match(got[i].message, expected, what);
}
assert.equal(fs.existsSync(LOOP_SETUP_PATH), false, "a refused save writes nothing");
const fresh = inStore((m, { project }) => m.readLoopSetup(project), { project: projectA });
assert.equal(fresh.phaseStrength, false);
assert.deepEqual(fresh.strengths, {});
});
test("loop-setup: an entry saved before the strength fields reads as off with no picks", () => {
fs.rmSync(LOOP_SETUP_PATH, { force: true });
const project = path.join(HOME, "store-project-old");
inStore((m, { p }) => (m.saveLoopSetup(p, { ...m.LOOP_DEFAULTS, fix: ["warning"] }), null), { p: project });
const store = JSON.parse(fs.readFileSync(LOOP_SETUP_PATH, "utf8"));
const [folded] = Object.keys(store.projects);
delete store.projects[folded].phaseStrength;
delete store.projects[folded].strengths;
fs.writeFileSync(LOOP_SETUP_PATH, JSON.stringify(store));
const got = inStore((m, { p }) => m.readLoopSetup(p), { p: project });
assert.equal(got.remembered, true);
assert.equal(got.phaseStrength, false);
assert.deepEqual(got.strengths, {});
assert.deepEqual(got.fix, ["warning"]);
});
test("loop-setup: picks for a removed spec folder or a renamed phase are pruned on read and on save", () => {
fs.rmSync(LOOP_SETUP_PATH, { force: true });
const project = fs.mkdtempSync(path.join(HOME, "prune-"));
specWith(project, "keep", [["x", 1, "Done"], [" ", 2, "Build it"], [" ", 3, "Ship it"]]);
specWith(project, "gone", [[" ", 1, "Only"]]);
const picks = {
"specs/keep": [
{ n: 2, name: "Build it", strength: "thrifty" },
{ n: 3, name: "Ship it renamed", strength: "best" },
{ n: 1, name: "Done", strength: "best" },
],
"specs/gone": [{ n: 1, name: "Only", strength: "best" }],
"specs/never": [{ n: 1, name: "x", strength: "best" }],
};
const out = inStore(
async (m, { project, picks }) => {
const fs = await import("node:fs");
const path = await import("node:path");
const left = (spec) => {
const text = fs.readFileSync(path.join(project, spec, "overview.md"), "utf8");
return [...text.matchAll(/^- \[ \] \*\*Phase (\d+):\*\* (.+?) - /gm)].map((x) => ({ n: Number(x[1]), name: x[2] }));
};
fs.rmSync(path.join(project, "specs", "gone"), { recursive: true });
const saved = m.saveLoopSetup(project, { ...m.LOOP_DEFAULTS, fix: [...m.LOOP_DEFAULTS.fix], phaseStrength: true, strengths: picks }, left);
const onDisk = JSON.parse(fs.readFileSync(m.LOOP_SETUP_FILE, "utf8"));
const raw = Object.values(onDisk.projects)[0].strengths;
// A later read, with the overview changed again, drops the next pick too.
fs.writeFileSync(path.join(project, "specs", "keep", "overview.md"), "# Demo\n\n## Phase Checklist\n\n- [ ] **Phase 3:** Ship it - [phase-3.md](phase-3.md)\n");
const later = m.readLoopSetup(project, left);
return { saved: saved.strengths, raw, later: later.strengths };
},
{ project, picks }
);
const kept = [{ n: 2, name: "Build it", strength: "thrifty" }];
assert.deepEqual(out.saved, { "specs/keep": kept }, "the removed folder, the never-made one, the renamed phase and the finished phase are dropped");
assert.deepEqual(out.raw, { "specs/keep": kept }, "the file holds the pruned copy");
assert.deepEqual(out.later, {}, "a later read prunes what the overview no longer lists");
});
test("loop-setup: the picks live only in loop-setup.json, never under specs/", () => {
fs.rmSync(LOOP_SETUP_PATH, { force: true });
const project = fs.mkdtempSync(path.join(HOME, "nowrite-"));
specWith(project, "demo", [[" ", 1, "One"]]);
const before = fs.readFileSync(path.join(project, "specs", "demo", "overview.md"), "utf8");
inStore(
(m, { project }) => (m.saveLoopSetup(project, { ...m.LOOP_DEFAULTS, fix: [...m.LOOP_DEFAULTS.fix], phaseStrength: true, strengths: { "specs/demo": [{ n: 1, name: "One", strength: "best" }] } }), null),
{ project }
);
assert.ok(fs.existsSync(LOOP_SETUP_PATH));
assert.deepEqual(fs.readdirSync(path.join(project, "specs", "demo")), ["overview.md"]);
assert.equal(fs.readFileSync(path.join(project, "specs", "demo", "overview.md"), "utf8"), before);
assert.deepEqual(fs.readdirSync(project), ["specs"], "nothing else appears in the project");
});
test("GET /loop-setup?agent= answers the loop agent's resolved strengths and this spec's picks", async () => {
fs.rmSync(LOOP_SETUP_PATH, { force: true });
const cwd = tempProject("route-strength");
specWith(cwd, "demo", [[" ", 1, "One"], [" ", 2, "Two"]]);
// The installed copy of the shipped model menu, with its strengths, as install.js drops it.
const shipped = path.join(MODELS_DIR, "bin", "models.json");
const hadShipped = fs.existsSync(shipped) ? fs.readFileSync(shipped) : null;
fs.mkdirSync(path.dirname(shipped), { recursive: true });
fs.copyFileSync(path.join(ROOT, "src", "launcher", "models.json"), shipped);
const opened = run(["open", "--workflow", "implement", "--no-open"], { cwd });
assert.equal(opened.status, 0, opened.stderr);
const { base, cookie } = sessionUrl(opened.json.url);
const get = async (query) => {
const res = await fetch(`${base}/loop-setup${query}`, { headers: { cookie, connection: "close" } });
return { status: res.status, body: await res.json() };
};
try {
const none = await get("?spec=specs/demo");
assert.equal(none.body.resolved, null, "no agent asked, none answered");
assert.deepEqual(none.body.picks, []);
assert.equal(none.body.phaseStrength, false);
const claude = await get("?spec=specs/demo&agent=claude");
assert.equal(claude.status, 200);
assert.deepEqual(Object.keys(claude.body.resolved).sort(), ["balanced", "best", "thrifty"], "the shipped Claude Code rows");
for (const word of ["thrifty", "balanced", "best"]) assert.equal(typeof claude.body.resolved[word].model, "string");
const codex = await get("?spec=specs/demo&agent=codex");
assert.deepEqual(codex.body.resolved, {}, "codex has nothing mapped");
for (const agent of ["nobody", "..%2Fx", ""]) assert.equal((await get(`?spec=specs/demo&agent=${agent}`)).body.resolved, null, `agent=${agent}`);
// A save with picks reads back for its own spec; a pick for a phase that is not there is pruned.
const entry = {
review: true, fixMode: "smart", fix: ["critical"], commit: true, onQuestion: "guess", afterLast: "offer-finalize", quickCheck: "",
notifyKind: "query", notifyUrl: "", notifyParam: "message", retryOnce: true, silenceMinutes: 20, phaseStrength: true,
strengths: { "specs/demo": [{ n: 2, name: "Two", strength: "best" }, { n: 9, name: "Gone", strength: "best" }] },
};
const save = await fetch(`${base}/loop-setup`, { method: "POST", headers: { "content-type": "application/json", cookie, origin: base, connection: "close" }, body: JSON.stringify(entry) });
assert.equal(save.status, 200, await save.clone().text());
const back = await get("?spec=specs/demo&agent=claude");
assert.equal(back.body.phaseStrength, true);
assert.deepEqual(back.body.picks, [{ n: 2, name: "Two", strength: "best" }]);
} finally {
if (hadShipped) fs.writeFileSync(shipped, hadShipped);
else fs.rmSync(shipped, { force: true });
cli(["stop", "--session", opened.json.sid, "--force"], { cwd, expectFail: true });
}
});
+195
View File
@@ -5,7 +5,11 @@
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { pageSuite, openSession, tempProject, textOf } from "./page-test-helpers.mjs";
import { MODELS_DIR } from "./web-console-test-helpers.mjs";
const suite = pageSuite(test);
@@ -46,6 +50,7 @@ test("the setup lists each phase on its own line, in the agreed order of fields
const labels = await s.page.locator(".loop-label").allInnerTexts();
assert.deepEqual(labels, [
"Strength per phase",
"Review",
"Fix findings",
"When a phase asks something",
@@ -256,6 +261,27 @@ test("the finish card counts each kind of decision per phase and opens on the fu
assert.deepEqual(seen.actions, ["Finalize", "Back to the dashboard"], "the way home is offered next to Finalize");
});
test("a stopped loop offers Ask a question, which tells the app and wakes no loop", async (t) => {
const s = await openSetup(t);
await frameInFakeApp(s);
const seen = await s.page.evaluate(async () => {
const m = await import(new URL("screens/ending.js", document.baseURI).href);
const card = m.renderLoopEnd({
kind: "stopped",
line: "Phase 5 stopped",
phases: [{ n: 5, name: "Docs", commit: null, unfixed: 0, retries: 0 }],
uncommitted: [],
can: ["ask", "resume", "fix", "dashboard"],
});
const buttons = [...card.querySelectorAll(".loop-end-actions button")];
const ask = buttons.find((b) => b.textContent === "Ask a question");
if (ask) ask.click();
return { labels: buttons.map((b) => b.textContent), told: window.__told.map((x) => x.plan2code) };
});
assert.deepEqual(seen.labels, ["Ask a question", "Resume the loop", "Fix it by hand", "Back to the dashboard"]);
assert.deepEqual(seen.told, ["loop-ask"]);
});
// A real loop session: `open --loop` under the app's signal stamps the lock.
async function openLoopPage(t) {
return openSession(await suite.browser(), {
@@ -329,3 +355,172 @@ test("a loop phase still starting says the loop is starting it, and promises no
await s.page.getByRole("heading", { name: "The loop is starting this phase" }).waitFor();
assert.doesNotMatch(await textOf(s.page.locator("#main")), /pause for approvals/);
});
/* ------------------------------------------- Strength per phase (the section) */
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const LAUNCH = { agent: "claude", agentLabel: "Claude Code", model: "opus", effort: "medium", permissionMode: "bypass", promptSupport: true, running: false };
// The installed copy of the shipped model menu, with its strengths, as install.js drops it.
function installShippedModels() {
const shipped = path.join(MODELS_DIR, "bin", "models.json");
fs.mkdirSync(path.dirname(shipped), { recursive: true });
fs.copyFileSync(path.join(ROOT, "src", "launcher", "models.json"), shipped);
}
// The setup screen with the app's answer in: this launch choice, then the strengths read for it.
async function openStrengthSetup(t, launch = LAUNCH) {
installShippedModels();
const s = await openSetup(t);
await s.page.evaluate(async (facts) => {
const { page, redraw } = await import(new URL("state.js", document.baseURI).href);
page.loopFacts = facts;
redraw();
}, launch);
return s;
}
const strengthRows = (s) =>
s.page.locator(".loop-strength-row").evaluateAll((rows) =>
rows.map((r) => ({ phase: r.querySelector(".loop-strength-phase").textContent, pick: r.querySelector("select").value, model: r.querySelector(".loop-strength-model").textContent }))
);
test("the strength section is first in the form and off by default", async (t) => {
const s = await openStrengthSetup(t);
const toggle = s.page.locator("#loop-phase-strength");
await toggle.waitFor();
assert.equal(await toggle.isChecked(), false);
assert.equal(await s.page.locator(".loop-label").first().innerText(), "Strength per phase");
assert.equal(await s.page.locator(".loop-strength-row").count(), 0, "no rows until the toggle is on");
assert.equal(await s.page.locator("#loop-strength-all").count(), 0);
});
test("turning it on shows one row per phase left with the model each pick becomes, and Set all to fills them", async (t) => {
const s = await openStrengthSetup(t);
await s.page.locator("#loop-phase-strength").check();
await s.page.locator(".loop-strength-row").first().waitFor();
assert.deepEqual(await strengthRows(s), [
{ phase: "Phase 1: One", pick: "", model: "opus, medium" },
{ phase: "Phase 2: Two", pick: "", model: "opus, medium" },
], "untouched rows match the session and show the launch choice");
await s.page.locator("#loop-strength-1").selectOption("thrifty");
assert.equal(await textOf(s.page.locator('[data-phase="1"] .loop-strength-model')), "sonnet, low");
assert.equal(await s.page.locator("#loop-strength-all").inputValue(), "-", "Set all to starts blank");
await s.page.locator("#loop-strength-all").selectOption("best");
assert.deepEqual((await strengthRows(s)).map((r) => [r.pick, r.model]), [["best", "opus, medium"], ["best", "opus, medium"]]);
assert.equal(await s.page.locator("#loop-strength-all").inputValue(), "-", "and goes blank again: it is not a saved value");
await s.page.locator("#loop-strength-2").selectOption("balanced");
assert.deepEqual((await strengthRows(s)).map((r) => [r.pick, r.model]), [["best", "opus, medium"], ["balanced", "sonnet, medium"]], "a row can change after Set all");
await s.page.locator("#loop-strength-all").selectOption("match");
assert.deepEqual((await strengthRows(s)).map((r) => r.pick), ["", ""], "Match my session fills every row too");
});
test("an agent with nothing mapped grays the section and links to Agents & models", async (t) => {
const s = await openStrengthSetup(t, { ...LAUNCH, agent: "codex", agentLabel: "Codex" });
const note = s.page.locator(".loop-strength-none");
await note.waitFor();
assert.match(await textOf(note), /This agent has no strengths mapped yet\./);
assert.equal(await s.page.locator("#loop-phase-strength").isDisabled(), true);
assert.equal(await s.page.locator(".loop-strength-row").count(), 0);
await note.getByRole("button").click();
await s.page.locator("#prefs-tab-agents[aria-selected='true']").waitFor();
});
test("a saved pick that is not mapped holds Start with a line naming the row, and a change releases it", async (t) => {
const s = await openStrengthSetup(t);
await s.page.locator("#loop-phase-strength").check();
await s.page.locator(".loop-strength-row").first().waitFor();
// Thrifty is no longer mapped for this agent (an override went away); Phase 2 still holds it.
await s.page.evaluate(async () => {
const { page, redraw } = await import(new URL("state.js", document.baseURI).href);
page.loopSetup.facts.resolved = { best: { model: "opus", effort: "medium" } };
page.loopSetup.form.strengths["specs/ready"] = [{ n: 2, name: "Two", strength: "thrifty" }];
redraw();
});
const why = s.page.locator(".loop-why");
assert.equal(await textOf(why), "Pick a new strength for Phase 2: not mapped for this agent.");
assert.equal(await s.page.locator("#loop-start").isDisabled(), true);
const pick = s.page.locator("#loop-strength-2");
assert.equal(await pick.inputValue(), "thrifty", "the saved word stays in the row");
assert.equal(await pick.locator("option:checked").innerText(), "Thrifty (not mapped)");
assert.equal(await pick.locator("option:checked").evaluate((o) => o.disabled), true);
await pick.selectOption("best");
assert.equal(await s.page.locator(".loop-why").count(), 0, "the hold is gone");
assert.equal(await s.page.locator("#loop-start").isDisabled(), false);
});
test("startBlock names every unmapped row, ignores the picks while the toggle is off, and waits for the strengths", async (t) => {
const s = await openSetup(t);
const lines = await s.page.evaluate(async () => {
const { startBlock } = await import(new URL("screens/loop-setup.js", document.baseURI).href);
const phasesLeft = [{ n: 2, name: "A" }, { n: 3, name: "B" }, { n: 4, name: "C" }];
const strengths = { "specs/x": [{ n: 2, name: "A", strength: "thrifty" }, { n: 3, name: "B", strength: "best" }, { n: 4, name: "C", strength: "thrifty" }] };
const facts = (resolved) => ({ spec: "specs/x", phasesLeft, pending: 0, resolved });
const form = (phaseStrength) => ({ commit: false, phaseStrength, strengths });
const app = { permissionMode: "bypass", promptSupport: true };
return {
two: startBlock(form(true), facts({ best: { model: "m" } }), app),
off: startBlock(form(false), facts({ best: { model: "m" } }), app),
allMapped: startBlock(form(true), facts({ thrifty: { model: "s" }, best: { model: "m" } }), app),
nothingMapped: startBlock(form(true), facts({}), app),
loading: startBlock(form(true), facts(undefined), app),
last: startBlock(form(true), { ...facts({ best: { model: "m" } }), phasesLeft: [] }, app),
};
});
assert.equal(lines.two, "Pick a new strength for Phase 2, Phase 4: not mapped for this agent.");
assert.equal(lines.off, "");
assert.equal(lines.allMapped, "");
assert.equal(lines.nothingMapped, "", "an agent with nothing mapped grays the section instead");
assert.equal(lines.loading, "Asking the app...");
assert.equal(lines.last, "Nothing left to build in this spec.", "the older checks keep their order");
});
test("the toggle and the picks survive reopening the screen, and the toggle off sends nothing for the loop", async (t) => {
const s = await openStrengthSetup(t);
await s.page.locator("#loop-phase-strength").check();
await s.page.locator("#loop-strength-1").selectOption("thrifty");
await s.page.locator("#loop-strength-2").selectOption("best");
const save = () =>
s.page.evaluate(async () => {
const { page } = await import(new URL("state.js", document.baseURI).href);
const { setupBody } = await import(new URL("screens/loop-setup.js", document.baseURI).href);
const res = await fetch("/loop-setup", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(setupBody(page.loopSetup.form, page.loopSetup)) });
return { status: res.status, body: await res.json() };
});
const saved = await save();
assert.equal(saved.status, 200, JSON.stringify(saved.body));
assert.equal(saved.body.setup.phaseStrength, true);
assert.deepEqual(saved.body.setup.strengths, { "specs/ready": [{ n: 1, name: "One", strength: "thrifty" }, { n: 2, name: "Two", strength: "best" }] });
const reopen = async () => {
await s.page.evaluate(async () => {
const m = await import(new URL("screens/loop-setup.js", document.baseURI).href);
const { page, redraw } = await import(new URL("state.js", document.baseURI).href);
await m.openLoopSetup({ dir: "specs/ready" });
redraw();
});
await s.page.locator("#loop-start").waitFor();
};
await reopen();
await s.page.locator("#loop-strength-1").waitFor();
assert.equal(await s.page.locator("#loop-phase-strength").isChecked(), true);
assert.deepEqual((await strengthRows(s)).map((r) => [r.pick, r.model]), [["thrifty", "sonnet, low"], ["best", "opus, medium"]]);
// Off: the picks stay saved, the loop is told nothing.
await s.page.locator("#loop-phase-strength").uncheck();
const off = await save();
assert.equal(off.body.setup.phaseStrength, false);
assert.equal(off.body.setup.strengths["specs/ready"].length, 2, "the toggle hides the picks, it does not erase them");
// Back to Match my session for every row removes the key.
await s.page.locator("#loop-phase-strength").check();
await s.page.locator("#loop-strength-all").selectOption("match");
const cleared = await save();
assert.deepEqual(cleared.body.setup.strengths, {});
});
+193
View File
@@ -162,6 +162,199 @@ test("Agents & models: the table lists Claude, Devin and Codex, and a model id t
assert.equal(await formField(s, "Model id").inputValue(), "gpt-test-codex-2");
});
// The "Strengths in the loop" fold: Thrifty, Balanced and Best mapped to the
// agent's own models. models.json under PLAN2CODE_MODELS_DIR is what Save writes.
const MODELS_FILE = path.join(MODELS_DIR, "models.json");
const strengthsFold = (s) => agentForm(s).locator("details.prefs-agents-strengths");
const strengthRow = (s, word) => strengthsFold(s).locator(`.strength-row[data-word="${word}"]`);
const savedStrengths = () => {
try {
return JSON.parse(fs.readFileSync(MODELS_FILE, "utf8")).strengths ?? null;
} catch {
return null; // not written yet
}
};
// The shipped list the tests start with has no models at all; the fold needs some
// (and Claude and Devin's shipped strengths), so a test writes them and puts the file back.
const SHIPPED_FILE = path.join(MODELS_DIR, "bin", "models.json");
function seedShippedModels(t) {
fs.mkdirSync(path.dirname(SHIPPED_FILE), { recursive: true });
fs.writeFileSync(
SHIPPED_FILE,
JSON.stringify({
claude: [{ id: "sonnet", label: "Sonnet" }, { id: "opus", label: "Opus" }],
devin: [{ id: "swe-2-medium", label: "SWE 2 Medium" }, { id: "swe-2-high", label: "SWE 2 High" }, { id: "claude-opus-5-5-medium", label: "Opus 5.5 Medium" }],
codex: [],
strengths: {
claude: { thrifty: { model: "sonnet", effort: "low" }, balanced: { model: "sonnet", effort: "medium" }, best: { model: "opus", effort: "medium" } },
devin: { thrifty: { model: "swe-2-medium" }, balanced: { model: "swe-2-high" }, best: { model: "claude-opus-5-5-medium" } },
},
})
);
t.after(() => fs.rmSync(SHIPPED_FILE, { force: true }));
}
// Save lands asynchronously and an earlier Saved mark may still be on screen, so wait on the file.
async function savedStrengthsAre(expected) {
for (let i = 0; i < 100; i++) {
if (JSON.stringify(savedStrengths()) === JSON.stringify(expected)) return;
await new Promise((r) => setTimeout(r, 50));
}
assert.deepEqual(savedStrengths(), expected);
}
async function openStrengths(s, name) {
await s.page.getByRole("button", { name: `Edit ${name}` }).click();
await formPane(s).waitFor();
await strengthsFold(s).locator("summary").click();
}
test("Agents & models: Claude's fold shows the shipped rows as default, and only a changed row is saved, until Reset", async (t) => {
noSavedLooks(t);
seedAgents(t);
t.after(() => fs.rmSync(MODELS_FILE, { force: true }));
seedShippedModels(t);
fs.rmSync(MODELS_FILE, { force: true });
const s = await openSession(await suite.browser(), { t, workflow: "plan", payload: PLAN_QUESTION });
await openAgentsTab(s);
await openStrengths(s, "Claude");
assert.equal(await strengthRow(s, "thrifty").locator('select[data-field="model"]').inputValue(), "sonnet");
assert.equal(await strengthRow(s, "thrifty").locator('select[data-field="effort"]').inputValue(), "low");
assert.equal(await strengthRow(s, "best").locator('select[data-field="model"]').inputValue(), "opus");
assert.equal(await strengthsFold(s).locator(".strength-default").count(), 3, "all three rows are untouched defaults");
assert.equal(await textOf(strengthRow(s, "balanced").locator(".strength-default")), "default");
assert.equal(await strengthsFold(s).locator(".strength-reset").count(), 0, "nothing to reset yet");
assert.equal(await textOf(strengthsFold(s).locator(".strength-offer")), "The loop will offer: Thrifty, Balanced, Best");
await strengthRow(s, "thrifty").locator('select[data-field="model"]').selectOption("opus");
await agentForm(s).locator(".prefs-unsaved-badge").waitFor();
assert.equal(await strengthRow(s, "thrifty").locator(".strength-reset").count(), 1, "a changed row has Reset");
assert.equal(await strengthRow(s, "thrifty").locator(".strength-default").count(), 0);
assert.equal(await strengthRow(s, "balanced").locator(".strength-reset").count(), 0, "the other rows stay default");
assert.equal(fs.existsSync(MODELS_FILE), false, "nothing is saved before Save");
await formButton(s, "Save").click();
await agentForm(s).locator(".prefs-saved").filter({ hasText: /^Saved. Restart plan2code/ }).waitFor();
assert.deepEqual(savedStrengths(), { claude: { thrifty: { model: "opus", effort: "low" } } }, "only the changed row is written");
await strengthRow(s, "thrifty").locator(".strength-reset").click();
assert.equal(await strengthRow(s, "thrifty").locator('select[data-field="model"]').inputValue(), "sonnet", "Reset restores the shipped row");
assert.equal(await textOf(strengthRow(s, "thrifty").locator(".strength-default")), "default");
await agentForm(s).locator(".prefs-unsaved-badge").waitFor();
await formButton(s, "Save").click();
await savedStrengthsAre(null); // with every row back at its default, no strengths are stored
});
test("Agents & models: Codex has an effort select on each row and Devin does not", async (t) => {
noSavedLooks(t);
seedAgents(t);
t.after(() => fs.rmSync(MODELS_FILE, { force: true }));
seedShippedModels(t);
const s = await openSession(await suite.browser(), { t, workflow: "plan", payload: PLAN_QUESTION });
await openAgentsTab(s);
await openStrengths(s, "Codex");
assert.equal(await textOf(strengthsFold(s).locator(".strength-empty")), "Add a model under Models first", "no models ship for Codex");
await formButton(s, "Add a model").click();
await formField(s, "Model id").fill("gpt-test-codex-1");
assert.equal(await strengthsFold(s).locator('select[data-field="effort"]').count(), 3);
assert.equal(await strengthRow(s, "thrifty").locator('select[data-field="model"]').inputValue(), "", "nothing ships for Codex, so the row is not mapped");
assert.equal(await textOf(strengthsFold(s).locator(".strength-offer")), "The loop will not offer a strength for this agent until one is mapped.");
await formButton(s, "Discard").click(); // the added model is unsaved: leaving would ask first
await s.page.getByRole("button", { name: "Edit Devin" }).click();
await strengthsFold(s).locator("summary").click();
assert.equal(await strengthsFold(s).locator('select[data-field="model"]').count(), 3);
assert.equal(await strengthsFold(s).locator('select[data-field="effort"]').count(), 0, "Devin takes no separate effort");
});
test("Agents & models: a strength-only edit is unsaved until Save, Discard puts the shipped rows back, and Reset keeps focus on its row", async (t) => {
noSavedLooks(t);
seedAgents(t);
t.after(() => fs.rmSync(MODELS_FILE, { force: true }));
seedShippedModels(t);
const s = await openSession(await suite.browser(), { t, workflow: "plan", payload: PLAN_QUESTION });
await openAgentsTab(s);
await openStrengths(s, "Claude");
assert.equal(await agentForm(s).locator(".prefs-unsaved-badge").isVisible(), false);
await strengthRow(s, "best").locator('select[data-field="model"]').selectOption("sonnet");
await agentForm(s).locator(".prefs-unsaved-badge").waitFor();
await formButton(s, "Discard").click();
assert.equal(await agentForm(s).locator(".prefs-unsaved-badge").isVisible(), false, "Discard clears the strength-only change");
assert.equal(await strengthRow(s, "best").locator('select[data-field="model"]').inputValue(), "opus", "the shipped row is back");
assert.equal(fs.existsSync(MODELS_FILE), false, "nothing was written");
await strengthRow(s, "best").locator('select[data-field="model"]').selectOption("sonnet");
await strengthRow(s, "best").locator(".strength-reset").focus();
await strengthRow(s, "best").locator(".strength-reset").click();
assert.equal(
await s.page.evaluate(() => document.activeElement?.closest("[data-word]")?.dataset.word),
"best",
"focus stays in the row the Reset was in"
);
});
test("Agents & models: a saved row whose model was removed reads not mapped while other models remain, and Save drops it", async (t) => {
noSavedLooks(t);
seedAgents(t);
t.after(() => fs.rmSync(MODELS_FILE, { force: true }));
seedShippedModels(t);
fs.rmSync(MODELS_FILE, { force: true });
const s = await openSession(await suite.browser(), { t, workflow: "plan", payload: PLAN_QUESTION });
await openAgentsTab(s);
await s.page.click("#agents-add");
await formPane(s).waitFor();
await formField(s, "Name").fill("Gemini");
await formField(s, "Command").fill("node");
await agentForm(s).locator("details.agent-fold > summary", { hasText: /^Models$/ }).click();
await formButton(s, "Add a model").click();
await formField(s, "Model id").fill("model-a");
await formButton(s, "Add a model").click();
await formField(s, "Model id").nth(1).fill("model-b");
await strengthsFold(s).locator("summary").click();
await strengthRow(s, "thrifty").locator('select[data-field="model"]').selectOption("model-a");
await strengthRow(s, "best").locator('select[data-field="model"]').selectOption("model-b");
await formButton(s, "Save").click();
await savedStrengthsAre({ gemini: { thrifty: { model: "model-a" }, best: { model: "model-b" } } });
await agentForm(s).locator(".agent-row.model").first().getByRole("button", { name: "Remove" }).click();
assert.equal(await strengthRow(s, "thrifty").locator('select[data-field="model"]').inputValue(), "", "the removed model reads not mapped");
assert.equal(await textOf(strengthsFold(s).locator(".strength-offer")), "The loop will offer: Best");
await formButton(s, "Save").click();
await savedStrengthsAre({ gemini: { best: { model: "model-b" } } });
});
test("Agents & models: an added agent maps strengths only from its own Models, and a removed model reads not mapped", async (t) => {
noSavedLooks(t);
seedAgents(t);
t.after(() => fs.rmSync(MODELS_FILE, { force: true }));
seedShippedModels(t);
fs.rmSync(MODELS_FILE, { force: true });
const s = await openSession(await suite.browser(), { t, workflow: "plan", payload: PLAN_QUESTION });
await openAgentsTab(s);
await s.page.click("#agents-add");
await formPane(s).waitFor();
await formField(s, "Name").fill("Copilot");
await formField(s, "Command").fill("node");
await strengthsFold(s).locator("summary").click();
assert.equal(await textOf(strengthsFold(s).locator(".strength-empty")), "Add a model under Models first");
assert.equal(await strengthsFold(s).locator("select").count(), 0, "no selects without a model");
await agentForm(s).locator("details.agent-fold > summary", { hasText: /^Models$/ }).click();
await formButton(s, "Add a model").click();
await formField(s, "Model id").fill("gpt-test-1");
assert.equal(await strengthsFold(s).locator(".strength-empty").count(), 0, "the fold follows the Models list as it changes");
await strengthRow(s, "thrifty").locator('select[data-field="model"]').selectOption("gpt-test-1");
assert.equal(await strengthRow(s, "thrifty").locator('select[data-field="effort"]').count(), 0, "an added agent has no effort select");
await formButton(s, "Save").click();
await agentForm(s).locator(".prefs-saved").waitFor();
assert.deepEqual(savedStrengths(), { copilot: { thrifty: { model: "gpt-test-1" } } }, "saved under the new agent's id");
await agentForm(s).locator(".agent-row.model").first().getByRole("button", { name: "Remove" }).click();
assert.equal(await textOf(strengthsFold(s).locator(".strength-empty")), "Add a model under Models first");
await formButton(s, "Save").click();
await savedStrengthsAre(null);
});
test("Agents & models: a person adds an agent, tests its command, sees it after a reload, and deletes it", async (t) => {
noSavedLooks(t);
seedAgents(t);
+44
View File
@@ -0,0 +1,44 @@
// Tests for src/web-console/strengths.mjs against the shared cases in
// scripts/fixtures/strength-cases.json (the Rust twin in models.rs loads the
// same file). Run with: node --test scripts/test-web-console-strengths.mjs
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { STRENGTH_ORDER, nextStrength, resolveStrengths } from "../src/web-console/strengths.mjs";
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const FIXTURE = JSON.parse(fs.readFileSync(path.join(ROOT, "scripts", "fixtures", "strength-cases.json"), "utf8"));
test("the fixture is schema 1 and covers every kind of case", () => {
assert.equal(FIXTURE.schema, 1);
assert.ok(FIXTURE.cases.length >= 10);
const names = FIXTURE.cases.map((c) => c.name);
assert.equal(new Set(names).size, names.length, "case names are unique");
});
for (const c of FIXTURE.cases) {
test(`resolveStrengths: ${c.name}`, () => {
const resolved = resolveStrengths({ shipped: c.shipped, user: c.user, models: c.models, hasEffort: c.hasEffort });
assert.deepEqual(resolved, c.expect.resolved);
});
test(`nextStrength: ${c.name}`, () => {
for (const word of STRENGTH_ORDER) {
assert.equal(nextStrength(c.expect.resolved, word), c.expect.next[word], `next after ${word}`);
}
});
}
test("nextStrength: an unknown word or a missing mapping has no next", () => {
assert.equal(nextStrength({ best: { model: "x" } }, "cheap"), null);
assert.equal(nextStrength(null, "thrifty"), null);
assert.equal(nextStrength(undefined, "balanced"), null);
});
test("resolveStrengths: called with nothing at all resolves nothing", () => {
assert.deepEqual(resolveStrengths(), {});
assert.deepEqual(resolveStrengths({}), {});
});
+66 -1
View File
@@ -9,10 +9,15 @@ import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
import { defaultInstruction, offered, onLines, STRENGTH_LINES, STRENGTHS } from "../src/web-console/public/subagents.js";
import { readSubagentCursor } from "../src/web-console/settings.mjs";
import { spawnSync } from "node:child_process";
import { readSubagentCursor, readSubagentSetting } from "../src/web-console/settings.mjs";
import { foldKey } from "../src/web-console/workspace.mjs";
import {
ROOT,
HOME,
ENV,
CONSOLE_CLI,
safeJson,
cli,
sessionUrl,
readState,
@@ -510,3 +515,63 @@ test("subagents: a change arrives as reply lines only, carrying state, strength
cli(["stop", "--session", s.sid, "--force"]);
}
});
/* ------------------------------------- a loop phase's helpers follow its strength */
const loopOpen = (cwd, env, workflow = "implement", extraArgs = ["--loop"]) => {
const clean = { ...process.env, ...ENV };
for (const key of ["PLAN2CODE_IN_APP", "PLAN2CODE_EMBED_ORIGIN", "PLAN2CODE_APP_ANNOUNCE", "PLAN2CODE_APP_TOKEN", "PLAN2CODE_CONSOLE_SID", "PLAN2CODE_LOOP_STRENGTH"]) delete clean[key];
const res = spawnSync(process.execPath, [CONSOLE_CLI, "open", "--workflow", workflow, ...extraArgs, "--no-open"], { env: { ...clean, ...env }, encoding: "utf8", cwd });
assert.equal(res.status, 0, res.stderr);
return safeJson(res.stdout);
};
test("subagents: open --loop on a phase strength tells the agent that strength, leaves the stored setting alone and off stays off", () => {
const cwd = fs.mkdtempSync(path.join(HOME, "sub-loop-"));
const stored = (entry) => fs.writeFileSync(SUBAGENTS_STORE, JSON.stringify({ [foldKey(cwd)]: { implement: entry } }));
const sessions = [];
try {
stored({ on: true, instruction: null, max: 3, strength: "balanced" });
const before = fs.readFileSync(SUBAGENTS_STORE, "utf8");
const thrifty = loopOpen(cwd, { PLAN2CODE_IN_APP: "1", PLAN2CODE_LOOP_STRENGTH: "thrifty" });
sessions.push(thrifty.sid);
assert.deepEqual(subagentLines(thrifty.reply), ["Subagents: on, thrifty strength, at most 3 helpers at once (a console setting, not typed by the person)."]);
assert.deepEqual(readState(thrifty.session).loop, { driving: true, strength: "thrifty" });
assert.equal(fs.readFileSync(SUBAGENTS_STORE, "utf8"), before, "the stored setting is not rewritten");
// The next phase in the same session runs on another strength, then none.
const best = loopOpen(cwd, { PLAN2CODE_IN_APP: "1", PLAN2CODE_LOOP_STRENGTH: "best" }, "implement", ["--resume", thrifty.sid, "--loop"]);
assert.deepEqual(subagentLines(best.reply).map((l) => l.replace(/ \(a console.*/, "")), ["Subagents: on, best strength, at most 3 helpers at once"]);
assert.equal(fs.readFileSync(SUBAGENTS_STORE, "utf8"), before);
// A phase with no strength after a strength phase hears the stored one.
const none = loopOpen(cwd, { PLAN2CODE_IN_APP: "1" }, "implement", ["--resume", thrifty.sid, "--loop"]);
assert.deepEqual(subagentLines(none.reply).map((l) => l.replace(/ \(a console.*/, "")), ["Subagents: on, balanced strength, at most 3 helpers at once"]);
assert.deepEqual(readState(thrifty.session).loop, { driving: true });
// Off stays off, whatever the phase's strength.
stored({ on: false, instruction: null, max: 3, strength: "balanced" });
const off = loopOpen(cwd, { PLAN2CODE_IN_APP: "1", PLAN2CODE_LOOP_STRENGTH: "thrifty" }, "implement", ["--resume", thrifty.sid, "--loop"]);
assert.ok(!subagentLines(off.reply).some((l) => /on,/.test(l)), JSON.stringify(off.reply));
assert.equal(readSubagentSetting(thrifty.session).on, false);
} finally {
for (const sid of sessions) cli(["stop", "--session", sid, "--force"], { expectFail: true });
}
});
test("subagents: open without --loop ignores PLAN2CODE_LOOP_STRENGTH, and a word that is not a strength is dropped", () => {
const cwd = fs.mkdtempSync(path.join(HOME, "sub-noloop-"));
fs.writeFileSync(SUBAGENTS_STORE, JSON.stringify({ [foldKey(cwd)]: { implement: { on: true, instruction: null, max: 3, strength: "balanced" } } }));
const sessions = [];
try {
const plain = loopOpen(cwd, { PLAN2CODE_LOOP_STRENGTH: "thrifty" }, "implement", []);
sessions.push(plain.sid);
assert.deepEqual(subagentLines(plain.reply).map((l) => l.replace(/ \(a console.*/, "")), ["Subagents: on, balanced strength, at most 3 helpers at once"]);
assert.equal("loop" in readState(plain.session), false);
const odd = loopOpen(cwd, { PLAN2CODE_IN_APP: "1", PLAN2CODE_LOOP_STRENGTH: "session" });
sessions.push(odd.sid);
assert.deepEqual(readState(odd.session).loop, { driving: true });
} finally {
for (const sid of sessions) cli(["stop", "--session", sid, "--force"], { expectFail: true });
}
});