mirror of
https://github.com/jparkerweb/plan2code.git
synced 2026-11-16 20:45:33 -08:00
feat: run each loop phase on its own strength (Thrifty, Balanced, Best)
AI Assisted
This commit is contained in:
1 parent
e963513a8a
commit
81d14c8e78
41 files changed
+3290
-81
No files matched your search
@@ -1891,6 +1891,169 @@
|
||||
"args": [
|
||||
"a $HOME & \"b\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "OpenCode's shape: a chosen model goes ahead of the prompt",
|
||||
"entry": {
|
||||
"id": "opencode",
|
||||
"name": "OpenCode",
|
||||
"command": "opencode",
|
||||
"args": [
|
||||
"{model}",
|
||||
"--prompt",
|
||||
"{prompt}"
|
||||
],
|
||||
"prompt": "Use the plan2code skill",
|
||||
"bypassArgs": [
|
||||
"--auto"
|
||||
],
|
||||
"modelArgs": [
|
||||
"--model",
|
||||
"{model}"
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"model": "anthropic/sonnet",
|
||||
"bypass": false
|
||||
},
|
||||
"args": [
|
||||
"--model",
|
||||
"anthropic/sonnet",
|
||||
"--prompt",
|
||||
"Use the plan2code skill"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "OpenCode's shape: no model, no model argument",
|
||||
"entry": {
|
||||
"id": "opencode",
|
||||
"name": "OpenCode",
|
||||
"command": "opencode",
|
||||
"args": [
|
||||
"{model}",
|
||||
"--prompt",
|
||||
"{prompt}"
|
||||
],
|
||||
"prompt": "Use the plan2code skill",
|
||||
"bypassArgs": [
|
||||
"--auto"
|
||||
],
|
||||
"modelArgs": [
|
||||
"--model",
|
||||
"{model}"
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"bypass": false
|
||||
},
|
||||
"args": [
|
||||
"--prompt",
|
||||
"Use the plan2code skill"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Gemini CLI's shape: a chosen model goes ahead of the prompt",
|
||||
"entry": {
|
||||
"id": "gemini",
|
||||
"name": "Gemini CLI",
|
||||
"command": "gemini",
|
||||
"args": [
|
||||
"{model}",
|
||||
"{prompt}"
|
||||
],
|
||||
"prompt": "Use the plan2code skill",
|
||||
"bypassArgs": [
|
||||
"--approval-mode=yolo"
|
||||
],
|
||||
"modelArgs": [
|
||||
"--model",
|
||||
"{model}"
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"model": "flash",
|
||||
"bypass": false
|
||||
},
|
||||
"args": [
|
||||
"--model",
|
||||
"flash",
|
||||
"Use the plan2code skill"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Gemini CLI's shape: no model, no model argument",
|
||||
"entry": {
|
||||
"id": "gemini",
|
||||
"name": "Gemini CLI",
|
||||
"command": "gemini",
|
||||
"args": [
|
||||
"{model}",
|
||||
"{prompt}"
|
||||
],
|
||||
"prompt": "Use the plan2code skill",
|
||||
"bypassArgs": [
|
||||
"--approval-mode=yolo"
|
||||
],
|
||||
"modelArgs": [
|
||||
"--model",
|
||||
"{model}"
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"bypass": false
|
||||
},
|
||||
"args": [
|
||||
"Use the plan2code skill"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pi's shape: a chosen model goes ahead of the prompt",
|
||||
"entry": {
|
||||
"id": "pi",
|
||||
"name": "Pi",
|
||||
"command": "pi",
|
||||
"args": [
|
||||
"{model}",
|
||||
"{prompt}"
|
||||
],
|
||||
"prompt": "/skill:plan2code",
|
||||
"modelArgs": [
|
||||
"--model",
|
||||
"{model}"
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"model": "sonnet:high",
|
||||
"bypass": false
|
||||
},
|
||||
"args": [
|
||||
"--model",
|
||||
"sonnet:high",
|
||||
"/skill:plan2code"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Pi's shape: no model, no model argument",
|
||||
"entry": {
|
||||
"id": "pi",
|
||||
"name": "Pi",
|
||||
"command": "pi",
|
||||
"args": [
|
||||
"{model}",
|
||||
"{prompt}"
|
||||
],
|
||||
"prompt": "/skill:plan2code",
|
||||
"modelArgs": [
|
||||
"--model",
|
||||
"{model}"
|
||||
]
|
||||
},
|
||||
"options": {
|
||||
"bypass": false
|
||||
},
|
||||
"args": [
|
||||
"/skill:plan2code"
|
||||
]
|
||||
}
|
||||
],
|
||||
"ids": [
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"cases": [
|
||||
{
|
||||
"name": "claude shipped only",
|
||||
"agent": "claude",
|
||||
"shipped": {
|
||||
"thrifty": {
|
||||
"model": "sonnet",
|
||||
"effort": "low"
|
||||
},
|
||||
"balanced": {
|
||||
"model": "sonnet",
|
||||
"effort": "medium"
|
||||
},
|
||||
"best": {
|
||||
"model": "opus",
|
||||
"effort": "medium"
|
||||
}
|
||||
},
|
||||
"user": null,
|
||||
"models": [
|
||||
"default",
|
||||
"sonnet",
|
||||
"opus",
|
||||
"haiku"
|
||||
],
|
||||
"hasEffort": true,
|
||||
"expect": {
|
||||
"resolved": {
|
||||
"thrifty": {
|
||||
"model": "sonnet",
|
||||
"effort": "low"
|
||||
},
|
||||
"balanced": {
|
||||
"model": "sonnet",
|
||||
"effort": "medium"
|
||||
},
|
||||
"best": {
|
||||
"model": "opus",
|
||||
"effort": "medium"
|
||||
}
|
||||
},
|
||||
"next": {
|
||||
"thrifty": "balanced",
|
||||
"balanced": "best",
|
||||
"best": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "devin shipped only",
|
||||
"agent": "devin",
|
||||
"shipped": {
|
||||
"thrifty": {
|
||||
"model": "swe-2-medium"
|
||||
},
|
||||
"balanced": {
|
||||
"model": "swe-2-high"
|
||||
},
|
||||
"best": {
|
||||
"model": "claude-opus-5-5-medium"
|
||||
}
|
||||
},
|
||||
"user": null,
|
||||
"models": [
|
||||
"adaptive",
|
||||
"swe-2-medium",
|
||||
"swe-2-high",
|
||||
"claude-opus-5-5-medium"
|
||||
],
|
||||
"hasEffort": false,
|
||||
"expect": {
|
||||
"resolved": {
|
||||
"thrifty": {
|
||||
"model": "swe-2-medium"
|
||||
},
|
||||
"balanced": {
|
||||
"model": "swe-2-high"
|
||||
},
|
||||
"best": {
|
||||
"model": "claude-opus-5-5-medium"
|
||||
}
|
||||
},
|
||||
"next": {
|
||||
"thrifty": "balanced",
|
||||
"balanced": "best",
|
||||
"best": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "a user row overrides one shipped row, the others follow shipped",
|
||||
"agent": "claude",
|
||||
"shipped": {
|
||||
"thrifty": {
|
||||
"model": "sonnet",
|
||||
"effort": "low"
|
||||
},
|
||||
"balanced": {
|
||||
"model": "sonnet",
|
||||
"effort": "medium"
|
||||
},
|
||||
"best": {
|
||||
"model": "opus",
|
||||
"effort": "medium"
|
||||
}
|
||||
},
|
||||
"user": {
|
||||
"balanced": {
|
||||
"model": "opus",
|
||||
"effort": "low"
|
||||
}
|
||||
},
|
||||
"models": [
|
||||
"default",
|
||||
"sonnet",
|
||||
"opus",
|
||||
"haiku"
|
||||
],
|
||||
"hasEffort": true,
|
||||
"expect": {
|
||||
"resolved": {
|
||||
"thrifty": {
|
||||
"model": "sonnet",
|
||||
"effort": "low"
|
||||
},
|
||||
"balanced": {
|
||||
"model": "opus",
|
||||
"effort": "low"
|
||||
},
|
||||
"best": {
|
||||
"model": "opus",
|
||||
"effort": "medium"
|
||||
}
|
||||
},
|
||||
"next": {
|
||||
"thrifty": "balanced",
|
||||
"balanced": "best",
|
||||
"best": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "user-only mapping for an agent with nothing shipped",
|
||||
"agent": "codex",
|
||||
"shipped": null,
|
||||
"user": {
|
||||
"thrifty": {
|
||||
"model": "gpt-a"
|
||||
},
|
||||
"balanced": {
|
||||
"model": "gpt-b"
|
||||
},
|
||||
"best": {
|
||||
"model": "gpt-c"
|
||||
}
|
||||
},
|
||||
"models": [
|
||||
"gpt-a",
|
||||
"gpt-b",
|
||||
"gpt-c"
|
||||
],
|
||||
"hasEffort": false,
|
||||
"expect": {
|
||||
"resolved": {
|
||||
"thrifty": {
|
||||
"model": "gpt-a"
|
||||
},
|
||||
"balanced": {
|
||||
"model": "gpt-b"
|
||||
},
|
||||
"best": {
|
||||
"model": "gpt-c"
|
||||
}
|
||||
},
|
||||
"next": {
|
||||
"thrifty": "balanced",
|
||||
"balanced": "best",
|
||||
"best": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "partly mapped: Balanced only",
|
||||
"agent": "opencode",
|
||||
"shipped": null,
|
||||
"user": {
|
||||
"balanced": {
|
||||
"model": "m1"
|
||||
}
|
||||
},
|
||||
"models": [
|
||||
"m1",
|
||||
"m2"
|
||||
],
|
||||
"hasEffort": false,
|
||||
"expect": {
|
||||
"resolved": {
|
||||
"balanced": {
|
||||
"model": "m1"
|
||||
}
|
||||
},
|
||||
"next": {
|
||||
"thrifty": "balanced",
|
||||
"balanced": null,
|
||||
"best": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "partly mapped: Thrifty and Best only",
|
||||
"agent": "opencode",
|
||||
"shipped": null,
|
||||
"user": {
|
||||
"thrifty": {
|
||||
"model": "m1"
|
||||
},
|
||||
"best": {
|
||||
"model": "m2"
|
||||
}
|
||||
},
|
||||
"models": [
|
||||
"m1",
|
||||
"m2"
|
||||
],
|
||||
"hasEffort": false,
|
||||
"expect": {
|
||||
"resolved": {
|
||||
"thrifty": {
|
||||
"model": "m1"
|
||||
},
|
||||
"best": {
|
||||
"model": "m2"
|
||||
}
|
||||
},
|
||||
"next": {
|
||||
"thrifty": "best",
|
||||
"balanced": "best",
|
||||
"best": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "a user row whose model left the list falls back to the shipped row",
|
||||
"agent": "claude",
|
||||
"shipped": {
|
||||
"thrifty": {
|
||||
"model": "sonnet",
|
||||
"effort": "low"
|
||||
},
|
||||
"balanced": {
|
||||
"model": "sonnet",
|
||||
"effort": "medium"
|
||||
},
|
||||
"best": {
|
||||
"model": "opus",
|
||||
"effort": "medium"
|
||||
}
|
||||
},
|
||||
"user": {
|
||||
"best": {
|
||||
"model": "gone",
|
||||
"effort": "high"
|
||||
}
|
||||
},
|
||||
"models": [
|
||||
"default",
|
||||
"sonnet",
|
||||
"opus",
|
||||
"haiku"
|
||||
],
|
||||
"hasEffort": true,
|
||||
"expect": {
|
||||
"resolved": {
|
||||
"thrifty": {
|
||||
"model": "sonnet",
|
||||
"effort": "low"
|
||||
},
|
||||
"balanced": {
|
||||
"model": "sonnet",
|
||||
"effort": "medium"
|
||||
},
|
||||
"best": {
|
||||
"model": "opus",
|
||||
"effort": "medium"
|
||||
}
|
||||
},
|
||||
"next": {
|
||||
"thrifty": "balanced",
|
||||
"balanced": "best",
|
||||
"best": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "a shipped row whose model left the list reads unmapped",
|
||||
"agent": "claude",
|
||||
"shipped": {
|
||||
"thrifty": {
|
||||
"model": "sonnet",
|
||||
"effort": "low"
|
||||
},
|
||||
"balanced": {
|
||||
"model": "sonnet",
|
||||
"effort": "medium"
|
||||
},
|
||||
"best": {
|
||||
"model": "opus",
|
||||
"effort": "medium"
|
||||
}
|
||||
},
|
||||
"user": null,
|
||||
"models": [
|
||||
"default",
|
||||
"opus"
|
||||
],
|
||||
"hasEffort": true,
|
||||
"expect": {
|
||||
"resolved": {
|
||||
"best": {
|
||||
"model": "opus",
|
||||
"effort": "medium"
|
||||
}
|
||||
},
|
||||
"next": {
|
||||
"thrifty": "best",
|
||||
"balanced": "best",
|
||||
"best": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "an effort on an agent without one is dropped",
|
||||
"agent": "devin",
|
||||
"shipped": null,
|
||||
"user": {
|
||||
"thrifty": {
|
||||
"model": "swe-2-medium",
|
||||
"effort": "low"
|
||||
},
|
||||
"best": {
|
||||
"model": "swe-2-high",
|
||||
"effort": "high"
|
||||
}
|
||||
},
|
||||
"models": [
|
||||
"adaptive",
|
||||
"swe-2-medium",
|
||||
"swe-2-high",
|
||||
"claude-opus-5-5-medium"
|
||||
],
|
||||
"hasEffort": false,
|
||||
"expect": {
|
||||
"resolved": {
|
||||
"thrifty": {
|
||||
"model": "swe-2-medium"
|
||||
},
|
||||
"best": {
|
||||
"model": "swe-2-high"
|
||||
}
|
||||
},
|
||||
"next": {
|
||||
"thrifty": "best",
|
||||
"balanced": "best",
|
||||
"best": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "an effort outside low, medium and high is dropped, the row stays",
|
||||
"agent": "claude",
|
||||
"shipped": null,
|
||||
"user": {
|
||||
"thrifty": {
|
||||
"model": "haiku",
|
||||
"effort": "max"
|
||||
},
|
||||
"balanced": {
|
||||
"model": "sonnet",
|
||||
"effort": "medium"
|
||||
}
|
||||
},
|
||||
"models": [
|
||||
"default",
|
||||
"sonnet",
|
||||
"opus",
|
||||
"haiku"
|
||||
],
|
||||
"hasEffort": true,
|
||||
"expect": {
|
||||
"resolved": {
|
||||
"thrifty": {
|
||||
"model": "haiku"
|
||||
},
|
||||
"balanced": {
|
||||
"model": "sonnet",
|
||||
"effort": "medium"
|
||||
}
|
||||
},
|
||||
"next": {
|
||||
"thrifty": "balanced",
|
||||
"balanced": null,
|
||||
"best": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "a bad word or a row that is not an object is ignored",
|
||||
"agent": "claude",
|
||||
"shipped": null,
|
||||
"user": {
|
||||
"cheap": {
|
||||
"model": "haiku"
|
||||
},
|
||||
"thrifty": "haiku",
|
||||
"balanced": [
|
||||
"sonnet"
|
||||
],
|
||||
"best": {
|
||||
"model": 7
|
||||
},
|
||||
"extra": null
|
||||
},
|
||||
"models": [
|
||||
"default",
|
||||
"sonnet",
|
||||
"opus",
|
||||
"haiku"
|
||||
],
|
||||
"hasEffort": true,
|
||||
"expect": {
|
||||
"resolved": {},
|
||||
"next": {
|
||||
"thrifty": null,
|
||||
"balanced": null,
|
||||
"best": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "nothing anywhere",
|
||||
"agent": "pi",
|
||||
"shipped": null,
|
||||
"user": null,
|
||||
"models": [],
|
||||
"hasEffort": false,
|
||||
"expect": {
|
||||
"resolved": {},
|
||||
"next": {
|
||||
"thrifty": null,
|
||||
"balanced": null,
|
||||
"best": null
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "mappings that are not objects",
|
||||
"agent": "pi",
|
||||
"shipped": [
|
||||
"x"
|
||||
],
|
||||
"user": "y",
|
||||
"models": [
|
||||
"x"
|
||||
],
|
||||
"hasEffort": false,
|
||||
"expect": {
|
||||
"resolved": {},
|
||||
"next": {
|
||||
"thrifty": null,
|
||||
"balanced": null,
|
||||
"best": null
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -11,7 +11,7 @@ import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { attentionSignal, barNextSteps, continueKeys, defaultRightFraction, folderName, pickDefaultAgent, taskbarSwitch, windowTitleFor, deadConsoleAction, loopBanner, loopLocks, loopStopLabels, loopWindowTitle, CONSOLE_ASIDE_BREAKPOINT_PX, NEXT_STEP_LABELS } from "../plan2code-app/ui/meter.js";
|
||||
import { attentionSignal, barNextSteps, continueKeys, defaultRightFraction, folderName, pickDefaultAgent, taskbarSwitch, windowTitleFor, deadConsoleAction, loopBanner, loopRunText, loopLocks, loopStopLabels, loopWindowTitle, CONSOLE_ASIDE_BREAKPOINT_PX, NEXT_STEP_LABELS } from "../plan2code-app/ui/meter.js";
|
||||
import { findBrowser, launchBrowser } from "./browser.mjs";
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
@@ -339,6 +339,18 @@ test("the loop banner says what the loop is doing at every moment", () => {
|
||||
assert.deepEqual(said({ phase: 0, phases: 0 }).pips, []);
|
||||
});
|
||||
|
||||
test("the loop banner names the strength and model a phase runs on, and nothing when the toggle is off", () => {
|
||||
const base = { phase: 3, phases: 5, phaseName: "API", step: "Build" };
|
||||
const said = (extra) => loopBanner({ ...base, ...extra }).detail;
|
||||
assert.equal(said({ strength: "thrifty", model: "sonnet", effort: "low" }), "Phase 3 of 5: API \u00b7 Build \u00b7 Thrifty (sonnet, low)");
|
||||
assert.equal(said({ strength: "best", model: "opus" }), "Phase 3 of 5: API \u00b7 Build \u00b7 Best (opus)");
|
||||
assert.equal(said({ strength: "balanced" }), "Phase 3 of 5: API \u00b7 Build \u00b7 Balanced");
|
||||
assert.equal(said({ strength: null, model: "opus", effort: "high" }), "Phase 3 of 5: API \u00b7 Build \u00b7 (opus, high)", "Match my session shows the launch model");
|
||||
assert.equal(said({ strength: null, model: null, effort: null }), "Phase 3 of 5: API \u00b7 Build", "toggle off or no model known: unchanged");
|
||||
assert.equal(said({}), "Phase 3 of 5: API \u00b7 Build");
|
||||
assert.equal(loopRunText({}), "");
|
||||
});
|
||||
|
||||
test("Stop now turns the banner to Stopping, ahead of a question or a stop on its way", () => {
|
||||
const base = { phase: 2, phases: 4, phaseName: "API", step: "Build", stopAfterPhase: false, stopping: true };
|
||||
for (const extra of [{}, { step: "Waiting on you" }, { stopAfterPhase: true }, { step: "Starting" }]) {
|
||||
|
||||
@@ -334,6 +334,26 @@ test("launcher: mergeModels keeps the shipped list first and drops user duplicat
|
||||
assert.deepEqual(modelEntries({ devin: [{ id: 1 }, null, { id: "ok", label: "Ok" }] }, "devin"), [{ id: "ok", label: "Ok" }]);
|
||||
});
|
||||
|
||||
test("launcher: a strengths key in the models files is never read as a model list, and every shipped strength names a listed model", () => {
|
||||
const { modelEntries, mergeModels } = require(LAUNCHER);
|
||||
const shipped = JSON.parse(fs.readFileSync(path.join(path.dirname(LAUNCHER), "models.json"), "utf8"));
|
||||
assert.ok(shipped.strengths && typeof shipped.strengths === "object");
|
||||
assert.deepEqual(modelEntries(shipped, "strengths"), []);
|
||||
const claude = modelEntries(shipped, "claude");
|
||||
assert.ok(claude.length > 0);
|
||||
const user = { claude: [{ id: "mine", label: "Mine" }], strengths: { claude: { best: { model: "mine" } } } };
|
||||
assert.deepEqual(modelEntries(user, "strengths"), []);
|
||||
assert.deepEqual(mergeModels(claude, modelEntries(user, "claude")), [...claude, { id: "mine", label: "Mine" }]);
|
||||
assert.deepEqual(mergeModels(claude, modelEntries({ claude: user.claude }, "claude")), mergeModels(claude, modelEntries(user, "claude")));
|
||||
// Only Claude Code and Devin ship a mapping, and every row names a model in that agent's list.
|
||||
assert.deepEqual(Object.keys(shipped.strengths).sort(), ["claude", "devin"]);
|
||||
for (const [agent, rows] of Object.entries(shipped.strengths)) {
|
||||
assert.deepEqual(Object.keys(rows), ["thrifty", "balanced", "best"], agent);
|
||||
const ids = new Set(shipped[agent].map((m) => m.id));
|
||||
for (const row of Object.values(rows)) assert.ok(ids.has(row.model), `${agent}: ${row.model}`);
|
||||
}
|
||||
});
|
||||
|
||||
test("launcher: the shipped models.json is curated Devin at low/medium/high only", () => {
|
||||
const shipped = JSON.parse(fs.readFileSync(path.join(path.dirname(LAUNCHER), "models.json"), "utf8"));
|
||||
assert.ok(shipped.claude.length > 0);
|
||||
|
||||
@@ -536,7 +536,7 @@ test("models: the user's additions round-trip, XHigh and Max are refused, and th
|
||||
assert.equal((await post("/models", { nope: true })).status, 400);
|
||||
assert.equal((await get()).user.devin.length, 1, "a refused write must not touch what is saved");
|
||||
|
||||
// Codex has no curated list, but the launcher reads a user `codex` list all the same.
|
||||
// The launcher reads a user `codex` list too. The shipped file this test serves has no `codex` key, so that list is empty here.
|
||||
const codex = await post("/models", { models: { codex: [{ id: " gpt-5-codex ", label: "GPT-5 Codex" }, { id: "gpt-5-codex" }] } });
|
||||
assert.equal(codex.status, 200);
|
||||
assert.deepEqual((await get()).user, { claude: [], devin: [], codex: [{ id: "gpt-5-codex", label: "GPT-5 Codex" }] });
|
||||
@@ -547,6 +547,88 @@ test("models: the user's additions round-trip, XHigh and Max are refused, and th
|
||||
assert.deepEqual((await get()).user, { claude: [], devin: [], codex: [] });
|
||||
});
|
||||
|
||||
test("models: strengths come back shipped and user, round-trip, survive a models-only save, and bad rows name their field", async () => {
|
||||
fs.mkdirSync(path.join(MODELS_DIR, "bin"), { recursive: true });
|
||||
const shippedStrengths = { claude: { thrifty: { model: "sonnet", effort: "low" }, best: { model: "opus" } } };
|
||||
fs.writeFileSync(
|
||||
path.join(MODELS_DIR, "bin", "models.json"),
|
||||
JSON.stringify({ claude: [{ id: "opus", label: "Opus" }, { id: "sonnet", label: "Sonnet" }], strengths: shippedStrengths })
|
||||
);
|
||||
const get = async () => (await fetch(baseUrl + "/models", { headers: { cookie: cookieFor(baseUrl, token) } })).json();
|
||||
const userFile = () => JSON.parse(fs.readFileSync(path.join(MODELS_DIR, "models.json"), "utf8"));
|
||||
|
||||
const first = await get();
|
||||
assert.deepEqual(first.strengths, { shipped: shippedStrengths, user: {} }, "shipped rows come back, no user rows yet");
|
||||
assert.ok(!("strengths" in first.shipped) && !("strengths" in first.user), "strengths is never read as an agent's model list");
|
||||
|
||||
const mapping = { claude: { balanced: { model: " sonnet ", effort: "medium" }, best: { model: "opus" } }, codex: { thrifty: { model: "gpt-5-codex" } } };
|
||||
const saved = await post("/models", { models: { claude: [{ id: "mine" }] }, strengths: mapping });
|
||||
assert.equal(saved.status, 200);
|
||||
const want = { claude: { balanced: { model: "sonnet", effort: "medium" }, best: { model: "opus" } }, codex: { thrifty: { model: "gpt-5-codex" } } };
|
||||
assert.deepEqual((await get()).strengths.user, want);
|
||||
assert.deepEqual(userFile().strengths, want, "the launcher's file carries the mapping beside the lists");
|
||||
assert.equal(userFile().claude.length, 1);
|
||||
|
||||
// A save with only a Models list never wipes the mapping.
|
||||
assert.equal((await post("/models", { models: { claude: [{ id: "mine" }, { id: "other" }] } })).status, 200);
|
||||
assert.deepEqual((await get()).strengths.user, want);
|
||||
assert.equal((await get()).user.claude.length, 2);
|
||||
|
||||
// The agents' own lists and the reset keep it too; an explicit empty mapping clears it.
|
||||
assert.equal((await post("/models", { models: {} })).status, 200);
|
||||
assert.deepEqual((await get()).strengths.user, want);
|
||||
|
||||
const refused = [
|
||||
[{ claude: { thrifty: { model: "bad id!" } } }, /strengths\.claude\.thrifty\.model/],
|
||||
[{ claude: { cheap: { model: "sonnet" } } }, /strengths\.claude\.cheap/],
|
||||
[{ claude: { best: { model: "opus", effort: "max" } } }, /strengths\.claude\.best\.effort/],
|
||||
[{ nobody: { best: { model: "opus" } } }, /strengths\.nobody/],
|
||||
[{ claude: { best: "opus" } }, /strengths\.claude\.best/],
|
||||
[["claude"], /strengths/],
|
||||
];
|
||||
for (const [body, field] of refused) {
|
||||
const res = await post("/models", { models: {}, strengths: body });
|
||||
assert.equal(res.status, 400, JSON.stringify(body));
|
||||
assert.match((await res.json()).error, field);
|
||||
}
|
||||
assert.deepEqual((await get()).strengths.user, want, "a refused write must not touch what is saved");
|
||||
|
||||
assert.equal((await post("/models", { models: {}, strengths: {} })).status, 200);
|
||||
assert.deepEqual((await get()).strengths.user, {});
|
||||
assert.ok(!("strengths" in userFile()), "an empty mapping leaves no key behind");
|
||||
});
|
||||
|
||||
test("models: a saved mapping for an agent that is gone is skipped, and never hides or wipes the others", async (t) => {
|
||||
const userFile = path.join(MODELS_DIR, "models.json");
|
||||
t.after(() => fs.rmSync(userFile, { force: true }));
|
||||
fs.writeFileSync(
|
||||
userFile,
|
||||
JSON.stringify({ claude: [], strengths: { claude: { best: { model: "opus" } }, "deleted-agent": { best: { model: "x" } }, devin: { cheap: { model: "y" } } } })
|
||||
);
|
||||
const get = async () => (await fetch(baseUrl + "/models", { headers: { cookie: cookieFor(baseUrl, token) } })).json();
|
||||
assert.deepEqual((await get()).strengths.user, { claude: { best: { model: "opus" } } });
|
||||
assert.equal((await post("/models", { models: { claude: [{ id: "mine" }] } })).status, 200);
|
||||
assert.deepEqual(JSON.parse(fs.readFileSync(userFile, "utf8")).strengths, { claude: { best: { model: "opus" } } }, "the models-only save kept the good rows");
|
||||
});
|
||||
|
||||
test("models: one bad saved row never drops the other rows of its agent, like the app", async (t) => {
|
||||
const userFile = path.join(MODELS_DIR, "models.json");
|
||||
t.after(() => fs.rmSync(userFile, { force: true }));
|
||||
fs.writeFileSync(
|
||||
userFile,
|
||||
JSON.stringify({
|
||||
claude: [],
|
||||
strengths: { claude: { thrifty: { model: "haiku", effort: "max" }, balanced: { model: "bad id!" }, cheap: { model: "x" }, best: { model: "opus", effort: "high" } } },
|
||||
})
|
||||
);
|
||||
const get = async () => (await fetch(baseUrl + "/models", { headers: { cookie: cookieFor(baseUrl, token) } })).json();
|
||||
assert.deepEqual(
|
||||
(await get()).strengths.user,
|
||||
{ claude: { thrifty: { model: "haiku" }, best: { model: "opus", effort: "high" } } },
|
||||
"a bad effort is dropped and its row stays; a bad model or word drops only that row"
|
||||
);
|
||||
});
|
||||
|
||||
test("agents: no file is an empty list, a save is written whole and keeps unknown keys, and a bad save names its entry and field", async (t) => {
|
||||
const file = path.join(MODELS_DIR, "agents.json");
|
||||
t.after(() => fs.rmSync(file, { force: true }));
|
||||
|
||||
@@ -18,7 +18,7 @@ import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { applyPatch, blankState, validate, switchWorkflow } from "../src/web-console/patches.mjs";
|
||||
import { ROOT, CONSOLE_CLI, ENV, HOME, cli, readState, sessionUrl, payload } from "./web-console-test-helpers.mjs";
|
||||
import { ROOT, CONSOLE_CLI, ENV, HOME, MODELS_DIR, cli, readState, sessionUrl, payload } from "./web-console-test-helpers.mjs";
|
||||
|
||||
const APP_VARS = ["PLAN2CODE_IN_APP", "PLAN2CODE_EMBED_ORIGIN", "PLAN2CODE_APP_ANNOUNCE", "PLAN2CODE_APP_TOKEN", "PLAN2CODE_CONSOLE_SID"];
|
||||
|
||||
@@ -472,10 +472,16 @@ test("loopEnd: a valid stopped with an evidence doc in the same patch is stored"
|
||||
assert.equal(replaced.next.loopEnd.line, "A later word replaces the earlier");
|
||||
});
|
||||
|
||||
test("loopEnd: ask is accepted in can, so a stopped loop can offer a question before anything runs", () => {
|
||||
const { problems, next } = tryPatch(loopState(), { loopEnd: goodEnd({ can: ["resume", "ask", "fix", "dashboard"] }) });
|
||||
assert.deepEqual(problems, []);
|
||||
assert.deepEqual(next.loopEnd.can, ["resume", "ask", "fix", "dashboard"]);
|
||||
});
|
||||
|
||||
test("loopEnd: an unknown evidence id, a push in can, and the other bad fields are refused, each named", () => {
|
||||
const cases = [
|
||||
["an unknown evidence id", goodEnd({ evidence: "nope" }), /loopEnd\.evidence must be the id of a doc on the page/],
|
||||
["a push in can", goodEnd({ can: ["resume", "push"] }), /loopEnd\.can must be an array with entries from resume, fix, finalize, dashboard/],
|
||||
["a push in can", goodEnd({ can: ["resume", "push"] }), /loopEnd\.can must be an array with entries from resume, ask, fix, finalize, dashboard/],
|
||||
["a bad kind", goodEnd({ kind: "paused" }), /loopEnd\.kind must be "stopped" or "finished"/],
|
||||
["a two-line line", goodEnd({ line: "one\ntwo" }), /loopEnd\.line must be one line/],
|
||||
["an absolute uncommitted path", goodEnd({ uncommitted: ["/etc/passwd"] }), /loopEnd\.uncommitted must be an array of relative paths/],
|
||||
@@ -772,7 +778,7 @@ test("loop-setup: GET /loop-setup lists the phases left for a spec and refuses a
|
||||
}
|
||||
|
||||
// Saving through the route, then reading it back in a new request.
|
||||
const save = await send({ review: true, fixMode: "smart", fix: ["critical"], commit: true, onQuestion: "stop", afterLast: "offer-finalize", quickCheck: "", notifyKind: "query", notifyUrl: "https://example.com/hook", notifyParam: "msg", retryOnce: true, silenceMinutes: 30 });
|
||||
const save = await send({ review: true, fixMode: "smart", fix: ["critical"], commit: true, onQuestion: "stop", afterLast: "offer-finalize", quickCheck: "", notifyKind: "query", notifyUrl: "https://example.com/hook", notifyParam: "msg", retryOnce: true, silenceMinutes: 30, phaseStrength: false, strengths: {} });
|
||||
assert.equal(save.status, 200, await save.clone().text());
|
||||
const after = await get("?spec=specs/demo");
|
||||
assert.equal(after.body.remembered, true);
|
||||
@@ -789,3 +795,170 @@ test("loop-setup: GET /loop-setup lists the phases left for a spec and refuses a
|
||||
cli(["stop", "--session", opened.json.sid, "--force"], { cwd, expectFail: true });
|
||||
}
|
||||
});
|
||||
|
||||
/* ------------------------------ phase strength: the setup's new fields */
|
||||
|
||||
const specWith = (project, name, phases) => {
|
||||
const dir = path.join(project, "specs", name);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const rows = phases.map(([mark, n, title]) => `- [${mark}] **Phase ${n}:** ${title} - [phase-${n}.md](phase-${n}.md)`);
|
||||
fs.writeFileSync(path.join(dir, "overview.md"), ["# Demo", "", "## Phase Checklist", "", ...rows, ""].join("\n"));
|
||||
};
|
||||
|
||||
test("loop-setup: the strength fields default off, and their bad values are named", () => {
|
||||
fs.rmSync(LOOP_SETUP_PATH, { force: true });
|
||||
const cases = [
|
||||
["phaseStrength as a string", { phaseStrength: "yes" }, /"phaseStrength" must be true or false/],
|
||||
["strengths as a list", { strengths: [] }, /"strengths" must be an object/],
|
||||
["a key outside specs/", { strengths: { "../x": [] } }, /must be a folder directly under specs\//],
|
||||
["a hidden spec folder", { strengths: { "specs/.hidden": [] } }, /must be a folder directly under specs\//],
|
||||
["a nested spec key", { strengths: { "specs/a/b": [] } }, /must be a folder directly under specs\//],
|
||||
["picks as an object", { strengths: { "specs/a": {} } }, /at most 100 phase picks/],
|
||||
["101 picks", { strengths: { "specs/a": Array.from({ length: 101 }, (_, i) => ({ n: i + 1, name: "p", strength: "best" })) } }, /at most 100 phase picks/],
|
||||
["n of 0", { strengths: { "specs/a": [{ n: 0, name: "p", strength: "best" }] } }, /positive whole number/],
|
||||
["n as text", { strengths: { "specs/a": [{ n: "2", name: "p", strength: "best" }] } }, /positive whole number/],
|
||||
["a repeated n", { strengths: { "specs/a": [{ n: 2, name: "p", strength: "best" }, { n: 2, name: "q", strength: "best" }] } }, /Phase 2 twice/],
|
||||
["a 201-character name", { strengths: { "specs/a": [{ n: 1, name: "x".repeat(201), strength: "best" }] } }, /at most 200 characters/],
|
||||
["an unknown word", { strengths: { "specs/a": [{ n: 1, name: "p", strength: "max" }] } }, /strength of thrifty, balanced, best/],
|
||||
];
|
||||
const got = inStore(
|
||||
(m, { project, list }) =>
|
||||
list.map(([what, change]) => {
|
||||
try {
|
||||
m.saveLoopSetup(project, { ...m.LOOP_DEFAULTS, fix: [...m.LOOP_DEFAULTS.fix], ...change });
|
||||
return { what, threw: false };
|
||||
} catch (e) {
|
||||
return { what, threw: true, message: e.message };
|
||||
}
|
||||
}),
|
||||
{ project: projectA, list: cases.map(([what, change]) => [what, change]) }
|
||||
);
|
||||
for (const [i, [what, , expected]] of cases.entries()) {
|
||||
assert.equal(got[i].threw, true, `${what} is refused`);
|
||||
assert.match(got[i].message, expected, what);
|
||||
}
|
||||
assert.equal(fs.existsSync(LOOP_SETUP_PATH), false, "a refused save writes nothing");
|
||||
const fresh = inStore((m, { project }) => m.readLoopSetup(project), { project: projectA });
|
||||
assert.equal(fresh.phaseStrength, false);
|
||||
assert.deepEqual(fresh.strengths, {});
|
||||
});
|
||||
|
||||
test("loop-setup: an entry saved before the strength fields reads as off with no picks", () => {
|
||||
fs.rmSync(LOOP_SETUP_PATH, { force: true });
|
||||
const project = path.join(HOME, "store-project-old");
|
||||
inStore((m, { p }) => (m.saveLoopSetup(p, { ...m.LOOP_DEFAULTS, fix: ["warning"] }), null), { p: project });
|
||||
const store = JSON.parse(fs.readFileSync(LOOP_SETUP_PATH, "utf8"));
|
||||
const [folded] = Object.keys(store.projects);
|
||||
delete store.projects[folded].phaseStrength;
|
||||
delete store.projects[folded].strengths;
|
||||
fs.writeFileSync(LOOP_SETUP_PATH, JSON.stringify(store));
|
||||
const got = inStore((m, { p }) => m.readLoopSetup(p), { p: project });
|
||||
assert.equal(got.remembered, true);
|
||||
assert.equal(got.phaseStrength, false);
|
||||
assert.deepEqual(got.strengths, {});
|
||||
assert.deepEqual(got.fix, ["warning"]);
|
||||
});
|
||||
|
||||
test("loop-setup: picks for a removed spec folder or a renamed phase are pruned on read and on save", () => {
|
||||
fs.rmSync(LOOP_SETUP_PATH, { force: true });
|
||||
const project = fs.mkdtempSync(path.join(HOME, "prune-"));
|
||||
specWith(project, "keep", [["x", 1, "Done"], [" ", 2, "Build it"], [" ", 3, "Ship it"]]);
|
||||
specWith(project, "gone", [[" ", 1, "Only"]]);
|
||||
const picks = {
|
||||
"specs/keep": [
|
||||
{ n: 2, name: "Build it", strength: "thrifty" },
|
||||
{ n: 3, name: "Ship it renamed", strength: "best" },
|
||||
{ n: 1, name: "Done", strength: "best" },
|
||||
],
|
||||
"specs/gone": [{ n: 1, name: "Only", strength: "best" }],
|
||||
"specs/never": [{ n: 1, name: "x", strength: "best" }],
|
||||
};
|
||||
const out = inStore(
|
||||
async (m, { project, picks }) => {
|
||||
const fs = await import("node:fs");
|
||||
const path = await import("node:path");
|
||||
const left = (spec) => {
|
||||
const text = fs.readFileSync(path.join(project, spec, "overview.md"), "utf8");
|
||||
return [...text.matchAll(/^- \[ \] \*\*Phase (\d+):\*\* (.+?) - /gm)].map((x) => ({ n: Number(x[1]), name: x[2] }));
|
||||
};
|
||||
fs.rmSync(path.join(project, "specs", "gone"), { recursive: true });
|
||||
const saved = m.saveLoopSetup(project, { ...m.LOOP_DEFAULTS, fix: [...m.LOOP_DEFAULTS.fix], phaseStrength: true, strengths: picks }, left);
|
||||
const onDisk = JSON.parse(fs.readFileSync(m.LOOP_SETUP_FILE, "utf8"));
|
||||
const raw = Object.values(onDisk.projects)[0].strengths;
|
||||
// A later read, with the overview changed again, drops the next pick too.
|
||||
fs.writeFileSync(path.join(project, "specs", "keep", "overview.md"), "# Demo\n\n## Phase Checklist\n\n- [ ] **Phase 3:** Ship it - [phase-3.md](phase-3.md)\n");
|
||||
const later = m.readLoopSetup(project, left);
|
||||
return { saved: saved.strengths, raw, later: later.strengths };
|
||||
},
|
||||
{ project, picks }
|
||||
);
|
||||
const kept = [{ n: 2, name: "Build it", strength: "thrifty" }];
|
||||
assert.deepEqual(out.saved, { "specs/keep": kept }, "the removed folder, the never-made one, the renamed phase and the finished phase are dropped");
|
||||
assert.deepEqual(out.raw, { "specs/keep": kept }, "the file holds the pruned copy");
|
||||
assert.deepEqual(out.later, {}, "a later read prunes what the overview no longer lists");
|
||||
});
|
||||
|
||||
test("loop-setup: the picks live only in loop-setup.json, never under specs/", () => {
|
||||
fs.rmSync(LOOP_SETUP_PATH, { force: true });
|
||||
const project = fs.mkdtempSync(path.join(HOME, "nowrite-"));
|
||||
specWith(project, "demo", [[" ", 1, "One"]]);
|
||||
const before = fs.readFileSync(path.join(project, "specs", "demo", "overview.md"), "utf8");
|
||||
inStore(
|
||||
(m, { project }) => (m.saveLoopSetup(project, { ...m.LOOP_DEFAULTS, fix: [...m.LOOP_DEFAULTS.fix], phaseStrength: true, strengths: { "specs/demo": [{ n: 1, name: "One", strength: "best" }] } }), null),
|
||||
{ project }
|
||||
);
|
||||
assert.ok(fs.existsSync(LOOP_SETUP_PATH));
|
||||
assert.deepEqual(fs.readdirSync(path.join(project, "specs", "demo")), ["overview.md"]);
|
||||
assert.equal(fs.readFileSync(path.join(project, "specs", "demo", "overview.md"), "utf8"), before);
|
||||
assert.deepEqual(fs.readdirSync(project), ["specs"], "nothing else appears in the project");
|
||||
});
|
||||
|
||||
test("GET /loop-setup?agent= answers the loop agent's resolved strengths and this spec's picks", async () => {
|
||||
fs.rmSync(LOOP_SETUP_PATH, { force: true });
|
||||
const cwd = tempProject("route-strength");
|
||||
specWith(cwd, "demo", [[" ", 1, "One"], [" ", 2, "Two"]]);
|
||||
// The installed copy of the shipped model menu, with its strengths, as install.js drops it.
|
||||
const shipped = path.join(MODELS_DIR, "bin", "models.json");
|
||||
const hadShipped = fs.existsSync(shipped) ? fs.readFileSync(shipped) : null;
|
||||
fs.mkdirSync(path.dirname(shipped), { recursive: true });
|
||||
fs.copyFileSync(path.join(ROOT, "src", "launcher", "models.json"), shipped);
|
||||
const opened = run(["open", "--workflow", "implement", "--no-open"], { cwd });
|
||||
assert.equal(opened.status, 0, opened.stderr);
|
||||
const { base, cookie } = sessionUrl(opened.json.url);
|
||||
const get = async (query) => {
|
||||
const res = await fetch(`${base}/loop-setup${query}`, { headers: { cookie, connection: "close" } });
|
||||
return { status: res.status, body: await res.json() };
|
||||
};
|
||||
try {
|
||||
const none = await get("?spec=specs/demo");
|
||||
assert.equal(none.body.resolved, null, "no agent asked, none answered");
|
||||
assert.deepEqual(none.body.picks, []);
|
||||
assert.equal(none.body.phaseStrength, false);
|
||||
|
||||
const claude = await get("?spec=specs/demo&agent=claude");
|
||||
assert.equal(claude.status, 200);
|
||||
assert.deepEqual(Object.keys(claude.body.resolved).sort(), ["balanced", "best", "thrifty"], "the shipped Claude Code rows");
|
||||
for (const word of ["thrifty", "balanced", "best"]) assert.equal(typeof claude.body.resolved[word].model, "string");
|
||||
|
||||
const codex = await get("?spec=specs/demo&agent=codex");
|
||||
assert.deepEqual(codex.body.resolved, {}, "codex has nothing mapped");
|
||||
|
||||
for (const agent of ["nobody", "..%2Fx", ""]) assert.equal((await get(`?spec=specs/demo&agent=${agent}`)).body.resolved, null, `agent=${agent}`);
|
||||
|
||||
// A save with picks reads back for its own spec; a pick for a phase that is not there is pruned.
|
||||
const entry = {
|
||||
review: true, fixMode: "smart", fix: ["critical"], commit: true, onQuestion: "guess", afterLast: "offer-finalize", quickCheck: "",
|
||||
notifyKind: "query", notifyUrl: "", notifyParam: "message", retryOnce: true, silenceMinutes: 20, phaseStrength: true,
|
||||
strengths: { "specs/demo": [{ n: 2, name: "Two", strength: "best" }, { n: 9, name: "Gone", strength: "best" }] },
|
||||
};
|
||||
const save = await fetch(`${base}/loop-setup`, { method: "POST", headers: { "content-type": "application/json", cookie, origin: base, connection: "close" }, body: JSON.stringify(entry) });
|
||||
assert.equal(save.status, 200, await save.clone().text());
|
||||
const back = await get("?spec=specs/demo&agent=claude");
|
||||
assert.equal(back.body.phaseStrength, true);
|
||||
assert.deepEqual(back.body.picks, [{ n: 2, name: "Two", strength: "best" }]);
|
||||
} finally {
|
||||
if (hadShipped) fs.writeFileSync(shipped, hadShipped);
|
||||
else fs.rmSync(shipped, { force: true });
|
||||
cli(["stop", "--session", opened.json.sid, "--force"], { cwd, expectFail: true });
|
||||
}
|
||||
});
|
||||
@@ -5,7 +5,11 @@
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { pageSuite, openSession, tempProject, textOf } from "./page-test-helpers.mjs";
|
||||
import { MODELS_DIR } from "./web-console-test-helpers.mjs";
|
||||
|
||||
const suite = pageSuite(test);
|
||||
|
||||
@@ -46,6 +50,7 @@ test("the setup lists each phase on its own line, in the agreed order of fields
|
||||
|
||||
const labels = await s.page.locator(".loop-label").allInnerTexts();
|
||||
assert.deepEqual(labels, [
|
||||
"Strength per phase",
|
||||
"Review",
|
||||
"Fix findings",
|
||||
"When a phase asks something",
|
||||
@@ -256,6 +261,27 @@ test("the finish card counts each kind of decision per phase and opens on the fu
|
||||
assert.deepEqual(seen.actions, ["Finalize", "Back to the dashboard"], "the way home is offered next to Finalize");
|
||||
});
|
||||
|
||||
test("a stopped loop offers Ask a question, which tells the app and wakes no loop", async (t) => {
|
||||
const s = await openSetup(t);
|
||||
await frameInFakeApp(s);
|
||||
const seen = await s.page.evaluate(async () => {
|
||||
const m = await import(new URL("screens/ending.js", document.baseURI).href);
|
||||
const card = m.renderLoopEnd({
|
||||
kind: "stopped",
|
||||
line: "Phase 5 stopped",
|
||||
phases: [{ n: 5, name: "Docs", commit: null, unfixed: 0, retries: 0 }],
|
||||
uncommitted: [],
|
||||
can: ["ask", "resume", "fix", "dashboard"],
|
||||
});
|
||||
const buttons = [...card.querySelectorAll(".loop-end-actions button")];
|
||||
const ask = buttons.find((b) => b.textContent === "Ask a question");
|
||||
if (ask) ask.click();
|
||||
return { labels: buttons.map((b) => b.textContent), told: window.__told.map((x) => x.plan2code) };
|
||||
});
|
||||
assert.deepEqual(seen.labels, ["Ask a question", "Resume the loop", "Fix it by hand", "Back to the dashboard"]);
|
||||
assert.deepEqual(seen.told, ["loop-ask"]);
|
||||
});
|
||||
|
||||
// A real loop session: `open --loop` under the app's signal stamps the lock.
|
||||
async function openLoopPage(t) {
|
||||
return openSession(await suite.browser(), {
|
||||
@@ -329,3 +355,172 @@ test("a loop phase still starting says the loop is starting it, and promises no
|
||||
await s.page.getByRole("heading", { name: "The loop is starting this phase" }).waitFor();
|
||||
assert.doesNotMatch(await textOf(s.page.locator("#main")), /pause for approvals/);
|
||||
});
|
||||
|
||||
/* ------------------------------------------- Strength per phase (the section) */
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const LAUNCH = { agent: "claude", agentLabel: "Claude Code", model: "opus", effort: "medium", permissionMode: "bypass", promptSupport: true, running: false };
|
||||
|
||||
// The installed copy of the shipped model menu, with its strengths, as install.js drops it.
|
||||
function installShippedModels() {
|
||||
const shipped = path.join(MODELS_DIR, "bin", "models.json");
|
||||
fs.mkdirSync(path.dirname(shipped), { recursive: true });
|
||||
fs.copyFileSync(path.join(ROOT, "src", "launcher", "models.json"), shipped);
|
||||
}
|
||||
|
||||
// The setup screen with the app's answer in: this launch choice, then the strengths read for it.
|
||||
async function openStrengthSetup(t, launch = LAUNCH) {
|
||||
installShippedModels();
|
||||
const s = await openSetup(t);
|
||||
await s.page.evaluate(async (facts) => {
|
||||
const { page, redraw } = await import(new URL("state.js", document.baseURI).href);
|
||||
page.loopFacts = facts;
|
||||
redraw();
|
||||
}, launch);
|
||||
return s;
|
||||
}
|
||||
|
||||
const strengthRows = (s) =>
|
||||
s.page.locator(".loop-strength-row").evaluateAll((rows) =>
|
||||
rows.map((r) => ({ phase: r.querySelector(".loop-strength-phase").textContent, pick: r.querySelector("select").value, model: r.querySelector(".loop-strength-model").textContent }))
|
||||
);
|
||||
|
||||
test("the strength section is first in the form and off by default", async (t) => {
|
||||
const s = await openStrengthSetup(t);
|
||||
const toggle = s.page.locator("#loop-phase-strength");
|
||||
await toggle.waitFor();
|
||||
assert.equal(await toggle.isChecked(), false);
|
||||
assert.equal(await s.page.locator(".loop-label").first().innerText(), "Strength per phase");
|
||||
assert.equal(await s.page.locator(".loop-strength-row").count(), 0, "no rows until the toggle is on");
|
||||
assert.equal(await s.page.locator("#loop-strength-all").count(), 0);
|
||||
});
|
||||
|
||||
test("turning it on shows one row per phase left with the model each pick becomes, and Set all to fills them", async (t) => {
|
||||
const s = await openStrengthSetup(t);
|
||||
await s.page.locator("#loop-phase-strength").check();
|
||||
await s.page.locator(".loop-strength-row").first().waitFor();
|
||||
assert.deepEqual(await strengthRows(s), [
|
||||
{ phase: "Phase 1: One", pick: "", model: "opus, medium" },
|
||||
{ phase: "Phase 2: Two", pick: "", model: "opus, medium" },
|
||||
], "untouched rows match the session and show the launch choice");
|
||||
|
||||
await s.page.locator("#loop-strength-1").selectOption("thrifty");
|
||||
assert.equal(await textOf(s.page.locator('[data-phase="1"] .loop-strength-model')), "sonnet, low");
|
||||
|
||||
assert.equal(await s.page.locator("#loop-strength-all").inputValue(), "-", "Set all to starts blank");
|
||||
await s.page.locator("#loop-strength-all").selectOption("best");
|
||||
assert.deepEqual((await strengthRows(s)).map((r) => [r.pick, r.model]), [["best", "opus, medium"], ["best", "opus, medium"]]);
|
||||
assert.equal(await s.page.locator("#loop-strength-all").inputValue(), "-", "and goes blank again: it is not a saved value");
|
||||
|
||||
await s.page.locator("#loop-strength-2").selectOption("balanced");
|
||||
assert.deepEqual((await strengthRows(s)).map((r) => [r.pick, r.model]), [["best", "opus, medium"], ["balanced", "sonnet, medium"]], "a row can change after Set all");
|
||||
|
||||
await s.page.locator("#loop-strength-all").selectOption("match");
|
||||
assert.deepEqual((await strengthRows(s)).map((r) => r.pick), ["", ""], "Match my session fills every row too");
|
||||
});
|
||||
|
||||
test("an agent with nothing mapped grays the section and links to Agents & models", async (t) => {
|
||||
const s = await openStrengthSetup(t, { ...LAUNCH, agent: "codex", agentLabel: "Codex" });
|
||||
const note = s.page.locator(".loop-strength-none");
|
||||
await note.waitFor();
|
||||
assert.match(await textOf(note), /This agent has no strengths mapped yet\./);
|
||||
assert.equal(await s.page.locator("#loop-phase-strength").isDisabled(), true);
|
||||
assert.equal(await s.page.locator(".loop-strength-row").count(), 0);
|
||||
await note.getByRole("button").click();
|
||||
await s.page.locator("#prefs-tab-agents[aria-selected='true']").waitFor();
|
||||
});
|
||||
|
||||
test("a saved pick that is not mapped holds Start with a line naming the row, and a change releases it", async (t) => {
|
||||
const s = await openStrengthSetup(t);
|
||||
await s.page.locator("#loop-phase-strength").check();
|
||||
await s.page.locator(".loop-strength-row").first().waitFor();
|
||||
// Thrifty is no longer mapped for this agent (an override went away); Phase 2 still holds it.
|
||||
await s.page.evaluate(async () => {
|
||||
const { page, redraw } = await import(new URL("state.js", document.baseURI).href);
|
||||
page.loopSetup.facts.resolved = { best: { model: "opus", effort: "medium" } };
|
||||
page.loopSetup.form.strengths["specs/ready"] = [{ n: 2, name: "Two", strength: "thrifty" }];
|
||||
redraw();
|
||||
});
|
||||
const why = s.page.locator(".loop-why");
|
||||
assert.equal(await textOf(why), "Pick a new strength for Phase 2: not mapped for this agent.");
|
||||
assert.equal(await s.page.locator("#loop-start").isDisabled(), true);
|
||||
const pick = s.page.locator("#loop-strength-2");
|
||||
assert.equal(await pick.inputValue(), "thrifty", "the saved word stays in the row");
|
||||
assert.equal(await pick.locator("option:checked").innerText(), "Thrifty (not mapped)");
|
||||
assert.equal(await pick.locator("option:checked").evaluate((o) => o.disabled), true);
|
||||
|
||||
await pick.selectOption("best");
|
||||
assert.equal(await s.page.locator(".loop-why").count(), 0, "the hold is gone");
|
||||
assert.equal(await s.page.locator("#loop-start").isDisabled(), false);
|
||||
});
|
||||
|
||||
test("startBlock names every unmapped row, ignores the picks while the toggle is off, and waits for the strengths", async (t) => {
|
||||
const s = await openSetup(t);
|
||||
const lines = await s.page.evaluate(async () => {
|
||||
const { startBlock } = await import(new URL("screens/loop-setup.js", document.baseURI).href);
|
||||
const phasesLeft = [{ n: 2, name: "A" }, { n: 3, name: "B" }, { n: 4, name: "C" }];
|
||||
const strengths = { "specs/x": [{ n: 2, name: "A", strength: "thrifty" }, { n: 3, name: "B", strength: "best" }, { n: 4, name: "C", strength: "thrifty" }] };
|
||||
const facts = (resolved) => ({ spec: "specs/x", phasesLeft, pending: 0, resolved });
|
||||
const form = (phaseStrength) => ({ commit: false, phaseStrength, strengths });
|
||||
const app = { permissionMode: "bypass", promptSupport: true };
|
||||
return {
|
||||
two: startBlock(form(true), facts({ best: { model: "m" } }), app),
|
||||
off: startBlock(form(false), facts({ best: { model: "m" } }), app),
|
||||
allMapped: startBlock(form(true), facts({ thrifty: { model: "s" }, best: { model: "m" } }), app),
|
||||
nothingMapped: startBlock(form(true), facts({}), app),
|
||||
loading: startBlock(form(true), facts(undefined), app),
|
||||
last: startBlock(form(true), { ...facts({ best: { model: "m" } }), phasesLeft: [] }, app),
|
||||
};
|
||||
});
|
||||
assert.equal(lines.two, "Pick a new strength for Phase 2, Phase 4: not mapped for this agent.");
|
||||
assert.equal(lines.off, "");
|
||||
assert.equal(lines.allMapped, "");
|
||||
assert.equal(lines.nothingMapped, "", "an agent with nothing mapped grays the section instead");
|
||||
assert.equal(lines.loading, "Asking the app...");
|
||||
assert.equal(lines.last, "Nothing left to build in this spec.", "the older checks keep their order");
|
||||
});
|
||||
|
||||
test("the toggle and the picks survive reopening the screen, and the toggle off sends nothing for the loop", async (t) => {
|
||||
const s = await openStrengthSetup(t);
|
||||
await s.page.locator("#loop-phase-strength").check();
|
||||
await s.page.locator("#loop-strength-1").selectOption("thrifty");
|
||||
await s.page.locator("#loop-strength-2").selectOption("best");
|
||||
|
||||
const save = () =>
|
||||
s.page.evaluate(async () => {
|
||||
const { page } = await import(new URL("state.js", document.baseURI).href);
|
||||
const { setupBody } = await import(new URL("screens/loop-setup.js", document.baseURI).href);
|
||||
const res = await fetch("/loop-setup", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(setupBody(page.loopSetup.form, page.loopSetup)) });
|
||||
return { status: res.status, body: await res.json() };
|
||||
});
|
||||
const saved = await save();
|
||||
assert.equal(saved.status, 200, JSON.stringify(saved.body));
|
||||
assert.equal(saved.body.setup.phaseStrength, true);
|
||||
assert.deepEqual(saved.body.setup.strengths, { "specs/ready": [{ n: 1, name: "One", strength: "thrifty" }, { n: 2, name: "Two", strength: "best" }] });
|
||||
|
||||
const reopen = async () => {
|
||||
await s.page.evaluate(async () => {
|
||||
const m = await import(new URL("screens/loop-setup.js", document.baseURI).href);
|
||||
const { page, redraw } = await import(new URL("state.js", document.baseURI).href);
|
||||
await m.openLoopSetup({ dir: "specs/ready" });
|
||||
redraw();
|
||||
});
|
||||
await s.page.locator("#loop-start").waitFor();
|
||||
};
|
||||
await reopen();
|
||||
await s.page.locator("#loop-strength-1").waitFor();
|
||||
assert.equal(await s.page.locator("#loop-phase-strength").isChecked(), true);
|
||||
assert.deepEqual((await strengthRows(s)).map((r) => [r.pick, r.model]), [["thrifty", "sonnet, low"], ["best", "opus, medium"]]);
|
||||
|
||||
// Off: the picks stay saved, the loop is told nothing.
|
||||
await s.page.locator("#loop-phase-strength").uncheck();
|
||||
const off = await save();
|
||||
assert.equal(off.body.setup.phaseStrength, false);
|
||||
assert.equal(off.body.setup.strengths["specs/ready"].length, 2, "the toggle hides the picks, it does not erase them");
|
||||
|
||||
// Back to Match my session for every row removes the key.
|
||||
await s.page.locator("#loop-phase-strength").check();
|
||||
await s.page.locator("#loop-strength-all").selectOption("match");
|
||||
const cleared = await save();
|
||||
assert.deepEqual(cleared.body.setup.strengths, {});
|
||||
});
|
||||
@@ -162,6 +162,199 @@ test("Agents & models: the table lists Claude, Devin and Codex, and a model id t
|
||||
assert.equal(await formField(s, "Model id").inputValue(), "gpt-test-codex-2");
|
||||
});
|
||||
|
||||
// The "Strengths in the loop" fold: Thrifty, Balanced and Best mapped to the
|
||||
// agent's own models. models.json under PLAN2CODE_MODELS_DIR is what Save writes.
|
||||
const MODELS_FILE = path.join(MODELS_DIR, "models.json");
|
||||
const strengthsFold = (s) => agentForm(s).locator("details.prefs-agents-strengths");
|
||||
const strengthRow = (s, word) => strengthsFold(s).locator(`.strength-row[data-word="${word}"]`);
|
||||
const savedStrengths = () => {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(MODELS_FILE, "utf8")).strengths ?? null;
|
||||
} catch {
|
||||
return null; // not written yet
|
||||
}
|
||||
};
|
||||
|
||||
// The shipped list the tests start with has no models at all; the fold needs some
|
||||
// (and Claude and Devin's shipped strengths), so a test writes them and puts the file back.
|
||||
const SHIPPED_FILE = path.join(MODELS_DIR, "bin", "models.json");
|
||||
function seedShippedModels(t) {
|
||||
fs.mkdirSync(path.dirname(SHIPPED_FILE), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
SHIPPED_FILE,
|
||||
JSON.stringify({
|
||||
claude: [{ id: "sonnet", label: "Sonnet" }, { id: "opus", label: "Opus" }],
|
||||
devin: [{ id: "swe-2-medium", label: "SWE 2 Medium" }, { id: "swe-2-high", label: "SWE 2 High" }, { id: "claude-opus-5-5-medium", label: "Opus 5.5 Medium" }],
|
||||
codex: [],
|
||||
strengths: {
|
||||
claude: { thrifty: { model: "sonnet", effort: "low" }, balanced: { model: "sonnet", effort: "medium" }, best: { model: "opus", effort: "medium" } },
|
||||
devin: { thrifty: { model: "swe-2-medium" }, balanced: { model: "swe-2-high" }, best: { model: "claude-opus-5-5-medium" } },
|
||||
},
|
||||
})
|
||||
);
|
||||
t.after(() => fs.rmSync(SHIPPED_FILE, { force: true }));
|
||||
}
|
||||
|
||||
// Save lands asynchronously and an earlier Saved mark may still be on screen, so wait on the file.
|
||||
async function savedStrengthsAre(expected) {
|
||||
for (let i = 0; i < 100; i++) {
|
||||
if (JSON.stringify(savedStrengths()) === JSON.stringify(expected)) return;
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
}
|
||||
assert.deepEqual(savedStrengths(), expected);
|
||||
}
|
||||
|
||||
async function openStrengths(s, name) {
|
||||
await s.page.getByRole("button", { name: `Edit ${name}` }).click();
|
||||
await formPane(s).waitFor();
|
||||
await strengthsFold(s).locator("summary").click();
|
||||
}
|
||||
|
||||
test("Agents & models: Claude's fold shows the shipped rows as default, and only a changed row is saved, until Reset", async (t) => {
|
||||
noSavedLooks(t);
|
||||
seedAgents(t);
|
||||
t.after(() => fs.rmSync(MODELS_FILE, { force: true }));
|
||||
seedShippedModels(t);
|
||||
fs.rmSync(MODELS_FILE, { force: true });
|
||||
const s = await openSession(await suite.browser(), { t, workflow: "plan", payload: PLAN_QUESTION });
|
||||
await openAgentsTab(s);
|
||||
await openStrengths(s, "Claude");
|
||||
assert.equal(await strengthRow(s, "thrifty").locator('select[data-field="model"]').inputValue(), "sonnet");
|
||||
assert.equal(await strengthRow(s, "thrifty").locator('select[data-field="effort"]').inputValue(), "low");
|
||||
assert.equal(await strengthRow(s, "best").locator('select[data-field="model"]').inputValue(), "opus");
|
||||
assert.equal(await strengthsFold(s).locator(".strength-default").count(), 3, "all three rows are untouched defaults");
|
||||
assert.equal(await textOf(strengthRow(s, "balanced").locator(".strength-default")), "default");
|
||||
assert.equal(await strengthsFold(s).locator(".strength-reset").count(), 0, "nothing to reset yet");
|
||||
assert.equal(await textOf(strengthsFold(s).locator(".strength-offer")), "The loop will offer: Thrifty, Balanced, Best");
|
||||
|
||||
await strengthRow(s, "thrifty").locator('select[data-field="model"]').selectOption("opus");
|
||||
await agentForm(s).locator(".prefs-unsaved-badge").waitFor();
|
||||
assert.equal(await strengthRow(s, "thrifty").locator(".strength-reset").count(), 1, "a changed row has Reset");
|
||||
assert.equal(await strengthRow(s, "thrifty").locator(".strength-default").count(), 0);
|
||||
assert.equal(await strengthRow(s, "balanced").locator(".strength-reset").count(), 0, "the other rows stay default");
|
||||
assert.equal(fs.existsSync(MODELS_FILE), false, "nothing is saved before Save");
|
||||
|
||||
await formButton(s, "Save").click();
|
||||
await agentForm(s).locator(".prefs-saved").filter({ hasText: /^Saved. Restart plan2code/ }).waitFor();
|
||||
assert.deepEqual(savedStrengths(), { claude: { thrifty: { model: "opus", effort: "low" } } }, "only the changed row is written");
|
||||
|
||||
await strengthRow(s, "thrifty").locator(".strength-reset").click();
|
||||
assert.equal(await strengthRow(s, "thrifty").locator('select[data-field="model"]').inputValue(), "sonnet", "Reset restores the shipped row");
|
||||
assert.equal(await textOf(strengthRow(s, "thrifty").locator(".strength-default")), "default");
|
||||
await agentForm(s).locator(".prefs-unsaved-badge").waitFor();
|
||||
await formButton(s, "Save").click();
|
||||
await savedStrengthsAre(null); // with every row back at its default, no strengths are stored
|
||||
});
|
||||
|
||||
test("Agents & models: Codex has an effort select on each row and Devin does not", async (t) => {
|
||||
noSavedLooks(t);
|
||||
seedAgents(t);
|
||||
t.after(() => fs.rmSync(MODELS_FILE, { force: true }));
|
||||
seedShippedModels(t);
|
||||
const s = await openSession(await suite.browser(), { t, workflow: "plan", payload: PLAN_QUESTION });
|
||||
await openAgentsTab(s);
|
||||
await openStrengths(s, "Codex");
|
||||
assert.equal(await textOf(strengthsFold(s).locator(".strength-empty")), "Add a model under Models first", "no models ship for Codex");
|
||||
await formButton(s, "Add a model").click();
|
||||
await formField(s, "Model id").fill("gpt-test-codex-1");
|
||||
assert.equal(await strengthsFold(s).locator('select[data-field="effort"]').count(), 3);
|
||||
assert.equal(await strengthRow(s, "thrifty").locator('select[data-field="model"]').inputValue(), "", "nothing ships for Codex, so the row is not mapped");
|
||||
assert.equal(await textOf(strengthsFold(s).locator(".strength-offer")), "The loop will not offer a strength for this agent until one is mapped.");
|
||||
await formButton(s, "Discard").click(); // the added model is unsaved: leaving would ask first
|
||||
await s.page.getByRole("button", { name: "Edit Devin" }).click();
|
||||
await strengthsFold(s).locator("summary").click();
|
||||
assert.equal(await strengthsFold(s).locator('select[data-field="model"]').count(), 3);
|
||||
assert.equal(await strengthsFold(s).locator('select[data-field="effort"]').count(), 0, "Devin takes no separate effort");
|
||||
});
|
||||
|
||||
test("Agents & models: a strength-only edit is unsaved until Save, Discard puts the shipped rows back, and Reset keeps focus on its row", async (t) => {
|
||||
noSavedLooks(t);
|
||||
seedAgents(t);
|
||||
t.after(() => fs.rmSync(MODELS_FILE, { force: true }));
|
||||
seedShippedModels(t);
|
||||
const s = await openSession(await suite.browser(), { t, workflow: "plan", payload: PLAN_QUESTION });
|
||||
await openAgentsTab(s);
|
||||
await openStrengths(s, "Claude");
|
||||
assert.equal(await agentForm(s).locator(".prefs-unsaved-badge").isVisible(), false);
|
||||
await strengthRow(s, "best").locator('select[data-field="model"]').selectOption("sonnet");
|
||||
await agentForm(s).locator(".prefs-unsaved-badge").waitFor();
|
||||
await formButton(s, "Discard").click();
|
||||
assert.equal(await agentForm(s).locator(".prefs-unsaved-badge").isVisible(), false, "Discard clears the strength-only change");
|
||||
assert.equal(await strengthRow(s, "best").locator('select[data-field="model"]').inputValue(), "opus", "the shipped row is back");
|
||||
assert.equal(fs.existsSync(MODELS_FILE), false, "nothing was written");
|
||||
|
||||
await strengthRow(s, "best").locator('select[data-field="model"]').selectOption("sonnet");
|
||||
await strengthRow(s, "best").locator(".strength-reset").focus();
|
||||
await strengthRow(s, "best").locator(".strength-reset").click();
|
||||
assert.equal(
|
||||
await s.page.evaluate(() => document.activeElement?.closest("[data-word]")?.dataset.word),
|
||||
"best",
|
||||
"focus stays in the row the Reset was in"
|
||||
);
|
||||
});
|
||||
|
||||
test("Agents & models: a saved row whose model was removed reads not mapped while other models remain, and Save drops it", async (t) => {
|
||||
noSavedLooks(t);
|
||||
seedAgents(t);
|
||||
t.after(() => fs.rmSync(MODELS_FILE, { force: true }));
|
||||
seedShippedModels(t);
|
||||
fs.rmSync(MODELS_FILE, { force: true });
|
||||
const s = await openSession(await suite.browser(), { t, workflow: "plan", payload: PLAN_QUESTION });
|
||||
await openAgentsTab(s);
|
||||
await s.page.click("#agents-add");
|
||||
await formPane(s).waitFor();
|
||||
await formField(s, "Name").fill("Gemini");
|
||||
await formField(s, "Command").fill("node");
|
||||
await agentForm(s).locator("details.agent-fold > summary", { hasText: /^Models$/ }).click();
|
||||
await formButton(s, "Add a model").click();
|
||||
await formField(s, "Model id").fill("model-a");
|
||||
await formButton(s, "Add a model").click();
|
||||
await formField(s, "Model id").nth(1).fill("model-b");
|
||||
await strengthsFold(s).locator("summary").click();
|
||||
await strengthRow(s, "thrifty").locator('select[data-field="model"]').selectOption("model-a");
|
||||
await strengthRow(s, "best").locator('select[data-field="model"]').selectOption("model-b");
|
||||
await formButton(s, "Save").click();
|
||||
await savedStrengthsAre({ gemini: { thrifty: { model: "model-a" }, best: { model: "model-b" } } });
|
||||
|
||||
await agentForm(s).locator(".agent-row.model").first().getByRole("button", { name: "Remove" }).click();
|
||||
assert.equal(await strengthRow(s, "thrifty").locator('select[data-field="model"]').inputValue(), "", "the removed model reads not mapped");
|
||||
assert.equal(await textOf(strengthsFold(s).locator(".strength-offer")), "The loop will offer: Best");
|
||||
await formButton(s, "Save").click();
|
||||
await savedStrengthsAre({ gemini: { best: { model: "model-b" } } });
|
||||
});
|
||||
|
||||
test("Agents & models: an added agent maps strengths only from its own Models, and a removed model reads not mapped", async (t) => {
|
||||
noSavedLooks(t);
|
||||
seedAgents(t);
|
||||
t.after(() => fs.rmSync(MODELS_FILE, { force: true }));
|
||||
seedShippedModels(t);
|
||||
fs.rmSync(MODELS_FILE, { force: true });
|
||||
const s = await openSession(await suite.browser(), { t, workflow: "plan", payload: PLAN_QUESTION });
|
||||
await openAgentsTab(s);
|
||||
await s.page.click("#agents-add");
|
||||
await formPane(s).waitFor();
|
||||
await formField(s, "Name").fill("Copilot");
|
||||
await formField(s, "Command").fill("node");
|
||||
await strengthsFold(s).locator("summary").click();
|
||||
assert.equal(await textOf(strengthsFold(s).locator(".strength-empty")), "Add a model under Models first");
|
||||
assert.equal(await strengthsFold(s).locator("select").count(), 0, "no selects without a model");
|
||||
|
||||
await agentForm(s).locator("details.agent-fold > summary", { hasText: /^Models$/ }).click();
|
||||
await formButton(s, "Add a model").click();
|
||||
await formField(s, "Model id").fill("gpt-test-1");
|
||||
assert.equal(await strengthsFold(s).locator(".strength-empty").count(), 0, "the fold follows the Models list as it changes");
|
||||
await strengthRow(s, "thrifty").locator('select[data-field="model"]').selectOption("gpt-test-1");
|
||||
assert.equal(await strengthRow(s, "thrifty").locator('select[data-field="effort"]').count(), 0, "an added agent has no effort select");
|
||||
await formButton(s, "Save").click();
|
||||
await agentForm(s).locator(".prefs-saved").waitFor();
|
||||
assert.deepEqual(savedStrengths(), { copilot: { thrifty: { model: "gpt-test-1" } } }, "saved under the new agent's id");
|
||||
|
||||
await agentForm(s).locator(".agent-row.model").first().getByRole("button", { name: "Remove" }).click();
|
||||
assert.equal(await textOf(strengthsFold(s).locator(".strength-empty")), "Add a model under Models first");
|
||||
await formButton(s, "Save").click();
|
||||
await savedStrengthsAre(null);
|
||||
});
|
||||
|
||||
test("Agents & models: a person adds an agent, tests its command, sees it after a reload, and deletes it", async (t) => {
|
||||
noSavedLooks(t);
|
||||
seedAgents(t);
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
// Tests for src/web-console/strengths.mjs against the shared cases in
|
||||
// scripts/fixtures/strength-cases.json (the Rust twin in models.rs loads the
|
||||
// same file). Run with: node --test scripts/test-web-console-strengths.mjs
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { STRENGTH_ORDER, nextStrength, resolveStrengths } from "../src/web-console/strengths.mjs";
|
||||
|
||||
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const FIXTURE = JSON.parse(fs.readFileSync(path.join(ROOT, "scripts", "fixtures", "strength-cases.json"), "utf8"));
|
||||
|
||||
test("the fixture is schema 1 and covers every kind of case", () => {
|
||||
assert.equal(FIXTURE.schema, 1);
|
||||
assert.ok(FIXTURE.cases.length >= 10);
|
||||
const names = FIXTURE.cases.map((c) => c.name);
|
||||
assert.equal(new Set(names).size, names.length, "case names are unique");
|
||||
});
|
||||
|
||||
for (const c of FIXTURE.cases) {
|
||||
test(`resolveStrengths: ${c.name}`, () => {
|
||||
const resolved = resolveStrengths({ shipped: c.shipped, user: c.user, models: c.models, hasEffort: c.hasEffort });
|
||||
assert.deepEqual(resolved, c.expect.resolved);
|
||||
});
|
||||
|
||||
test(`nextStrength: ${c.name}`, () => {
|
||||
for (const word of STRENGTH_ORDER) {
|
||||
assert.equal(nextStrength(c.expect.resolved, word), c.expect.next[word], `next after ${word}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test("nextStrength: an unknown word or a missing mapping has no next", () => {
|
||||
assert.equal(nextStrength({ best: { model: "x" } }, "cheap"), null);
|
||||
assert.equal(nextStrength(null, "thrifty"), null);
|
||||
assert.equal(nextStrength(undefined, "balanced"), null);
|
||||
});
|
||||
|
||||
test("resolveStrengths: called with nothing at all resolves nothing", () => {
|
||||
assert.deepEqual(resolveStrengths(), {});
|
||||
assert.deepEqual(resolveStrengths({}), {});
|
||||
});
|
||||
@@ -9,10 +9,15 @@ import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { defaultInstruction, offered, onLines, STRENGTH_LINES, STRENGTHS } from "../src/web-console/public/subagents.js";
|
||||
import { readSubagentCursor } from "../src/web-console/settings.mjs";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { readSubagentCursor, readSubagentSetting } from "../src/web-console/settings.mjs";
|
||||
import { foldKey } from "../src/web-console/workspace.mjs";
|
||||
import {
|
||||
ROOT,
|
||||
HOME,
|
||||
ENV,
|
||||
CONSOLE_CLI,
|
||||
safeJson,
|
||||
cli,
|
||||
sessionUrl,
|
||||
readState,
|
||||
@@ -510,3 +515,63 @@ test("subagents: a change arrives as reply lines only, carrying state, strength
|
||||
cli(["stop", "--session", s.sid, "--force"]);
|
||||
}
|
||||
});
|
||||
|
||||
/* ------------------------------------- a loop phase's helpers follow its strength */
|
||||
|
||||
const loopOpen = (cwd, env, workflow = "implement", extraArgs = ["--loop"]) => {
|
||||
const clean = { ...process.env, ...ENV };
|
||||
for (const key of ["PLAN2CODE_IN_APP", "PLAN2CODE_EMBED_ORIGIN", "PLAN2CODE_APP_ANNOUNCE", "PLAN2CODE_APP_TOKEN", "PLAN2CODE_CONSOLE_SID", "PLAN2CODE_LOOP_STRENGTH"]) delete clean[key];
|
||||
const res = spawnSync(process.execPath, [CONSOLE_CLI, "open", "--workflow", workflow, ...extraArgs, "--no-open"], { env: { ...clean, ...env }, encoding: "utf8", cwd });
|
||||
assert.equal(res.status, 0, res.stderr);
|
||||
return safeJson(res.stdout);
|
||||
};
|
||||
|
||||
test("subagents: open --loop on a phase strength tells the agent that strength, leaves the stored setting alone and off stays off", () => {
|
||||
const cwd = fs.mkdtempSync(path.join(HOME, "sub-loop-"));
|
||||
const stored = (entry) => fs.writeFileSync(SUBAGENTS_STORE, JSON.stringify({ [foldKey(cwd)]: { implement: entry } }));
|
||||
const sessions = [];
|
||||
try {
|
||||
stored({ on: true, instruction: null, max: 3, strength: "balanced" });
|
||||
const before = fs.readFileSync(SUBAGENTS_STORE, "utf8");
|
||||
const thrifty = loopOpen(cwd, { PLAN2CODE_IN_APP: "1", PLAN2CODE_LOOP_STRENGTH: "thrifty" });
|
||||
sessions.push(thrifty.sid);
|
||||
assert.deepEqual(subagentLines(thrifty.reply), ["Subagents: on, thrifty strength, at most 3 helpers at once (a console setting, not typed by the person)."]);
|
||||
assert.deepEqual(readState(thrifty.session).loop, { driving: true, strength: "thrifty" });
|
||||
assert.equal(fs.readFileSync(SUBAGENTS_STORE, "utf8"), before, "the stored setting is not rewritten");
|
||||
|
||||
// The next phase in the same session runs on another strength, then none.
|
||||
const best = loopOpen(cwd, { PLAN2CODE_IN_APP: "1", PLAN2CODE_LOOP_STRENGTH: "best" }, "implement", ["--resume", thrifty.sid, "--loop"]);
|
||||
assert.deepEqual(subagentLines(best.reply).map((l) => l.replace(/ \(a console.*/, "")), ["Subagents: on, best strength, at most 3 helpers at once"]);
|
||||
assert.equal(fs.readFileSync(SUBAGENTS_STORE, "utf8"), before);
|
||||
|
||||
// A phase with no strength after a strength phase hears the stored one.
|
||||
const none = loopOpen(cwd, { PLAN2CODE_IN_APP: "1" }, "implement", ["--resume", thrifty.sid, "--loop"]);
|
||||
assert.deepEqual(subagentLines(none.reply).map((l) => l.replace(/ \(a console.*/, "")), ["Subagents: on, balanced strength, at most 3 helpers at once"]);
|
||||
assert.deepEqual(readState(thrifty.session).loop, { driving: true });
|
||||
|
||||
// Off stays off, whatever the phase's strength.
|
||||
stored({ on: false, instruction: null, max: 3, strength: "balanced" });
|
||||
const off = loopOpen(cwd, { PLAN2CODE_IN_APP: "1", PLAN2CODE_LOOP_STRENGTH: "thrifty" }, "implement", ["--resume", thrifty.sid, "--loop"]);
|
||||
assert.ok(!subagentLines(off.reply).some((l) => /on,/.test(l)), JSON.stringify(off.reply));
|
||||
assert.equal(readSubagentSetting(thrifty.session).on, false);
|
||||
} finally {
|
||||
for (const sid of sessions) cli(["stop", "--session", sid, "--force"], { expectFail: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("subagents: open without --loop ignores PLAN2CODE_LOOP_STRENGTH, and a word that is not a strength is dropped", () => {
|
||||
const cwd = fs.mkdtempSync(path.join(HOME, "sub-noloop-"));
|
||||
fs.writeFileSync(SUBAGENTS_STORE, JSON.stringify({ [foldKey(cwd)]: { implement: { on: true, instruction: null, max: 3, strength: "balanced" } } }));
|
||||
const sessions = [];
|
||||
try {
|
||||
const plain = loopOpen(cwd, { PLAN2CODE_LOOP_STRENGTH: "thrifty" }, "implement", []);
|
||||
sessions.push(plain.sid);
|
||||
assert.deepEqual(subagentLines(plain.reply).map((l) => l.replace(/ \(a console.*/, "")), ["Subagents: on, balanced strength, at most 3 helpers at once"]);
|
||||
assert.equal("loop" in readState(plain.session), false);
|
||||
const odd = loopOpen(cwd, { PLAN2CODE_IN_APP: "1", PLAN2CODE_LOOP_STRENGTH: "session" });
|
||||
sessions.push(odd.sid);
|
||||
assert.deepEqual(readState(odd.session).loop, { driving: true });
|
||||
} finally {
|
||||
for (const sid of sessions) cli(["stop", "--session", sid, "--force"], { expectFail: true });
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user