diff --git a/plan2code-app/fake-agent/src/main.rs b/plan2code-app/fake-agent/src/main.rs index 8086875..0f62371 100644 --- a/plan2code-app/fake-agent/src/main.rs +++ b/plan2code-app/fake-agent/src/main.rs @@ -116,6 +116,10 @@ pub fn parse(line: &str) -> Command { fn main() { say("FAKE-AGENT ready"); + // What the app launched it with: the launch prompt is one of these, so a + // test can read back which prompt a generation was started on. + let args: Vec = std::env::args().skip(1).collect(); + say(&format!("ARGS {}", args.join(" "))); if let Some(size) = size_text() { say(&size); } diff --git a/plan2code-app/src-tauri/src/agent_manager.rs b/plan2code-app/src-tauri/src/agent_manager.rs index 318257c..1de779c 100644 --- a/plan2code-app/src-tauri/src/agent_manager.rs +++ b/plan2code-app/src-tauri/src/agent_manager.rs @@ -95,7 +95,8 @@ fn append_replay(replay: &mut Vec, bytes: &[u8], limit: usize) { struct Inner { state: AgentState, running: Option, - /// What a Restart starts again: the choice and the brief. + /// What a Restart starts again: the choice and its brief (the prompt a + /// restart runs again, never a transient first prompt). last_start: Option<(StartChoice, Option)>, size: (u16, u16), brief_note: Option, @@ -194,12 +195,10 @@ impl AgentManager { mode: PermissionMode, brief: Option, ) -> Result<(), String> { - let mut inner = lock(&self.shared.inner); - if inner.state == AgentState::Running { - return Err("an agent is already running for this project".to_string()); - } - inner.brief_note = None; - let previous = inner.last_start.replace(( + let prompt = brief + .clone() + .unwrap_or_else(|| kind.default_prompt().to_string()); + self.begin( StartChoice { kind, model, @@ -207,8 +206,50 @@ impl AgentManager { mode, }, brief, - )); - match self.shared.spawn(&mut inner) { + prompt, + ) + } + + /// Like `start`, but `prompt` is this launch's first prompt alone: a later + /// Restart still opens the placeholder brief (the dashboard), it does not + /// run `prompt` again. The loop's own agent starts take this path, so a + /// phase prompt never becomes the brief a New Session repeats. + pub fn start_transient( + &self, + kind: Agent, + model: Option, + effort: Option, + mode: PermissionMode, + prompt: String, + ) -> Result<(), String> { + self.begin( + StartChoice { + kind, + model, + effort, + mode, + }, + None, + prompt, + ) + } + + /// `start` and `start_transient` behind one door: remember `choice` with + /// `restart_brief` (what a later Restart runs again), then spawn the agent + /// on `first_prompt`, which is not always the same string. + fn begin( + &self, + choice: StartChoice, + restart_brief: Option, + first_prompt: String, + ) -> Result<(), String> { + let mut inner = lock(&self.shared.inner); + if inner.state == AgentState::Running { + return Err("an agent is already running for this project".to_string()); + } + inner.brief_note = None; + let previous = inner.last_start.replace((choice, restart_brief)); + match self.shared.spawn(&mut inner, first_prompt) { Ok(()) => Ok(()), Err(e) => { // A start that never ran must not take away what Restart would @@ -225,14 +266,15 @@ impl AgentManager { /// Restart's brief note; the app still forgets the console session, so /// the new agent's `open` makes a new one. An agent that ended, or never /// ran, starts directly. Needs a launch choice from an earlier start. + /// `prompt` belongs to this launch alone: the stored brief stays what it + /// was, so a later New Session still opens the dashboard, not this + /// prompt again. pub fn start_fresh(&self, prompt: String) -> Result<(), String> { let mut inner = lock(&self.shared.inner); - let (choice, _) = inner - .last_start - .clone() - .ok_or("no agent has been started")?; + if inner.last_start.is_none() { + return Err("no agent has been started".to_string()); + } inner.brief_note = None; - inner.last_start = Some((choice, Some(prompt))); self.shared.forget(); if inner.state == AgentState::Running { self.shared.generation.fetch_add(1, Ordering::SeqCst); @@ -240,7 +282,7 @@ impl AgentManager { let _ = old.host.kill(); } } - match self.shared.spawn(&mut inner) { + match self.shared.spawn(&mut inner, prompt) { Ok(()) => Ok(()), Err(e) => { inner.state = AgentState::Ended { @@ -381,13 +423,15 @@ impl Shared { (self.sink)(AgentEvent::State(self.snapshot(inner))); } - /// Start the agent in `inner.last_start` and watch for it to end. - fn spawn(self: &Arc, inner: &mut Inner) -> Result<(), String> { - let (choice, brief) = inner + /// Start the agent in `inner.last_start` on `prompt` and watch for it to + /// end. `prompt` is this launch's first prompt: the stored brief for a + /// start or a Restart, or a transient one (`start_fresh`, + /// `start_transient`) that a Restart must not repeat. + fn spawn(self: &Arc, inner: &mut Inner, prompt: String) -> Result<(), String> { + let (choice, _) = inner .last_start .clone() .ok_or("no agent has been started")?; - let prompt = brief.unwrap_or_else(|| choice.kind.default_prompt().to_string()); let mut spec = agent_launch_spec( &choice.kind, &self.folder, @@ -467,7 +511,16 @@ impl Shared { } let hook = self.before_fresh.lock().ok().and_then(|slot| slot.clone()); inner.brief_note = hook.and_then(|run| run()); - match self.spawn(inner) { + let prompt = inner + .last_start + .as_ref() + .map(|(choice, brief)| { + brief + .clone() + .unwrap_or_else(|| choice.kind.default_prompt().to_string()) + }) + .ok_or("no agent has been started")?; + match self.spawn(inner, prompt) { Ok(()) => Ok(()), Err(e) => { inner.state = AgentState::Ended { diff --git a/plan2code-app/src-tauri/src/loop_driver.rs b/plan2code-app/src-tauri/src/loop_driver.rs index 05fd6be..b8d186b 100644 --- a/plan2code-app/src-tauri/src/loop_driver.rs +++ b/plan2code-app/src-tauri/src/loop_driver.rs @@ -573,6 +573,23 @@ fn suggestion_warning(review: &ReviewPart) -> Option { }) } +/// The `loopExpect` patch: the setup's judging criteria, stamped into the +/// phase's session so the console refuses a report that misses one at post +/// time, while the agent can still fix it. `fix` is "smart" or the level +/// list; an empty list means findings are reported, not fixed. +fn expect_patch(setup: &LoopSetup) -> Value { + json!({ + "loopExpect": { + "review": setup.review, + "fix": match setup.fix_mode { + FixMode::Smart => json!("smart"), + FixMode::Levels => json!(setup.fix.iter().map(|l| l.word()).collect::>()), + }, + "guess": setup.on_question == OnQuestion::Guess, + } + }) +} + /// Whether to take the skill's report at its word, cross-checked against the /// spec's own marks (`phase_status` is the phase's entry from `specs.mjs /// status`). The quick check runs after an Approve, not here. @@ -1310,14 +1327,16 @@ pub fn note_page(slot: &LoopSlot, workflow: &str, sid: &str) { } /// Start a fresh agent on `prompt`: on the agent's stored launch choice when it -/// has one, else on `launch` (an agent the loop stopped has forgotten it). +/// has one, else on `launch` (an agent the loop stopped has forgotten it). The +/// prompt is transient either way: a later New Session opens the dashboard, it +/// does not run the phase's (or Fix it's or Finalize's) prompt again. pub fn start_with(agent: &AgentManager, launch: &Launch, prompt: String) -> Result<(), String> { if agent.launch_choice().is_some() { agent.start_fresh(prompt) } else { agent.forget_session(); let (kind, model, effort, mode) = launch.clone(); - agent.start(kind, model, effort, mode, Some(prompt)) + agent.start_transient(kind, model, effort, mode, prompt) } } @@ -1504,6 +1523,7 @@ fn tick(ctx: &Ctx) -> bool { if state["loopOutcome"].is_object() { return handle_outcome(ctx, &snap, &state["loopOutcome"]); } + ensure_expect(ctx, &snap, &sid, state); if let Some(go) = crash_or_outcome(ctx, &snap, &sid) { return go; } @@ -1544,6 +1564,32 @@ fn crash_or_outcome(ctx: &Ctx, snap: &Snapshot, sid: &str) -> Option { Some(fail(ctx, snap, StopCause::Crash, reason)) } +/// Whether `e` is a whole `loopExpect` stamp: the three fields the console +/// enforces, each the right kind. Anything less counts as no stamp, so a +/// corrupt or partial one is replaced rather than left to fail every post. +fn valid_expect(e: &Value) -> bool { + let fix_ok = e["fix"].as_str() == Some("smart") + || e["fix"].as_array().is_some_and(|a| { + a.iter() + .all(|l| ["critical", "warning", "suggestion"].contains(&l.as_str().unwrap_or(""))) + }); + e.is_object() && e["review"].is_boolean() && e["guess"].is_boolean() && fix_ok +} + +/// The judging criteria (`loopExpect`), posted into the session so the +/// console refuses a report that misses one at post time, while the agent can +/// still fix and repost, instead of the phase stopping on the verdict later. +/// It lands only once the agent has locked the session with `open --loop`, +/// and is reposted on every look while the stored stamp is missing or not the +/// full shape (a fresh session, an unlock that dropped it, a corrupt stamp). +/// A post that does not land only weakens this early check and is retried. +fn ensure_expect(ctx: &Ctx, snap: &Snapshot, sid: &str, state: &Value) { + if !state["loop"]["driving"].as_bool().unwrap_or(false) || valid_expect(&state["loopExpect"]) { + return; + } + let _ = loop_tools::console_post(&ctx.console_dir, sid, &expect_patch(&snap.setup)); +} + /// Count (or clear) the looks that could not read the session. fn note_read_failure(ctx: &Ctx, failed: bool) { let mut guard = lock(&ctx.slot); @@ -2955,6 +3001,45 @@ mod tests { assert_eq!(LoopSetup::default().on_question, OnQuestion::Guess); } + #[test] + fn the_expect_patch_carries_the_setup() { + assert_eq!( + expect_patch(&setup()), + json!({ "loopExpect": { "review": true, "fix": ["critical", "warning", "suggestion"], "guess": false } }) + ); + let mut smart = setup(); + smart.fix_mode = FixMode::Smart; + smart.on_question = OnQuestion::Guess; + assert_eq!(expect_patch(&smart)["loopExpect"]["fix"], json!("smart")); + assert_eq!(expect_patch(&smart)["loopExpect"]["guess"], json!(true)); + let mut no_review = setup(); + no_review.review = false; + no_review.fix = vec![FixLevel::Critical]; + assert_eq!( + expect_patch(&no_review)["loopExpect"], + json!({ "review": false, "fix": ["critical"], "guess": false }) + ); + } + + #[test] + fn a_stamp_counts_only_when_whole() { + assert!(valid_expect( + &json!({ "review": true, "fix": "smart", "guess": false }) + )); + assert!(valid_expect( + &json!({ "review": true, "fix": ["critical"], "guess": true }) + )); + for bad in [ + json!(5), + json!({ "review": "yes", "fix": "smart", "guess": false }), + json!({ "review": true, "guess": false }), + json!({ "review": true, "fix": ["major"], "guess": false }), + json!({ "review": true, "fix": "everything", "guess": false }), + ] { + assert!(!valid_expect(&bad), "{bad}"); + } + } + fn phase(n: u32) -> PhaseRecord { PhaseRecord { n, @@ -3933,7 +4018,7 @@ second line" console_dir: PathBuf::from("."), home: report_home, launch: ( - Agent::Builtin(crate::agents::AgentKind::Claude), + Agent::Builtin(crate::agents::builtin_entry("claude").unwrap()), None, None, PermissionMode::Bypass, @@ -5874,4 +5959,88 @@ second line" "/plan2code-4-finalize specs/x/overview.md --web" ); } + + /// The `ARGS` lines the fake agent has printed, one per generation, in the + /// order they started: the prompts each launch ran with. + fn launched_prompts(seen: &Arc>) -> Vec { + lock(seen) + .lines() + .filter_map(|line| { + line.trim() + .strip_prefix("ARGS") + .map(|rest| rest.trim().to_string()) + }) + .collect() + } + + /// A plain folder to run the fake agent in: no git or specs needed, the + /// manager only spawns the program there. + fn agent_folder(name: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("plan2code-agent-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + fn wait_for_generations(seen: &Arc>, count: usize) { + wait_until("the agent generations", || { + launched_prompts(seen).len() >= count + }); + } + + #[test] + fn new_session_opens_the_dashboard_not_the_phase_prompt_again() { + let folder = agent_folder("restart"); + let (agent, seen) = fake_agent_manager(&folder); + // The loop's cold start, then the usual fresh start: both transient. + start_with(&agent, &fake_agent_launch(), "phase one prompt".into()).unwrap(); + wait_for_generations(&seen, 1); + start_with(&agent, &fake_agent_launch(), "phase two prompt".into()).unwrap(); + wait_for_generations(&seen, 2); + // New Session starts on the placeholder brief, not the last prompt. + agent.restart().unwrap(); + wait_for_generations(&seen, 3); + assert_eq!( + launched_prompts(&seen), + vec![ + "phase one prompt".to_string(), + "phase two prompt".to_string(), + String::new() + ], + "a transient prompt must never become what Restart starts again" + ); + agent.stop(); + } + + #[test] + fn a_real_brief_survives_a_transient_prompt() { + let folder = agent_folder("brief"); + let (agent, seen) = fake_agent_manager(&folder); + let (kind, model, effort, mode) = fake_agent_launch(); + agent + .start( + kind, + model, + effort, + mode, + Some("the original brief".to_string()), + ) + .unwrap(); + wait_for_generations(&seen, 1); + agent.start_fresh("a phase prompt".into()).unwrap(); + wait_for_generations(&seen, 2); + agent.restart().unwrap(); + wait_for_generations(&seen, 3); + assert_eq!( + launched_prompts(&seen), + vec![ + "the original brief".to_string(), + "a phase prompt".to_string(), + "the original brief".to_string() + ], + "Restart starts the stored brief again" + ); + agent.stop(); + } } diff --git a/scripts/test-web-console-loop.mjs b/scripts/test-web-console-loop.mjs index 1324ded..27a7c28 100644 --- a/scripts/test-web-console-loop.mjs +++ b/scripts/test-web-console-loop.mjs @@ -17,7 +17,7 @@ import fs from "node:fs"; import path from "node:path"; import { spawnSync } from "node:child_process"; import { pathToFileURL } from "node:url"; -import { applyPatch, blankState, validate } from "../src/web-console/patches.mjs"; +import { applyPatch, blankState, validate, switchWorkflow } from "../src/web-console/patches.mjs"; import { ROOT, CONSOLE_CLI, ENV, HOME, cli, readState, sessionUrl, payload } from "./web-console-test-helpers.mjs"; const APP_VARS = ["PLAN2CODE_IN_APP", "PLAN2CODE_EMBED_ORIGIN", "PLAN2CODE_APP_ANNOUNCE", "PLAN2CODE_APP_TOKEN", "PLAN2CODE_CONSOLE_SID"]; @@ -109,10 +109,14 @@ test("a later open --resume without --loop clears the lock and the loop's cards; assert.equal(posted.status, 0, posted.stderr); assert.equal(readState(s.session).items[0].handledBy, "loop"); + const stamped = post(s, { loopExpect: { review: true, fix: "smart", guess: true } }); + assert.equal(stamped.status, 0, stamped.stderr); + const plain = run(["open", "--resume", s.sid, "--no-open"], { cwd: s.cwd }); assert.equal(plain.status, 0, plain.stderr); const unlocked = readState(s.session); assert.equal("loop" in unlocked, false, "the lock is gone"); + assert.equal("loopExpect" in unlocked, false, "the criteria go with the lock"); assert.equal(unlocked.items.length, 1, "the card stays as a record"); assert.equal("handledBy" in unlocked.items[0], false, "but it is no longer the loop's"); assert.equal(post(s, { agent: { status: "waiting" } }).status, 0, "the unlocked session takes an ordinary post"); @@ -120,6 +124,10 @@ test("a later open --resume without --loop clears the lock and the loop's cards; const again = run(["open", "--resume", s.sid, "--workflow", "implement", "--loop", "--no-open"], { cwd: s.cwd, extra: IN_APP }); assert.equal(again.status, 0, again.stderr); assert.deepEqual(readState(s.session).loop, { driving: true }); + + const restamped = post(s, { loopExpect: { review: false, fix: [], guess: true } }); + assert.equal(restamped.status, 0, restamped.stderr); + assert.deepEqual(readState(s.session).loopExpect, { review: false, fix: [], guess: true }, "a re-locked session takes a fresh stamp"); } finally { s.stop(); } @@ -295,6 +303,151 @@ test("a loopOutcome through post is refused outside a loop session", () => { } }); +/* --------------------------------------------------------- loopExpect */ + +const expectSmart = (extra = {}) => ({ review: true, fix: "smart", guess: true, ...extra }); +const withExpect = (stamp, extra = {}) => loopState({ loopExpect: stamp, ...extra }); + +test("loopExpect: the app stamps its criteria into a loop session, refused elsewhere and after the report", () => { + const stored = tryPatch(loopState(), { loopExpect: expectSmart() }); + assert.deepEqual(stored.problems, []); + assert.deepEqual(stored.next.loopExpect, expectSmart()); + + const unlocked = loopState(); + delete unlocked.loop; + assert.match(tryPatch(unlocked, { loopExpect: expectSmart() }).problems.join("\n"), /loopExpect: is only accepted in a loop session/); + + const reported = tryPatch(loopState(), { loopOutcome: withTriage([skip()]) }); + assert.match(tryPatch(reported.next, { loopExpect: expectSmart() }).problems.join("\n"), /loopExpect: came after the run's report/); + + const second = tryPatch(stored.next, { loopExpect: expectSmart({ guess: false }) }); + assert.match(second.problems.join("\n"), /loopExpect: was already stamped for this session/, "one stamp per session, from anyone"); + + // A corrupt stored stamp is replaceable, not a wedge; and a hop drops it. + const corrupt = loopState({ loopExpect: 5 }); + assert.deepEqual(tryPatch(corrupt, { loopExpect: expectSmart() }).problems, [], "a non-object stamp is overwritten"); + const corruptObj = loopState({ loopExpect: { review: "yes" } }); + assert.deepEqual(tryPatch(corruptObj, { loopExpect: expectSmart() }).problems, [], "a malformed object stamp is overwritten too"); + const hopped = switchWorkflow(stored.next, { workflow: "implement" }); + assert.equal("loopExpect" in hopped, false, "a hop drops the stamp"); +}); + +test("a null patch cannot delete the loop's report fields", () => { + for (const key of ["loopExpect", "loopOutcome", "loopEnd"]) { + const { problems } = tryPatch(loopState(), { [key]: null }); + assert.ok(problems.some((p) => p.includes(`${key}: cannot be deleted by a patch`)), `${key}: got ${JSON.stringify(problems)}`); + } + assert.deepEqual(tryPatch(loopState(), { agent: null }).problems, [], "ordinary fields still delete"); +}); + +test("a failed outcome is never held to the loopExpect: the driver judges criteria on passed only", () => { + const smart = withExpect(expectSmart({ guess: false })); + const failed = goodOutcome(); + Object.assign(failed, { result: "failed", reason: "blocked" }); + failed.review.ran = false; + failed.review.recheck = false; + failed.tests.passed = false; + assert.deepEqual(tryPatch(smart, { loopOutcome: failed }).problems, [], "a failed report lands with its reason"); +}); + +test("a passed report is held to the loopExpect on leftovers and failed tests, like the driver", () => { + const levels = withExpect({ review: true, fix: ["critical", "warning"], guess: true }); + const leftCritical = goodOutcome(); + leftCritical.review.left.critical = 2; + assert.match(tryPatch(levels, { loopOutcome: leftCritical }).problems.join("\n"), /loopOutcome\.review\.left\.critical must be 0/); + // The unfixed list counts too, not only the counter. + const unfixedWarning = goodOutcome(); + unfixedWarning.review.unfixed = [{ level: "warning", title: "Still open", file: "src/x.js" }]; + assert.match(tryPatch(levels, { loopOutcome: unfixedWarning }).problems.join("\n"), /loopOutcome\.review\.left\.warning must be 0/); + // A leftover at a level the loop did not choose is fine. + const leftSuggestion = goodOutcome(); + leftSuggestion.review.left.suggestion = 3; + assert.deepEqual(tryPatch(levels, { loopOutcome: leftSuggestion }).problems, [], "suggestions never fail a phase"); + const warningOnly = withExpect({ review: true, fix: ["warning"], guess: true }); + assert.deepEqual(tryPatch(warningOnly, { loopOutcome: leftCritical }).problems, [], "critical left is only refused when chosen"); + // Failed tests on a passed report. + const failedTests = goodOutcome(); + failedTests.tests.passed = false; + assert.match(tryPatch(levels, { loopOutcome: failedTests }).problems.join("\n"), /loopOutcome\.tests\.passed must be true on a passed report/); + const noTests = goodOutcome(); + noTests.tests.ran = false; + assert.deepEqual(tryPatch(levels, { loopOutcome: noTests }).problems, [], "tests that never ran are not failed tests"); +}); + +test("loopExpect: each bad field is refused with a message naming it", () => { + const cases = [ + ["not an object", "yes", /loopExpect must be an object/], + ["review not a boolean", expectSmart({ review: "yes" }), /loopExpect\.review must be true or false/], + ["fix an unknown word", expectSmart({ fix: "everything" }), /loopExpect\.fix must be "smart" or an array of/], + ["fix an unknown level", expectSmart({ fix: ["critical", "major"] }), /loopExpect\.fix must be "smart" or an array of/], + ["guess not a boolean", expectSmart({ guess: 1 }), /loopExpect\.guess must be true or false/], + ]; + for (const [what, stamp, wanted] of cases) { + const { problems } = tryPatch(loopState(), { loopExpect: stamp }); + assert.ok(problems.some((p) => wanted.test(p)), `${what}: expected ${wanted}, got ${JSON.stringify(problems)}`); + } +}); + +test("a loopOutcome is held to the loopExpect the app stamped", () => { + const smart = withExpect(expectSmart()); + // The shape that stopped a real phase: a review run with no triage block. + assert.match(tryPatch(smart, { loopOutcome: goodOutcome() }).problems.join("\n"), /loopOutcome\.review\.triage\.ran must be true: Smart pick/); + const noTriage = withTriage([]); + noTriage.review.triage.ran = false; + assert.match(tryPatch(smart, { loopOutcome: noTriage }).problems.join("\n"), /loopOutcome\.review\.triage\.ran must be true: Smart pick/); + assert.match(tryPatch(smart, { loopOutcome: withTriage([skip({ level: "critical" })]) }).problems.join("\n"), /loopOutcome\.review\.triage\.skipped must not hold a skipped Critical/); + const noRecheck = withTriage([skip()]); + noRecheck.review.recheck = false; + assert.match(tryPatch(smart, { loopOutcome: noRecheck }).problems.join("\n"), /loopOutcome\.review\.recheck must be true/); + const noReview = withTriage([skip()]); + noReview.review.ran = false; + assert.match(tryPatch(smart, { loopOutcome: noReview }).problems.join("\n"), /loopOutcome\.review\.ran must be true: this loop asked for a review/); + assert.deepEqual(tryPatch(smart, { loopOutcome: withTriage([skip()]) }).problems, [], "triage ran, a warning skipped, the re-check done"); +}); + +test("the loopExpect checks follow the stamped setup: levels, no review, no guess", () => { + // Levels: a level list needs the re-check but never a triage block. + const levels = withExpect({ review: true, fix: ["warning"], guess: true }); + assert.deepEqual(tryPatch(levels, { loopOutcome: goodOutcome() }).problems, [], "a triage block is only a Smart pick need"); + const noRecheck = goodOutcome(); + noRecheck.review.recheck = false; + assert.match(tryPatch(levels, { loopOutcome: noRecheck }).problems.join("\n"), /loopOutcome\.review\.recheck must be true/); + // An empty level list means findings are reported, not fixed: no re-check. + const reportOnly = withExpect({ review: true, fix: [], guess: true }); + assert.deepEqual(tryPatch(reportOnly, { loopOutcome: noRecheck }).problems, []); + // No review at all: review.ran may be false (a plain Implement). + const noReview = goodOutcome(); + noReview.review.ran = false; + const build = withExpect({ review: false, fix: [], guess: true }); + assert.deepEqual(tryPatch(build, { loopOutcome: noReview }).problems, []); + // Guesses not allowed: an assumed task refuses the report. + const noGuess = withExpect(expectSmart({ guess: false })); + assert.match(tryPatch(noGuess, { loopOutcome: withTriage([skip()]) }).problems.join("\n"), /loopOutcome\.tasks\.assumed must be empty: this loop does not allow best guesses/); + const noneAssumed = withTriage([skip()]); + noneAssumed.tasks.assumed = []; + assert.deepEqual(tryPatch(noGuess, { loopOutcome: noneAssumed }).problems, [], "no assumptions, nothing refused"); +}); + +test("a loopExpect through post is stored, and a report that misses it is refused with the field named", () => { + const s = openLoop("implement-review", "expect-post"); + try { + const stamped = post(s, { loopExpect: expectSmart() }); + assert.equal(stamped.status, 0, stamped.stderr); + assert.deepEqual(readState(s.session).loopExpect, expectSmart()); + + const refused = post(s, { loopOutcome: goodOutcome() }); + assert.equal(refused.status, 3); + assert.match(refused.stderr, /loopOutcome\.review\.triage\.ran must be true: Smart pick/); + assert.equal("loopOutcome" in readState(s.session), false, "the bad report was not written"); + + const stored = post(s, { loopOutcome: withTriage([skip()]) }); + assert.equal(stored.status, 0, stored.stderr); + assert.equal("loopOutcome" in readState(s.session), true, "the corrected report lands"); + } finally { + s.stop(); + } +}); + const goodEnd = (extra = {}) => ({ kind: "stopped", line: "Stopped at phase 2: a Critical finding was skipped", diff --git a/src/web-console/building.md b/src/web-console/building.md index 6cb7437..c78c395 100644 --- a/src/web-console/building.md +++ b/src/web-console/building.md @@ -42,7 +42,7 @@ Publish it with `post --wait`. A question stops the build: never post `"status": | A spec gap only they can close | `text` | | | Something learned for AGENTS.md | `confirm` | proposed lines in `consequences` | -Titles stay plain: "Which phase next", "Tests are failing", "Keep going?". In loop mode (`notes/loop.md`) a question is held or stopped by the app, never answered by it; the findings pick and fixes are `handledBy:"loop"` cards. +Titles stay plain: "Which phase next", "Tests are failing", "Keep going?". In loop mode (`notes/loop.md`) a question is held or stopped by the app, never answered by it; the findings pick and fixes are `handledBy:"loop"` cards. A `loopOutcome` refused at post time names the missing piece against the app's `loopExpect`: do that step and post again. ## Stop during a build diff --git a/src/web-console/notes/loop.md b/src/web-console/notes/loop.md index 498de97..c519f57 100644 --- a/src/web-console/notes/loop.md +++ b/src/web-console/notes/loop.md @@ -177,6 +177,15 @@ At the end of the run, post once a top-level `loopOutcome`: - `subject` is one imperative line under 100 characters saying what changed (it becomes the commit subject). `summary` is one line under 200 characters. +The app stamps `loopExpect` into the session as the phase starts: the criteria +it will judge the report by (`review`, `fix` as `"smart"` or the level list, +`guess`). It is the app's field, never yours to post. The console holds every +`loopOutcome` to it, so a report missing a required piece (`review.ran: false` +on a review run, no `review.triage.ran` under Smart pick, a skipped Critical in +the triage, no re-check with fixes chosen, assumed tasks without `--guess`) is +refused at post time with the field named, while the run can still do that +step and repost. + Then post `{"agent":{"status":"waiting"}}` and keep waiting with the usual bounded `wait` slices until the app ends the session. Never end the turn on text. If the post is refused (exit 3), fix the field it names and post again. diff --git a/src/web-console/patches.mjs b/src/web-console/patches.mjs index 57afc55..6063114 100644 --- a/src/web-console/patches.mjs +++ b/src/web-console/patches.mjs @@ -518,6 +518,9 @@ export function applyPatch(state, patch) { if (Object.hasOwn(patch, "loop")) throw new Error(rule("", "loop", "is set by open --loop, not by a patch")); for (const [k, v] of Object.entries(patch)) { if (v === null) { + // The loop's report fields are set, never deleted: a null here would + // strip the app's criteria or an already-posted report. + if (["loopExpect", "loopOutcome", "loopEnd"].includes(k)) throw new Error(rule("", k, "cannot be deleted by a patch")); delete next[k]; continue; } @@ -578,6 +581,20 @@ export function applyPatch(state, patch) { if (state.loopOutcome !== undefined) throw new Error(rule("", "loopOutcome", "was already posted for this session")); next.loopOutcome = v; break; + case "loopExpect": + // The app's judging criteria for the run's report: a `loopOutcome` is + // held to them in validate(), so a report missing a required piece is + // refused here while the agent can still fix it. The app owns the + // field and stamps once per session (an unlock or a hop clears it + // first); a second stamp, from anyone, is refused while the first is + // valid, but a corrupt stored stamp can be overwritten. An early + // self-stamp only weakens this early check: the driver's verdict is + // unchanged. + if (!state.loop?.driving) throw new Error(rule("", "loopExpect", "is only accepted in a loop session")); + if (isPlainObject(state.loopExpect) && loopExpectProblems(state.loopExpect).length === 0) throw new Error(rule("", "loopExpect", "was already stamped for this session")); + if (state.loopOutcome !== undefined) throw new Error(rule("", "loopExpect", "came after the run's report; post it while the phase builds")); + next.loopExpect = v; + break; case "loopEnd": // The app's word on how the loop ended; a later one replaces the earlier. next.loopEnd = v; @@ -826,8 +843,10 @@ export function validate(state) { } } - // The loop's two reports: the skill's outcome and the app's ending. - if (state.loopOutcome !== undefined) problems.push(...loopOutcomeProblems(state.loopOutcome)); + // The loop's reports: the app's criteria, the skill's outcome and the + // app's ending. + if (state.loopExpect !== undefined) problems.push(...loopExpectProblems(state.loopExpect)); + if (state.loopOutcome !== undefined) problems.push(...loopOutcomeProblems(state.loopOutcome, state.loopExpect)); if (state.loopEnd !== undefined) problems.push(...loopEndProblems(state.loopEnd, state)); // A note on a card is unanswered the moment no agent words follow it in its @@ -1067,13 +1086,29 @@ const isPosInt = (n) => Number.isInteger(n) && n > 0; // One line of 1..max characters. const oneLine = (s, max) => typeof s === "string" && s.length >= 1 && s.length <= max && !/[\r\n]/.test(s); +/** + * Problems with a `loopExpect` (the app's judging criteria for the run's + * report), as sentences naming the field; an empty array when it is good. + */ +export function loopExpectProblems(e) { + const p = []; + if (!isPlainObject(e)) return ["loopExpect must be an object"]; + if (typeof e.review !== "boolean") p.push("loopExpect.review must be true or false"); + if (e.fix !== "smart" && !(Array.isArray(e.fix) && e.fix.every((l) => LOOP_LEVELS.includes(l)))) { + p.push(`loopExpect.fix must be "smart" or an array of ${LOOP_LEVELS.join(", ")}`); + } + if (typeof e.guess !== "boolean") p.push("loopExpect.guess must be true or false"); + return p; +} + /** * Problems with a `loopOutcome` (what the skill reports after building a phase * under the loop), as sentences naming the field; an empty array when it is * good. Rules are the overview's API section; house rules 4 and 5 apply to - * every string. + * every string. `expect` is the app's `loopExpect`, when the session holds + * one: the report is then also held to what this loop asked for. */ -export function loopOutcomeProblems(o) { +export function loopOutcomeProblems(o, expect) { const p = []; const bad = (field, rule) => p.push(`loopOutcome.${field} ${rule}`); if (!isPlainObject(o)) return ["loopOutcome must be an object"]; @@ -1178,6 +1213,43 @@ export function loopOutcomeProblems(o) { if (!oneLine(o.subject, 100)) bad("subject", "must be one line of 1 to 100 characters"); else prose("subject", o.subject); + + // A report that misses what the app said it would judge by is refused here, + // while the run can still do the step and repost, rather than stopping the + // phase on the driver's verdict later. The driver checks the criteria only + // on a "passed" report, so a "failed" one always lands (its reason is the + // phase's own word). + if (isPlainObject(expect) && o.result === "passed") { + if (isPlainObject(o.tests) && o.tests.ran === true && o.tests.passed === false) { + bad("tests.passed", "must be true on a passed report: the app refuses failed tests"); + } + if (expect.review === true && isPlainObject(o.review)) { + if (o.review.ran === false) bad("review.ran", "must be true: this loop asked for a review"); + if (expect.fix === "smart") { + if (!isPlainObject(o.review.triage) || o.review.triage.ran !== true) { + bad("review.triage.ran", "must be true: Smart pick requires the triage and its skipped list in the report"); + } else if (Array.isArray(o.review.triage.skipped) && o.review.triage.skipped.some((u) => isPlainObject(u) && u.level === "critical")) { + bad("review.triage.skipped", "must not hold a skipped Critical: the app stops the phase on one"); + } + } + const chosen = expect.fix === "smart" ? LOOP_LEVELS : Array.isArray(expect.fix) ? expect.fix : []; + if (chosen.length > 0 && o.review.recheck === false) { + bad("review.recheck", "must be true: findings at the loop's chosen levels need the re-check"); + } + // The driver's `leftover()`: the counter and the unfixed list, whichever + // is larger; a leftover at a chosen Critical or Warning fails the phase. + const left = (level) => + Math.max(o.review.left?.[level] || 0, (Array.isArray(o.review.unfixed) ? o.review.unfixed : []).filter((u) => isPlainObject(u) && u.level === level).length); + for (const level of ["critical", "warning"]) { + if (chosen.includes(level) && left(level) > 0) bad(`review.left.${level}`, "must be 0: a leftover at a chosen level fails the phase"); + } + } + // The report's own claim only: the `[?]` marks in the spec stay the + // driver's call (`specs.mjs status`), which this post-time check cannot see. + if (expect.guess === false && isPlainObject(o.tasks) && Array.isArray(o.tasks.assumed) && o.tasks.assumed.length > 0) { + bad("tasks.assumed", "must be empty: this loop does not allow best guesses"); + } + } return p; } @@ -1225,9 +1297,10 @@ export function loopEndProblems(e, state) { return p; } -// Loop mode adds three optional keys that a blank state does not hold (absent, -// not null): `loop: { driving: true }` (set by `open --loop`), `loopOutcome` -// (the skill's report) and `loopEnd` (the app's). +// Loop mode adds four optional keys that a blank state does not hold (absent, +// not null): `loop: { driving: true }` (set by `open --loop`), `loopExpect` +// (the app's judging criteria), `loopOutcome` (the skill's report) and +// `loopEnd` (the app's ending). export function blankState({ sid, workflow, title, project, specDir }) { return { schema: SCHEMA, @@ -1261,9 +1334,10 @@ export function blankState({ sid, workflow, title, project, specDir }) { */ export function switchWorkflow(state, { workflow, title, specDir }) { const next = { ...blankState({ sid: state.sid, project: state.project, workflow, title, specDir }), created: state.created }; - // A blank state holds no `loop`, `loopOutcome` or `loopEnd`, so a hop never - // carries them over; `open --loop` stamps `loop` again after the switch. - for (const key of ["loop", "loopOutcome", "loopEnd"]) delete next[key]; + // A blank state holds no `loop`, `loopExpect`, `loopOutcome` or `loopEnd`, + // so a hop never carries them over; `open --loop` stamps `loop` again after + // the switch, and the app restamps `loopExpect` on its next look. + for (const key of ["loop", "loopExpect", "loopOutcome", "loopEnd"]) delete next[key]; if (state.chat) next.chat = state.chat; return next; }