mirror of
https://github.com/jparkerweb/plan2code.git
synced 2026-07-21 18:33:22 -07:00
plan2code-bot: replace auto-responder with LLM-as-judge evaluation
Bot now acts as an authentic QA agent instead of rubber-stamping every question and step. - intelligent-responder answers AskUserQuestion via LLM using current observations (tools used, files created, errors) instead of keyword matching; auto-responder removed. - evaluator runs after each step with step-specific criteria (prompts/evaluation-criteria.ts), produces 0-100 score plus strengths/weaknesses/critical issues. Avg <60 blocks finalization. - observation-collector captures tool_use, file writes, errors, and question reasoning; session-runner falls back to scraping tool_use blocks when canUseTool doesn't fire and dedupes both sources. - Bot writes BOT-EVALUATION.md and BOT-NOTES.md for metrics analysis. - Idea generator: 12 categories instead of CLI/web-app coin flip, stronger seed adherence, less developer-tool bias. - Init step now writes a minimal AGENTS.md stub instead of running the full init skill, avoiding hallucinated architecture before plan. - Implement step uses Read/Write/Edit/Glob/Grep directly instead of a Skill sub-session that produced no visible tool observations. - bin: add --help, accept --idea="value" form, strip surrounding quotes; evaluator maxTurns 3 -> 30; warn when parser misses SCORE.
This commit is contained in:
@@ -0,0 +1,158 @@
|
||||
import type { StepName } from '../types.js';
|
||||
|
||||
export interface StepEvaluationCriteria {
|
||||
step: StepName;
|
||||
keyArtifacts: string[];
|
||||
qualityChecks: string[];
|
||||
commonPitfalls: string[];
|
||||
scoringGuidance: string;
|
||||
}
|
||||
|
||||
export const EVALUATION_CRITERIA: Record<StepName, StepEvaluationCriteria> = {
|
||||
init: {
|
||||
step: 'init',
|
||||
keyArtifacts: ['AGENTS.md', 'IDEA.md'],
|
||||
qualityChecks: [
|
||||
'AGENTS.md exists with project name and description',
|
||||
'AGENTS.md includes a brief intro line and a Status section indicating the project is in planning phase',
|
||||
'AGENTS.md does NOT contain hallucinated architecture, commands, file structures, or tech stack details',
|
||||
'No .agents-docs/ directory was created (too early for detail files)',
|
||||
'No project scaffolding (package.json, dependencies, src/) was created',
|
||||
'IDEA.md exists with the project idea',
|
||||
],
|
||||
commonPitfalls: [
|
||||
'Hallucinating architecture or tech stack details before the plan step',
|
||||
'Creating .agents-docs/ detail files with invented content',
|
||||
'Scaffolding project files or installing dependencies prematurely',
|
||||
],
|
||||
scoringGuidance: `
|
||||
100 = Perfect: AGENTS.md stub with intro line, project name/description, and status section. IDEA.md present. Nothing else created.
|
||||
85-95 = Good but minor extra content beyond the expected stub format (e.g., an extra placeholder heading)
|
||||
70-84 = AGENTS.md exists but includes some hallucinated details (e.g., assumed tech stack or commands)
|
||||
50-69 = Significant hallucination (e.g., .agents-docs/ created with invented content, project scaffolded)
|
||||
<50 = Major problems (e.g., AGENTS.md missing, full project structure hallucinated)
|
||||
|
||||
The expected AGENTS.md format is: intro line, Project Overview (name + description), and a Status section. This is the target for a 100 score.
|
||||
`,
|
||||
},
|
||||
|
||||
plan: {
|
||||
step: 'plan',
|
||||
keyArtifacts: ['specs/*/PLAN-DRAFT-*.md', 'IDEA.md'],
|
||||
qualityChecks: [
|
||||
'Plan breaks work into clear, achievable phases',
|
||||
'Each phase has specific goals and deliverables',
|
||||
'Technical approach is appropriate',
|
||||
'Scope is realistic for the idea',
|
||||
'Dependencies between phases are identified',
|
||||
],
|
||||
commonPitfalls: [
|
||||
'Phases too vague ("polish the app")',
|
||||
'Missing specific tasks within phases',
|
||||
'No testing strategy mentioned',
|
||||
'Overly ambitious scope',
|
||||
'Missing file paths or specific actions',
|
||||
],
|
||||
scoringGuidance: `
|
||||
100 = Exceptional plan: detailed phases, realistic scope, clear tasks, testing included
|
||||
85-95 = Good plan with minor improvements possible (e.g., one phase could be more specific)
|
||||
70-84 = Acceptable but has vague sections or missing testing strategy
|
||||
50-69 = Significant issues (e.g., multiple vague phases, unrealistic scope)
|
||||
<50 = Major problems (e.g., no clear phases, plan doesn't match idea)
|
||||
|
||||
Most plans should score 70-85. Be critical of vague language.
|
||||
`,
|
||||
},
|
||||
|
||||
document: {
|
||||
step: 'document',
|
||||
keyArtifacts: ['specs/*/overview.md', 'specs/*/phase-*.md files'],
|
||||
qualityChecks: [
|
||||
'overview.md provides clear project summary',
|
||||
'Each phase file has specific tasks with checkboxes',
|
||||
'File paths are explicit (not generic)',
|
||||
'Dependencies between tasks are identified',
|
||||
'Technical details are specific',
|
||||
'Acceptance criteria are clear',
|
||||
],
|
||||
commonPitfalls: [
|
||||
'Tasks too generic ("implement feature X")',
|
||||
'Missing file paths',
|
||||
'No checkboxes or unclear task structure',
|
||||
'Missing dependencies',
|
||||
'Overly verbose or lacking specifics',
|
||||
],
|
||||
scoringGuidance: `
|
||||
100 = Exceptional documentation: specific tasks, explicit file paths, clear dependencies
|
||||
85-95 = Good documentation with minor vagueness in one or two tasks
|
||||
70-84 = Acceptable but multiple tasks lack specifics or file paths
|
||||
50-69 = Significant issues (e.g., many generic tasks, missing file paths)
|
||||
<50 = Major problems (e.g., tasks don't match plan, fundamentally vague)
|
||||
|
||||
Most documentation should score 70-85. Penalize generic language heavily.
|
||||
`,
|
||||
},
|
||||
|
||||
implement: {
|
||||
step: 'implement',
|
||||
keyArtifacts: ['actual code files', 'checked-off tasks in phase files'],
|
||||
qualityChecks: [
|
||||
'Phase tasks are being completed',
|
||||
'Code files are actually created/modified',
|
||||
'Implementation follows the documented plan',
|
||||
'No major errors blocking progress',
|
||||
'Tests are written (if applicable)',
|
||||
],
|
||||
commonPitfalls: [
|
||||
'Tasks marked complete but files not actually changed',
|
||||
'Implementation deviates significantly from plan',
|
||||
'Errors not addressed',
|
||||
'Skipping tests without justification',
|
||||
'Working on wrong phase',
|
||||
],
|
||||
scoringGuidance: `
|
||||
100 = Exceptional implementation: all tasks complete, code works, tests pass
|
||||
85-95 = Good implementation with minor issues or incomplete tests
|
||||
70-84 = Acceptable but some tasks incomplete or code has issues
|
||||
50-69 = Significant issues (e.g., many tasks incomplete, code doesn't work)
|
||||
<50 = Major problems (e.g., wrong phase, no actual work done)
|
||||
|
||||
Implementation scoring depends heavily on actual progress. Be realistic.
|
||||
`,
|
||||
},
|
||||
|
||||
finalize: {
|
||||
step: 'finalize',
|
||||
keyArtifacts: ['specs--completed/', 'README or docs', 'final code state'],
|
||||
qualityChecks: [
|
||||
'All phases are marked complete',
|
||||
'Spec moved to specs--completed/',
|
||||
'Documentation is updated',
|
||||
'Code is in working state',
|
||||
'No obvious loose ends',
|
||||
],
|
||||
commonPitfalls: [
|
||||
'Incomplete phases',
|
||||
'Missing specs--completed/ move',
|
||||
'Documentation not updated',
|
||||
'Code broken or incomplete',
|
||||
'Unrealistic self-assessment',
|
||||
],
|
||||
scoringGuidance: `
|
||||
100 = Exceptional finalization: everything complete, polished, documented
|
||||
85-95 = Good finalization with minor issues
|
||||
70-84 = Acceptable but some loose ends or documentation gaps
|
||||
50-69 = Significant issues (e.g., incomplete phases, broken code)
|
||||
<50 = Major problems (e.g., work not actually done, fundamentally incomplete)
|
||||
|
||||
Finalize scores should reflect overall project quality. Be honest.
|
||||
`,
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Gets evaluation criteria for a specific step.
|
||||
*/
|
||||
export function getCriteriaForStep(step: StepName): StepEvaluationCriteria {
|
||||
return EVALUATION_CRITERIA[step];
|
||||
}
|
||||
@@ -20,15 +20,37 @@ export function buildStepPrompt(step: StepName, config: BotConfig): string {
|
||||
function buildInitPrompt(config: BotConfig): string {
|
||||
return `${AUTONOMOUS_PREAMBLE}
|
||||
|
||||
Run the /plan2code-init skill to generate an AGENTS.md file for this project.
|
||||
This is a brand new project with no existing code. Create a minimal stub AGENTS.md file and an IDEA.md file. Do NOT run the /plan2code-init skill � there is no codebase to analyze yet.
|
||||
|
||||
The project idea is: ${config.ideaDescription}
|
||||
The project name is: ${config.ideaName}
|
||||
|
||||
When asked questions:
|
||||
- For the project description, use the idea above
|
||||
- Accept all defaults and approve all gates
|
||||
- Complete the init process fully`;
|
||||
## What to create
|
||||
|
||||
### AGENTS.md
|
||||
Create a minimal stub with ONLY the following � do NOT invent architecture, tech stack details, commands, or file structures:
|
||||
|
||||
\`\`\`markdown
|
||||
# AGENTS.md
|
||||
|
||||
This file provides guidance to AI coding agents when working with code in this repository.
|
||||
|
||||
## Project Overview
|
||||
**Name:** ${config.ideaName}
|
||||
**Description:** ${config.ideaDescription}
|
||||
|
||||
## Status
|
||||
This project is in the planning phase. Architecture, commands, and detailed documentation will be added after the plan and document steps are complete.
|
||||
\`\`\`
|
||||
|
||||
### IDEA.md
|
||||
If IDEA.md does not already exist, create it with the project name and description.
|
||||
|
||||
## Rules
|
||||
- Do NOT create .agents-docs/ or any detail files � there is nothing to document yet
|
||||
- Do NOT hallucinate architecture, dependencies, file structures, or tech stack choices
|
||||
- Do NOT install dependencies or scaffold project files
|
||||
- ONLY create the two files above`;
|
||||
}
|
||||
|
||||
function buildPlanPrompt(config: BotConfig): string {
|
||||
@@ -63,14 +85,37 @@ When making decisions:
|
||||
function buildImplementPrompt(config: BotConfig): string {
|
||||
return `${AUTONOMOUS_PREAMBLE}
|
||||
|
||||
Run the /plan2code-3-implement skill to implement the next available phase.
|
||||
You are a senior software engineer implementing a project phase. Do NOT use the Skill tool � implement directly using Read, Write, Edit, Glob, and Grep tools.
|
||||
|
||||
When making decisions:
|
||||
- Pick the first available/unchecked phase
|
||||
- Write working code that fulfills the spec tasks
|
||||
- Mark tasks as complete as you finish them
|
||||
- Approve and sign off when the phase is done
|
||||
- Skip running tests if asked (or select the simplest test option)`;
|
||||
## Process
|
||||
|
||||
1. **Find the spec**: Read \`specs/*/overview.md\` to find the phase checklist
|
||||
2. **Pick the next phase**: Find the first phase marked \`[ ]\` (pending) or \`[/]\` (in-progress)
|
||||
3. **Read the phase file**: Read the corresponding \`phase-X.md\` from the same directory
|
||||
4. **Mark phase in-progress**: Update \`[ ]\` to \`[/]\` in overview.md
|
||||
5. **Implement each task sequentially**:
|
||||
- Read the task specification completely
|
||||
- Write the code using Write or Edit tools � create real files, not code blocks
|
||||
- Mark the task \`[x]\` in the phase file immediately after completing it
|
||||
6. **Complete the phase**: After all tasks, fill in the "Phase Completion Summary" in the phase file
|
||||
7. **Mark phase complete**: Update \`[/]\` to \`[x]\` in overview.md
|
||||
|
||||
## Rules
|
||||
|
||||
- Follow AGENTS.md if it exists
|
||||
- Implement specs EXACTLY � no creative additions or unsolicited improvements
|
||||
- Write task completion status (\`[x]\`) to disk immediately after each task � never batch
|
||||
- Only create files mentioned in the spec tasks
|
||||
- Use the specified file paths, function names, and structures from the spec
|
||||
- No placeholder code � fully implement every function
|
||||
- Match existing codebase conventions
|
||||
- Do NOT run git commands
|
||||
- Skip running tests unless explicitly listed as a phase task
|
||||
|
||||
## Project info
|
||||
- Project: ${config.ideaName}
|
||||
- Description: ${config.ideaDescription}
|
||||
- Project directory: ${config.projectDir}`;
|
||||
}
|
||||
|
||||
function buildFinalizePrompt(config: BotConfig): string {
|
||||
|
||||
Reference in New Issue
Block a user