Files
plan2code/plan2code-bot/src/prompts/evaluation-criteria.ts
T
jparkerweb 1d75934e45 plan2code-bot: replace auto-responder with LLM-as-judge evaluation
Bot now acts as an authentic QA agent instead of rubber-stamping every
question and step.

- intelligent-responder answers AskUserQuestion via LLM using current
  observations (tools used, files created, errors) instead of keyword
  matching; auto-responder removed.
- evaluator runs after each step with step-specific criteria
  (prompts/evaluation-criteria.ts), produces 0-100 score plus
  strengths/weaknesses/critical issues. Avg <60 blocks finalization.
- observation-collector captures tool_use, file writes, errors, and
  question reasoning; session-runner falls back to scraping tool_use
  blocks when canUseTool doesn't fire and dedupes both sources.
- Bot writes BOT-EVALUATION.md and BOT-NOTES.md for metrics analysis.
- Idea generator: 12 categories instead of CLI/web-app coin flip,
  stronger seed adherence, less developer-tool bias.
- Init step now writes a minimal AGENTS.md stub instead of running
  the full init skill, avoiding hallucinated architecture before plan.
- Implement step uses Read/Write/Edit/Glob/Grep directly instead of
  a Skill sub-session that produced no visible tool observations.
- bin: add --help, accept --idea="value" form, strip surrounding
  quotes; evaluator maxTurns 3 -> 30; warn when parser misses SCORE.
2026-05-11 13:14:28 -07:00

159 lines
6.3 KiB
TypeScript

import type { StepName } from '../types.js';
export interface StepEvaluationCriteria {
step: StepName;
keyArtifacts: string[];
qualityChecks: string[];
commonPitfalls: string[];
scoringGuidance: string;
}
export const EVALUATION_CRITERIA: Record<StepName, StepEvaluationCriteria> = {
init: {
step: 'init',
keyArtifacts: ['AGENTS.md', 'IDEA.md'],
qualityChecks: [
'AGENTS.md exists with project name and description',
'AGENTS.md includes a brief intro line and a Status section indicating the project is in planning phase',
'AGENTS.md does NOT contain hallucinated architecture, commands, file structures, or tech stack details',
'No .agents-docs/ directory was created (too early for detail files)',
'No project scaffolding (package.json, dependencies, src/) was created',
'IDEA.md exists with the project idea',
],
commonPitfalls: [
'Hallucinating architecture or tech stack details before the plan step',
'Creating .agents-docs/ detail files with invented content',
'Scaffolding project files or installing dependencies prematurely',
],
scoringGuidance: `
100 = Perfect: AGENTS.md stub with intro line, project name/description, and status section. IDEA.md present. Nothing else created.
85-95 = Good but minor extra content beyond the expected stub format (e.g., an extra placeholder heading)
70-84 = AGENTS.md exists but includes some hallucinated details (e.g., assumed tech stack or commands)
50-69 = Significant hallucination (e.g., .agents-docs/ created with invented content, project scaffolded)
<50 = Major problems (e.g., AGENTS.md missing, full project structure hallucinated)
The expected AGENTS.md format is: intro line, Project Overview (name + description), and a Status section. This is the target for a 100 score.
`,
},
plan: {
step: 'plan',
keyArtifacts: ['specs/*/PLAN-DRAFT-*.md', 'IDEA.md'],
qualityChecks: [
'Plan breaks work into clear, achievable phases',
'Each phase has specific goals and deliverables',
'Technical approach is appropriate',
'Scope is realistic for the idea',
'Dependencies between phases are identified',
],
commonPitfalls: [
'Phases too vague ("polish the app")',
'Missing specific tasks within phases',
'No testing strategy mentioned',
'Overly ambitious scope',
'Missing file paths or specific actions',
],
scoringGuidance: `
100 = Exceptional plan: detailed phases, realistic scope, clear tasks, testing included
85-95 = Good plan with minor improvements possible (e.g., one phase could be more specific)
70-84 = Acceptable but has vague sections or missing testing strategy
50-69 = Significant issues (e.g., multiple vague phases, unrealistic scope)
<50 = Major problems (e.g., no clear phases, plan doesn't match idea)
Most plans should score 70-85. Be critical of vague language.
`,
},
document: {
step: 'document',
keyArtifacts: ['specs/*/overview.md', 'specs/*/phase-*.md files'],
qualityChecks: [
'overview.md provides clear project summary',
'Each phase file has specific tasks with checkboxes',
'File paths are explicit (not generic)',
'Dependencies between tasks are identified',
'Technical details are specific',
'Acceptance criteria are clear',
],
commonPitfalls: [
'Tasks too generic ("implement feature X")',
'Missing file paths',
'No checkboxes or unclear task structure',
'Missing dependencies',
'Overly verbose or lacking specifics',
],
scoringGuidance: `
100 = Exceptional documentation: specific tasks, explicit file paths, clear dependencies
85-95 = Good documentation with minor vagueness in one or two tasks
70-84 = Acceptable but multiple tasks lack specifics or file paths
50-69 = Significant issues (e.g., many generic tasks, missing file paths)
<50 = Major problems (e.g., tasks don't match plan, fundamentally vague)
Most documentation should score 70-85. Penalize generic language heavily.
`,
},
implement: {
step: 'implement',
keyArtifacts: ['actual code files', 'checked-off tasks in phase files'],
qualityChecks: [
'Phase tasks are being completed',
'Code files are actually created/modified',
'Implementation follows the documented plan',
'No major errors blocking progress',
'Tests are written (if applicable)',
],
commonPitfalls: [
'Tasks marked complete but files not actually changed',
'Implementation deviates significantly from plan',
'Errors not addressed',
'Skipping tests without justification',
'Working on wrong phase',
],
scoringGuidance: `
100 = Exceptional implementation: all tasks complete, code works, tests pass
85-95 = Good implementation with minor issues or incomplete tests
70-84 = Acceptable but some tasks incomplete or code has issues
50-69 = Significant issues (e.g., many tasks incomplete, code doesn't work)
<50 = Major problems (e.g., wrong phase, no actual work done)
Implementation scoring depends heavily on actual progress. Be realistic.
`,
},
finalize: {
step: 'finalize',
keyArtifacts: ['specs--completed/', 'README or docs', 'final code state'],
qualityChecks: [
'All phases are marked complete',
'Spec moved to specs--completed/',
'Documentation is updated',
'Code is in working state',
'No obvious loose ends',
],
commonPitfalls: [
'Incomplete phases',
'Missing specs--completed/ move',
'Documentation not updated',
'Code broken or incomplete',
'Unrealistic self-assessment',
],
scoringGuidance: `
100 = Exceptional finalization: everything complete, polished, documented
85-95 = Good finalization with minor issues
70-84 = Acceptable but some loose ends or documentation gaps
50-69 = Significant issues (e.g., incomplete phases, broken code)
<50 = Major problems (e.g., work not actually done, fundamentally incomplete)
Finalize scores should reflect overall project quality. Be honest.
`,
},
};
/**
* Gets evaluation criteria for a specific step.
*/
export function getCriteriaForStep(step: StepName): StepEvaluationCriteria {
return EVALUATION_CRITERIA[step];
}