plan2code-bot: replace auto-responder with LLM-as-judge evaluation

Bot now acts as an authentic QA agent instead of rubber-stamping every
question and step.

- intelligent-responder answers AskUserQuestion via LLM using current
  observations (tools used, files created, errors) instead of keyword
  matching; auto-responder removed.
- evaluator runs after each step with step-specific criteria
  (prompts/evaluation-criteria.ts), produces 0-100 score plus
  strengths/weaknesses/critical issues. Avg <60 blocks finalization.
- observation-collector captures tool_use, file writes, errors, and
  question reasoning; session-runner falls back to scraping tool_use
  blocks when canUseTool doesn't fire and dedupes both sources.
- Bot writes BOT-EVALUATION.md and BOT-NOTES.md for metrics analysis.
- Idea generator: 12 categories instead of CLI/web-app coin flip,
  stronger seed adherence, less developer-tool bias.
- Init step now writes a minimal AGENTS.md stub instead of running
  the full init skill, avoiding hallucinated architecture before plan.
- Implement step uses Read/Write/Edit/Glob/Grep directly instead of
  a Skill sub-session that produced no visible tool observations.
- bin: add --help, accept --idea="value" form, strip surrounding
  quotes; evaluator maxTurns 3 -> 30; warn when parser misses SCORE.
This commit is contained in:
2026-05-11 13:14:28 -07:00
parent 9d1104d105
commit c2579a7b6a
15 changed files with 1434 additions and 254 deletions
+43
View File
@@ -10,12 +10,55 @@ export interface BotConfig {
export type StepName = 'init' | 'plan' | 'document' | 'implement' | 'finalize';
export interface ToolObservation {
toolName: string;
input: Record<string, unknown>;
output?: unknown;
timestamp: number;
allowed: boolean;
autoAnswered?: boolean;
}
export interface QuestionContext {
question: string;
options: Array<{ label: string; description: string }>;
llmReasoning: string;
selectedAnswer: string;
timestamp: number;
}
export interface ExecutionObservation {
step: StepName;
startTime: number;
endTime: number;
tools: ToolObservation[];
assistantMessages: string[];
errors: string[];
filesCreated: string[];
filesModified: string[];
questionsAsked: QuestionContext[];
}
export interface EvaluationResult {
step: StepName;
score: number;
strengths: string[];
weaknesses: string[];
suggestions: string[];
criticalIssues: string[];
timestamp: number;
evaluatorModel: string;
reasoning: string;
}
export interface StepResult {
step: StepName;
success: boolean;
sessionId: string | null;
duration: number;
error: string | null;
evaluation?: EvaluationResult;
observations?: ExecutionObservation;
}
export interface BotState {