mirror of
https://github.com/jparkerweb/plan2code.git
synced 2026-07-21 18:33:22 -07:00
plan2code-bot: replace auto-responder with LLM-as-judge evaluation
Bot now acts as an authentic QA agent instead of rubber-stamping every question and step. - intelligent-responder answers AskUserQuestion via LLM using current observations (tools used, files created, errors) instead of keyword matching; auto-responder removed. - evaluator runs after each step with step-specific criteria (prompts/evaluation-criteria.ts), produces 0-100 score plus strengths/weaknesses/critical issues. Avg <60 blocks finalization. - observation-collector captures tool_use, file writes, errors, and question reasoning; session-runner falls back to scraping tool_use blocks when canUseTool doesn't fire and dedupes both sources. - Bot writes BOT-EVALUATION.md and BOT-NOTES.md for metrics analysis. - Idea generator: 12 categories instead of CLI/web-app coin flip, stronger seed adherence, less developer-tool bias. - Init step now writes a minimal AGENTS.md stub instead of running the full init skill, avoiding hallucinated architecture before plan. - Implement step uses Read/Write/Edit/Glob/Grep directly instead of a Skill sub-session that produced no visible tool observations. - bin: add --help, accept --idea="value" form, strip surrounding quotes; evaluator maxTurns 3 -> 30; warn when parser misses SCORE.
This commit is contained in:
@@ -10,12 +10,55 @@ export interface BotConfig {
|
||||
|
||||
export type StepName = 'init' | 'plan' | 'document' | 'implement' | 'finalize';
|
||||
|
||||
export interface ToolObservation {
|
||||
toolName: string;
|
||||
input: Record<string, unknown>;
|
||||
output?: unknown;
|
||||
timestamp: number;
|
||||
allowed: boolean;
|
||||
autoAnswered?: boolean;
|
||||
}
|
||||
|
||||
export interface QuestionContext {
|
||||
question: string;
|
||||
options: Array<{ label: string; description: string }>;
|
||||
llmReasoning: string;
|
||||
selectedAnswer: string;
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
export interface ExecutionObservation {
|
||||
step: StepName;
|
||||
startTime: number;
|
||||
endTime: number;
|
||||
tools: ToolObservation[];
|
||||
assistantMessages: string[];
|
||||
errors: string[];
|
||||
filesCreated: string[];
|
||||
filesModified: string[];
|
||||
questionsAsked: QuestionContext[];
|
||||
}
|
||||
|
||||
export interface EvaluationResult {
|
||||
step: StepName;
|
||||
score: number;
|
||||
strengths: string[];
|
||||
weaknesses: string[];
|
||||
suggestions: string[];
|
||||
criticalIssues: string[];
|
||||
timestamp: number;
|
||||
evaluatorModel: string;
|
||||
reasoning: string;
|
||||
}
|
||||
|
||||
export interface StepResult {
|
||||
step: StepName;
|
||||
success: boolean;
|
||||
sessionId: string | null;
|
||||
duration: number;
|
||||
error: string | null;
|
||||
evaluation?: EvaluationResult;
|
||||
observations?: ExecutionObservation;
|
||||
}
|
||||
|
||||
export interface BotState {
|
||||
|
||||
Reference in New Issue
Block a user