Files
plan2code/plan2code-bot/src/session-runner.ts
T
jparkerweb c2579a7b6a plan2code-bot: replace auto-responder with LLM-as-judge evaluation
Bot now acts as an authentic QA agent instead of rubber-stamping every
question and step.

- intelligent-responder answers AskUserQuestion via LLM using current
  observations (tools used, files created, errors) instead of keyword
  matching; auto-responder removed.
- evaluator runs after each step with step-specific criteria
  (prompts/evaluation-criteria.ts), produces 0-100 score plus
  strengths/weaknesses/critical issues. Avg <60 blocks finalization.
- observation-collector captures tool_use, file writes, errors, and
  question reasoning; session-runner falls back to scraping tool_use
  blocks when canUseTool doesn't fire and dedupes both sources.
- Bot writes BOT-EVALUATION.md and BOT-NOTES.md for metrics analysis.
- Idea generator: 12 categories instead of CLI/web-app coin flip,
  stronger seed adherence, less developer-tool bias.
- Init step now writes a minimal AGENTS.md stub instead of running
  the full init skill, avoiding hallucinated architecture before plan.
- Implement step uses Read/Write/Edit/Glob/Grep directly instead of
  a Skill sub-session that produced no visible tool observations.
- bin: add --help, accept --idea="value" form, strip surrounding
  quotes; evaluator maxTurns 3 -> 30; warn when parser misses SCORE.
2026-05-11 13:14:28 -07:00

78 lines
2.5 KiB
TypeScript

import { query } from '@anthropic-ai/claude-agent-sdk';
import { createIntelligentResponder } from './intelligent-responder.js';
import type { BotConfig, ExecutionObservation, StepName } from './types.js';
import type { ObservationCollector } from './observation-collector.js';
export interface SessionOptions {
prompt: string;
config: BotConfig;
step: StepName;
maxTurns?: number;
collector: ObservationCollector;
}
export interface SessionResult {
sessionId: string | null;
output: string;
success: boolean;
duration: number;
observations: ExecutionObservation;
}
export async function runSession(options: SessionOptions): Promise<SessionResult> {
const { prompt, config, step, maxTurns = 50, collector } = options;
const startTime = Date.now();
let output = '';
let sessionId: string | null = null;
try {
const session = query({
prompt,
options: {
cwd: config.projectDir,
maxTurns,
permissionMode: 'bypassPermissions',
allowDangerouslySkipPermissions: true,
canUseTool: createIntelligentResponder(config, step, collector),
systemPrompt: { type: 'preset', preset: 'claude_code' },
settingSources: ['project'],
},
});
for await (const message of session) {
// Record all messages for observations
collector.recordMessage(message);
if (message.type === 'assistant') {
sessionId = message.session_id ?? sessionId;
for (const block of message.message.content) {
if (block.type === 'text') {
output += block.text + '\n';
} else if (block.type === 'tool_use') {
// Capture tool invocations from the message stream as a fallback
// in case canUseTool doesn't fire (e.g., Skill sub-sessions)
collector.recordToolUse(
block.name,
block.input as Record<string, unknown>,
undefined,
true
);
}
}
} else if (message.type === 'result') {
sessionId = message.session_id ?? sessionId;
}
}
const duration = Date.now() - startTime;
const hasOutput = output.trim().length > 0;
const observations = collector.finalize();
return { sessionId, output, success: hasOutput, duration, observations };
} catch (error) {
const duration = Date.now() - startTime;
const errorMsg = error instanceof Error ? error.message : String(error);
const observations = collector.finalize();
return { sessionId, output, success: false, duration, observations };
}
}