plan2code-bot: replace auto-responder with LLM-as-judge evaluation

Bot now acts as an authentic QA agent instead of rubber-stamping every
question and step.

- intelligent-responder answers AskUserQuestion via LLM using current
  observations (tools used, files created, errors) instead of keyword
  matching; auto-responder removed.
- evaluator runs after each step with step-specific criteria
  (prompts/evaluation-criteria.ts), produces 0-100 score plus
  strengths/weaknesses/critical issues. Avg <60 blocks finalization.
- observation-collector captures tool_use, file writes, errors, and
  question reasoning; session-runner falls back to scraping tool_use
  blocks when canUseTool doesn't fire and dedupes both sources.
- Bot writes BOT-EVALUATION.md and BOT-NOTES.md for metrics analysis.
- Idea generator: 12 categories instead of CLI/web-app coin flip,
  stronger seed adherence, less developer-tool bias.
- Init step now writes a minimal AGENTS.md stub instead of running
  the full init skill, avoiding hallucinated architecture before plan.
- Implement step uses Read/Write/Edit/Glob/Grep directly instead of
  a Skill sub-session that produced no visible tool observations.
- bin: add --help, accept --idea="value" form, strip surrounding
  quotes; evaluator maxTurns 3 -> 30; warn when parser misses SCORE.
This commit is contained in:
jparkerweb committed 2026-05-11 13:14:28 -07:00
1 parent 9d1104d105
commit c2579a7b6a
15 files changed
+1434 -254

No files matched your search

+95 -4
View File
@@ -1,16 +1,107 @@
import { runCLI } from '../cli.js';
function parseArgs(): { idea?: string; resume?: boolean } {
function showHelp(): void {
console.log(`
+----------------------------------------------------------------+
� PLAN2CODEDE-BOT �
�----------------------------------------------------------------�
� Autonomous workflow test runner foplan2codede �
� Features LLM-as-judge for honest quality evaluation �
+----------------------------------------------------------------+
Usage:
plan2code-bot [options]
Options:
--help Show this help message
--idea <string> Seed the idea generator with a specific concept
Example: --idea "web app for weather"
Example: --idea="CLI tool for CSV conversion"
--resume Resume a previous incomplete run
Modes:
� New Project Mode - Run from empty directory
The bot generates an app idea, creates a subdirectory, writes
IDEA.md, runs init, then all 4 workflow steps.
� Enhancement Mode - Run from directory with AGENTS.md
The bot scans the existing codebase, proposes an enhancement,
writes IDEA.md, then runs plan through finalize.
Evaluation:
The bot acts as an authentic QA agent, using LLM-based decision
making during execution and providing honest quality assessments
after each step. Results are written to specs/<feature>/BOT-EVALUATION.md
and specs/<feature>/BOT-NOTES.md for metrics analysis.
Examples:
# New project (from empty directory)
plan2code-bot
# Enhancement (from existing project)
cd my-project && plan2code-bot
# With specific idea
plan2code-bot --idea "markdown editor with live preview"
# Resume incomplete run
plan2code-bot --resume
Documentation:
https://jparkerweb.github.io/plan2code
`);
}
function stripQuotes(str: string): string {
// Remove surrounding quotes if present (both single and double)
if ((str.startsWith('"') && str.endsWith('"')) ||
(str.startsWith("'") && str.endsWith("'"))) {
return str.slice(1, -1);
}
return str;
}
function parseArgs(): { idea?: string; resume?: boolean; help?: boolean } {
const args = process.argv.slice(2);
const ideaIdx = args.indexOf('--idea');
const idea = ideaIdx !== -1 && ideaIdx + 1 < args.length ? args[ideaIdx + 1] : undefined;
// Check for --help
if (args.includes('--help') || args.includes('-h')) {
return { help: true };
}
// Parse --idea (supports both --idea="value" and --idea "value")
let idea: string | undefined;
for (let i = 0; i < args.length; i++) {
const arg = args[i];
// Format: --idea="value"
if (arg.startsWith('--idea=')) {
idea = stripQuotes(arg.substring('--idea='.length));
break;
}
// Format: --idea "value"
if (arg === '--idea' && i + 1 < args.length) {
idea = stripQuotes(args[i + 1]);
break;
}
}
// Parse --resume
const resume = args.includes('--resume');
return { idea, resume: resume || undefined };
}
async function main() {
try {
const { idea, resume } = parseArgs();
const { idea, resume, help } = parseArgs();
if (help) {
showHelp();
process.exit(0);
}
await runCLI({ idea, resume });
process.exit(0);
} catch (err) {