mirror of
https://github.com/jparkerweb/plan2code.git
synced 2026-07-21 18:33:22 -07:00
v1.8.0 — add plan2code-metrics recursive self-improvement toolchain
New package for contributors to collect, aggregate, analyze, and improve workflow prompts based on real project data. Includes unit tests (vitest), installer integration, and loop/finalize hints.
This commit is contained in:
@@ -0,0 +1,154 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { avg, rate, buildCohortKey, backfillPromptVersions } from './aggregator.js';
|
||||
import type { PromptVersions } from './types.js';
|
||||
|
||||
// ── avg() ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('avg', () => {
|
||||
it('returns null for empty array', () => {
|
||||
expect(avg([])).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for all-null/undefined values', () => {
|
||||
expect(avg([null, undefined, null])).toBeNull();
|
||||
});
|
||||
|
||||
it('computes correct average for valid numbers', () => {
|
||||
expect(avg([10, 20, 30])).toBe(20);
|
||||
});
|
||||
|
||||
it('filters out null/undefined/NaN from mixed arrays', () => {
|
||||
expect(avg([10, null, 20, undefined, NaN, 30])).toBe(20);
|
||||
});
|
||||
});
|
||||
|
||||
// ── rate() ────────────────────────────────────────────────────────────────────
|
||||
|
||||
describe('rate', () => {
|
||||
it('returns null for empty array', () => {
|
||||
expect(rate([])).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for all-null values', () => {
|
||||
expect(rate([null, null])).toBeNull();
|
||||
});
|
||||
|
||||
it('returns 1.0 for all-true', () => {
|
||||
expect(rate([true, true, true])).toBe(1.0);
|
||||
});
|
||||
|
||||
it('returns 0.0 for all-false', () => {
|
||||
expect(rate([false, false, false])).toBe(0.0);
|
||||
});
|
||||
|
||||
it('computes correct rate for mixed true/false', () => {
|
||||
expect(rate([true, false, true, false])).toBe(0.5);
|
||||
});
|
||||
|
||||
it('filters out null/undefined from mixed arrays', () => {
|
||||
expect(rate([true, null, false, undefined])).toBe(0.5);
|
||||
});
|
||||
});
|
||||
|
||||
// ── backfillPromptVersions() ──────────────────────────────────────────────────
|
||||
|
||||
const FULL_VERSIONS: PromptVersions = {
|
||||
plan: 'sha256:aaa',
|
||||
revise_plan: 'sha256:bbb',
|
||||
document: 'sha256:ccc',
|
||||
implement: 'sha256:ddd',
|
||||
finalize: 'sha256:eee',
|
||||
init: 'sha256:fff',
|
||||
init_update: 'sha256:ggg',
|
||||
quick_task: 'sha256:hhh',
|
||||
};
|
||||
|
||||
describe('backfillPromptVersions', () => {
|
||||
it('returns all 8 fields with sentinels when given empty-ish object', () => {
|
||||
const result = backfillPromptVersions({} as PromptVersions);
|
||||
expect(Object.keys(result)).toHaveLength(8);
|
||||
for (const val of Object.values(result)) {
|
||||
expect(val).toBe('sha256:missing');
|
||||
}
|
||||
});
|
||||
|
||||
it('preserves existing values, fills missing with sha256:missing', () => {
|
||||
const partial = { plan: 'sha256:aaa', implement: 'sha256:ddd' } as PromptVersions;
|
||||
const result = backfillPromptVersions(partial);
|
||||
expect(result.plan).toBe('sha256:aaa');
|
||||
expect(result.implement).toBe('sha256:ddd');
|
||||
expect(result.revise_plan).toBe('sha256:missing');
|
||||
expect(result.document).toBe('sha256:missing');
|
||||
expect(result.finalize).toBe('sha256:missing');
|
||||
expect(result.init).toBe('sha256:missing');
|
||||
expect(result.init_update).toBe('sha256:missing');
|
||||
expect(result.quick_task).toBe('sha256:missing');
|
||||
});
|
||||
|
||||
it('returns unchanged object when all 8 fields present', () => {
|
||||
const result = backfillPromptVersions(FULL_VERSIONS);
|
||||
expect(result).toEqual(FULL_VERSIONS);
|
||||
});
|
||||
});
|
||||
|
||||
// ── buildCohortKey() ──────────────────────────────────────────────────────────
|
||||
|
||||
describe('buildCohortKey', () => {
|
||||
it('returns 12-char hex string', () => {
|
||||
const key = buildCohortKey(FULL_VERSIONS);
|
||||
expect(key).toMatch(/^[0-9a-f]{12}$/);
|
||||
});
|
||||
|
||||
it('is deterministic (same input → same output)', () => {
|
||||
const key1 = buildCohortKey(FULL_VERSIONS);
|
||||
const key2 = buildCohortKey(FULL_VERSIONS);
|
||||
expect(key1).toBe(key2);
|
||||
});
|
||||
|
||||
it('old 4-field run with backfill sentinels produces same key as raw 4-field object', () => {
|
||||
// Simulate an old run that only had 4 fields
|
||||
const oldRun = {
|
||||
plan: 'sha256:aaa',
|
||||
implement: 'sha256:ddd',
|
||||
document: 'sha256:ccc',
|
||||
finalize: 'sha256:eee',
|
||||
} as PromptVersions;
|
||||
|
||||
// After backfill, the missing fields get 'sha256:missing'
|
||||
const backfilled = backfillPromptVersions(oldRun);
|
||||
|
||||
// buildCohortKey filters out 'sha256:missing', so both should match
|
||||
const keyDirect = buildCohortKey(oldRun);
|
||||
const keyBackfilled = buildCohortKey(backfilled);
|
||||
expect(keyDirect).toBe(keyBackfilled);
|
||||
});
|
||||
|
||||
it('different prompt versions → different keys', () => {
|
||||
const altered = { ...FULL_VERSIONS, plan: 'sha256:zzz' };
|
||||
expect(buildCohortKey(FULL_VERSIONS)).not.toBe(buildCohortKey(altered));
|
||||
});
|
||||
|
||||
it('key is independent of field insertion order', () => {
|
||||
const ordered: PromptVersions = {
|
||||
plan: 'sha256:aaa',
|
||||
revise_plan: 'sha256:bbb',
|
||||
document: 'sha256:ccc',
|
||||
implement: 'sha256:ddd',
|
||||
finalize: 'sha256:eee',
|
||||
init: 'sha256:fff',
|
||||
init_update: 'sha256:ggg',
|
||||
quick_task: 'sha256:hhh',
|
||||
};
|
||||
const reversed: PromptVersions = {
|
||||
quick_task: 'sha256:hhh',
|
||||
init_update: 'sha256:ggg',
|
||||
init: 'sha256:fff',
|
||||
finalize: 'sha256:eee',
|
||||
implement: 'sha256:ddd',
|
||||
document: 'sha256:ccc',
|
||||
revise_plan: 'sha256:bbb',
|
||||
plan: 'sha256:aaa',
|
||||
};
|
||||
expect(buildCohortKey(ordered)).toBe(buildCohortKey(reversed));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,219 @@
|
||||
/**
|
||||
* aggregator.ts
|
||||
* Merges all run JSON files into aggregated.json, grouped by prompt generation (SHA fingerprint).
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import crypto from 'crypto';
|
||||
import type { RunMetrics, AggregatedMetrics, CohortMetrics, PromptVersions } from './types.js';
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export function avg(values: (number | null | undefined)[]): number | null {
|
||||
const valid = values.filter((v): v is number => v != null && !isNaN(v));
|
||||
if (valid.length === 0) return null;
|
||||
return valid.reduce((a, b) => a + b, 0) / valid.length;
|
||||
}
|
||||
|
||||
export function rate(values: (boolean | null | undefined)[]): number | null {
|
||||
const valid = values.filter((v): v is boolean => v != null);
|
||||
if (valid.length === 0) return null;
|
||||
return valid.filter(v => v).length / valid.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a stable cohort key from the sorted prompt_versions object.
|
||||
* Two runs with identical prompt files get the same cohort key.
|
||||
*/
|
||||
export function buildCohortKey(promptVersions: PromptVersions): string {
|
||||
// Filter out missing sentinels so old runs (4 fields) keep their original key
|
||||
const entries = Object.entries(promptVersions)
|
||||
.filter(([, v]) => v !== 'sha256:missing')
|
||||
.sort(([a], [b]) => a.localeCompare(b));
|
||||
const str = JSON.stringify(Object.fromEntries(entries));
|
||||
return crypto.createHash('sha256').update(str).digest('hex').slice(0, 12);
|
||||
}
|
||||
|
||||
/**
|
||||
* Backfill missing PromptVersions fields for old run files (pre-v1.1).
|
||||
*/
|
||||
export function backfillPromptVersions(pv: PromptVersions): PromptVersions {
|
||||
return {
|
||||
plan: pv.plan ?? 'sha256:missing',
|
||||
revise_plan: pv.revise_plan ?? 'sha256:missing',
|
||||
document: pv.document ?? 'sha256:missing',
|
||||
implement: pv.implement ?? 'sha256:missing',
|
||||
finalize: pv.finalize ?? 'sha256:missing',
|
||||
init: pv.init ?? 'sha256:missing',
|
||||
init_update: pv.init_update ?? 'sha256:missing',
|
||||
quick_task: pv.quick_task ?? 'sha256:missing',
|
||||
};
|
||||
}
|
||||
|
||||
// ── Load run files ────────────────────────────────────────────────────────────
|
||||
|
||||
export function loadRunFiles(runsDir: string): RunMetrics[] {
|
||||
if (!fs.existsSync(runsDir)) return [];
|
||||
|
||||
const files = fs.readdirSync(runsDir)
|
||||
.filter(f => f.endsWith('.json') && f.startsWith('run-'))
|
||||
.sort();
|
||||
|
||||
const runs: RunMetrics[] = [];
|
||||
for (const file of files) {
|
||||
try {
|
||||
const content = fs.readFileSync(path.join(runsDir, file), 'utf8');
|
||||
const data = JSON.parse(content) as RunMetrics;
|
||||
data.prompt_versions = backfillPromptVersions(data.prompt_versions);
|
||||
runs.push(data);
|
||||
} catch (err) {
|
||||
console.warn(`Warning: could not parse ${file}: ${err}`);
|
||||
}
|
||||
}
|
||||
return runs;
|
||||
}
|
||||
|
||||
// ── Build cohort metrics ──────────────────────────────────────────────────────
|
||||
|
||||
function buildCohort(runs: RunMetrics[], cohortKey: string): CohortMetrics {
|
||||
const runIds = runs.map(r => r.run_id);
|
||||
const timestamps = runs
|
||||
.flatMap(r => [r.project.started_at, r.project.completed_at])
|
||||
.filter((t): t is string => t != null)
|
||||
.sort();
|
||||
|
||||
// Step 1 averages
|
||||
const avgConfidence = avg(runs.map(r => r.step1_plan.final_confidence));
|
||||
const avgClarification = avg(runs.map(r => r.step1_plan.clarification_rounds));
|
||||
const avgVerifGaps = avg(runs.map(r => r.step1_plan.verification_gaps_found));
|
||||
const avgFRCount = avg(runs.map(r => r.step1_plan.functional_requirements_count));
|
||||
const avgNFRCount = avg(runs.map(r => r.step1_plan.non_functional_requirements_count));
|
||||
const avgRiskCount = avg(runs.map(r => r.step1_plan.risk_count));
|
||||
const avgPhaseStep1 = avg(runs.map(r => r.step1_plan.phase_count));
|
||||
|
||||
// Step 2 averages
|
||||
const avgTotalTasks = avg(runs.map(r => r.step2_document.total_tasks));
|
||||
const avgPhaseStep2 = avg(runs.map(r => r.step2_document.phase_count));
|
||||
const avgParallelGroups = avg(runs.map(r => r.step2_document.parallel_groups_identified));
|
||||
const avgReqCoverage = avg(runs.map(r => r.step2_document.requirement_coverage_percent));
|
||||
const avgVerifItems = avg(runs.map(r => r.step2_document.verification_items_added));
|
||||
|
||||
// Step 3 (loop only)
|
||||
const loopRuns = runs.filter(r => r.step3_implement.used_loop_mode);
|
||||
const avgCompletionRate = avg(loopRuns.map(r => r.step3_implement.task_completion_rate));
|
||||
const avgBlockerCount = avg(loopRuns.map(r => r.step3_implement.blocker_count));
|
||||
const avgTotalIter = avg(loopRuns.map(r => r.step3_implement.total_iterations));
|
||||
const avgIterDuration = avg(loopRuns.map(r => r.step3_implement.avg_iteration_duration_ms));
|
||||
const avgMarkerSuccess = avg(loopRuns.map(r => r.step3_implement.completion_marker_success_rate));
|
||||
|
||||
// Step 4 averages
|
||||
const avgCompletionAtAudit = avg(runs.map(r => r.step4_finalize.completion_rate_at_audit));
|
||||
const avgVerifFailures = avg(runs.map(r => r.step4_finalize.verification_failures_found));
|
||||
const avgDocUpdates = avg(runs.map(r => r.step4_finalize.documentation_updates_needed));
|
||||
const archivalRate = rate(runs.map(r => r.step4_finalize.archival_succeeded));
|
||||
|
||||
return {
|
||||
cohort_key: cohortKey,
|
||||
prompt_versions: runs[0].prompt_versions,
|
||||
run_count: runs.length,
|
||||
run_ids: runIds,
|
||||
first_seen: timestamps[0] ?? runs[0].run_id,
|
||||
last_seen: timestamps[timestamps.length - 1] ?? runs[runs.length - 1].run_id,
|
||||
|
||||
avg_confidence: avgConfidence,
|
||||
avg_clarification_rounds: avgClarification,
|
||||
avg_verification_gaps_found: avgVerifGaps,
|
||||
avg_functional_requirements_count: avgFRCount,
|
||||
avg_non_functional_requirements_count: avgNFRCount,
|
||||
avg_risk_count: avgRiskCount,
|
||||
avg_phase_count_step1: avgPhaseStep1,
|
||||
|
||||
avg_total_tasks: avgTotalTasks,
|
||||
avg_phase_count_step2: avgPhaseStep2,
|
||||
avg_parallel_groups: avgParallelGroups,
|
||||
avg_requirement_coverage_percent: avgReqCoverage,
|
||||
avg_verification_items_added: avgVerifItems,
|
||||
|
||||
loop_run_count: loopRuns.length,
|
||||
avg_task_completion_rate: avgCompletionRate,
|
||||
avg_blocker_count: avgBlockerCount,
|
||||
avg_total_iterations: avgTotalIter,
|
||||
avg_iteration_duration_ms: avgIterDuration,
|
||||
avg_completion_marker_success_rate: avgMarkerSuccess,
|
||||
|
||||
avg_completion_rate_at_audit: avgCompletionAtAudit,
|
||||
avg_verification_failures_found: avgVerifFailures,
|
||||
avg_documentation_updates_needed: avgDocUpdates,
|
||||
archival_success_rate: archivalRate,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Main aggregator ───────────────────────────────────────────────────────────
|
||||
|
||||
export function aggregate(runsDir: string, outputPath: string): AggregatedMetrics {
|
||||
const runs = loadRunFiles(runsDir);
|
||||
|
||||
// Group by cohort key
|
||||
const cohortMap = new Map<string, RunMetrics[]>();
|
||||
for (const run of runs) {
|
||||
const key = buildCohortKey(run.prompt_versions);
|
||||
if (!cohortMap.has(key)) cohortMap.set(key, []);
|
||||
cohortMap.get(key)!.push(run);
|
||||
}
|
||||
|
||||
// Sort cohorts by first seen
|
||||
const cohorts: CohortMetrics[] = [];
|
||||
for (const [key, cohortRuns] of cohortMap) {
|
||||
cohorts.push(buildCohort(cohortRuns, key));
|
||||
}
|
||||
cohorts.sort((a, b) => a.first_seen.localeCompare(b.first_seen));
|
||||
|
||||
// Determine current cohort (most recent)
|
||||
const currentCohortKey = cohorts.length > 0
|
||||
? cohorts[cohorts.length - 1].cohort_key
|
||||
: null;
|
||||
|
||||
const aggregated: AggregatedMetrics = {
|
||||
schema_version: '1.0',
|
||||
last_updated: new Date().toISOString(),
|
||||
total_runs: runs.length,
|
||||
cohorts,
|
||||
current_cohort_key: currentCohortKey,
|
||||
};
|
||||
|
||||
// Write to output path
|
||||
const dir = path.dirname(outputPath);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(outputPath, JSON.stringify(aggregated, null, 2), 'utf8');
|
||||
|
||||
return aggregated;
|
||||
}
|
||||
|
||||
export function loadAggregated(outputPath: string): AggregatedMetrics | null {
|
||||
try {
|
||||
const content = fs.readFileSync(outputPath, 'utf8');
|
||||
return JSON.parse(content) as AggregatedMetrics;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Import a single run JSON from another project into the local runs dir.
|
||||
* Returns true if imported, false if already present.
|
||||
*/
|
||||
export function importRun(runJsonPath: string, runsDir: string): boolean {
|
||||
const content = fs.readFileSync(runJsonPath, 'utf8');
|
||||
const run = JSON.parse(content) as RunMetrics;
|
||||
run.prompt_versions = backfillPromptVersions(run.prompt_versions);
|
||||
const destPath = path.join(runsDir, `${run.run_id}.json`);
|
||||
|
||||
if (fs.existsSync(destPath)) {
|
||||
return false; // Already imported
|
||||
}
|
||||
|
||||
fs.mkdirSync(runsDir, { recursive: true });
|
||||
fs.writeFileSync(destPath, JSON.stringify(run, null, 2), 'utf8');
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* analyzer.ts
|
||||
* Reads aggregated.json + prompt file contents, builds analysis prompt, invokes AI.
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import type { AggregatedMetrics } from './types.js';
|
||||
import { invokeLLM, type AgentType } from './invoke-llm.js';
|
||||
|
||||
const ANALYZE_PROMPT_PATH = new URL('../src/prompts/analyze.md', import.meta.url).pathname
|
||||
.replace(/^\/([A-Za-z]:)/, '$1'); // Fix Windows path
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
function readPromptFiles(plan2codeRoot: string): Record<string, string> {
|
||||
const srcDir = path.join(plan2codeRoot, 'src');
|
||||
const promptFiles = [
|
||||
'plan2code-1--plan.md',
|
||||
'plan2code-1b--revise-plan.md',
|
||||
'plan2code-2--document.md',
|
||||
'plan2code-3--implement.md',
|
||||
'plan2code-4--finalize.md',
|
||||
'plan2code---init.md',
|
||||
'plan2code---init-update.md',
|
||||
'plan2code---quick-task.md',
|
||||
];
|
||||
|
||||
const contents: Record<string, string> = {};
|
||||
for (const file of promptFiles) {
|
||||
try {
|
||||
contents[file] = fs.readFileSync(path.join(srcDir, file), 'utf8');
|
||||
} catch {
|
||||
contents[file] = '(file not found)';
|
||||
}
|
||||
}
|
||||
return contents;
|
||||
}
|
||||
|
||||
function interpolate(template: string, vars: Record<string, string>): string {
|
||||
let result = template;
|
||||
for (const [key, value] of Object.entries(vars)) {
|
||||
result = result.replaceAll(`{{${key}}}`, value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function generateDiagnosisId(): string {
|
||||
const now = new Date();
|
||||
const ts = now.toISOString().replace(/[-:T.Z]/g, '').slice(0, 14);
|
||||
return `diag-${ts}`;
|
||||
}
|
||||
|
||||
// ── Main analyzer ─────────────────────────────────────────────────────────────
|
||||
|
||||
export interface AnalyzerOptions {
|
||||
aggregatedPath: string; // Path to aggregated.json
|
||||
plan2codeRoot: string; // Path to plan2code repo root
|
||||
proposalsDir: string; // Where to save diagnosis output
|
||||
model?: string; // AI model to use (default: claude-opus-4-6)
|
||||
agent?: AgentType; // Agent to use (default: claude-code)
|
||||
}
|
||||
|
||||
export async function runAnalysis(opts: AnalyzerOptions): Promise<string> {
|
||||
const { aggregatedPath, plan2codeRoot, proposalsDir, model = 'claude-opus-4-6', agent = 'claude-code' } = opts;
|
||||
|
||||
// Load aggregated metrics
|
||||
let aggregated: AggregatedMetrics | null = null;
|
||||
try {
|
||||
aggregated = JSON.parse(fs.readFileSync(aggregatedPath, 'utf8')) as AggregatedMetrics;
|
||||
} catch {
|
||||
throw new Error(`Could not read aggregated metrics at ${aggregatedPath}. Run "Collect metrics" and "Import run data" first.`);
|
||||
}
|
||||
|
||||
if (aggregated.total_runs === 0) {
|
||||
throw new Error('No runs in aggregated metrics. Collect and import some runs first.');
|
||||
}
|
||||
|
||||
// Read prompt files
|
||||
const promptContents = readPromptFiles(plan2codeRoot);
|
||||
|
||||
// Format for interpolation
|
||||
const promptContentsStr = Object.entries(promptContents)
|
||||
.map(([file, content]) => `## ${file}\n\n${content}`)
|
||||
.join('\n\n---\n\n');
|
||||
|
||||
const aggregatedStr = JSON.stringify(aggregated, null, 2);
|
||||
|
||||
// Load analyze prompt template
|
||||
let analyzeTemplate: string;
|
||||
try {
|
||||
analyzeTemplate = fs.readFileSync(ANALYZE_PROMPT_PATH, 'utf8');
|
||||
} catch {
|
||||
// Fallback: look relative to cwd
|
||||
const altPath = path.join(process.cwd(), 'src', 'prompts', 'analyze.md');
|
||||
analyzeTemplate = fs.readFileSync(altPath, 'utf8');
|
||||
}
|
||||
|
||||
const fullPrompt = interpolate(analyzeTemplate, {
|
||||
aggregatedMetrics: aggregatedStr,
|
||||
promptContents: promptContentsStr,
|
||||
});
|
||||
|
||||
// Invoke Claude
|
||||
console.log(`\nInvoking AI analysis (model: ${model})...`);
|
||||
console.log('This may take a minute...\n');
|
||||
|
||||
let diagnosisContent: string;
|
||||
try {
|
||||
diagnosisContent = await invokeLLM({
|
||||
prompt: fullPrompt,
|
||||
model,
|
||||
agent,
|
||||
timeout: 300_000,
|
||||
});
|
||||
} catch (err) {
|
||||
throw new Error(`AI invocation failed: ${err instanceof Error ? err.message : String(err)}\n\nMake sure the ${agent} CLI is installed and authenticated.`);
|
||||
}
|
||||
|
||||
// Save diagnosis
|
||||
const diagId = generateDiagnosisId();
|
||||
fs.mkdirSync(proposalsDir, { recursive: true });
|
||||
const diagPath = path.join(proposalsDir, `${diagId}-diagnosis.md`);
|
||||
fs.writeFileSync(diagPath, diagnosisContent, 'utf8');
|
||||
|
||||
console.log(`Diagnosis saved to: ${diagPath}`);
|
||||
return diagPath;
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
/**
|
||||
* applier.ts
|
||||
* Interactive review of a PromptProposal: display colored diff per edit,
|
||||
* user approves/rejects each edit, apply approved edits to src files.
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { confirm } from '@inquirer/prompts';
|
||||
import chalk from 'chalk';
|
||||
import type { PromptEdit, PromptProposal } from './types.js';
|
||||
import { validateEdit } from './improver.js';
|
||||
|
||||
const CHAR_LIMIT = 11_000;
|
||||
|
||||
// ── Diff display ──────────────────────────────────────────────────────────────
|
||||
|
||||
function displayEditDiff(edit: PromptEdit, index: number, total: number): void {
|
||||
console.log();
|
||||
console.log(chalk.bold(`─── Edit ${index + 1} of ${total} ───────────────────────────────────`));
|
||||
console.log(chalk.cyan(`File: ${edit.file}`));
|
||||
console.log(chalk.gray(`Rationale: ${edit.rationale}`));
|
||||
console.log(chalk.gray(`Expected impact: ${edit.expected_metric_impact}`));
|
||||
console.log(chalk.gray(`Char impact: ${edit.char_count_before} → ${edit.char_count_after} (${edit.char_count_delta >= 0 ? '+' : ''}${edit.char_count_delta})`));
|
||||
|
||||
// Show char count status
|
||||
if (edit.char_count_after > CHAR_LIMIT) {
|
||||
console.log(chalk.red(`⚠ WARNING: Would exceed ${CHAR_LIMIT} char limit!`));
|
||||
} else {
|
||||
const headroom = CHAR_LIMIT - edit.char_count_after;
|
||||
console.log(chalk.green(`✓ Char count OK (${headroom} chars headroom after edit)`));
|
||||
}
|
||||
|
||||
console.log();
|
||||
console.log(chalk.bold('── REMOVED (old_text) ──'));
|
||||
|
||||
// Show old text with line-level context
|
||||
const oldLines = edit.old_text.split('\n');
|
||||
for (const line of oldLines) {
|
||||
console.log(chalk.red('- ') + chalk.red(line));
|
||||
}
|
||||
|
||||
console.log();
|
||||
console.log(chalk.bold('── ADDED (new_text) ──'));
|
||||
|
||||
const newLines = edit.new_text.split('\n');
|
||||
for (const line of newLines) {
|
||||
console.log(chalk.green('+ ') + chalk.green(line));
|
||||
}
|
||||
|
||||
console.log();
|
||||
}
|
||||
|
||||
// ── Apply edit to file ────────────────────────────────────────────────────────
|
||||
|
||||
function applyEdit(edit: PromptEdit, plan2codeRoot: string): boolean {
|
||||
// Reject path traversal attempts
|
||||
if (edit.file.includes('..') || path.isAbsolute(edit.file)) {
|
||||
console.error(chalk.red(`✗ Rejected: "${edit.file}" contains path traversal or absolute path`));
|
||||
return false;
|
||||
}
|
||||
|
||||
const filePath = path.join(plan2codeRoot, 'src', edit.file);
|
||||
try {
|
||||
let content = fs.readFileSync(filePath, 'utf8');
|
||||
if (!content.includes(edit.old_text)) {
|
||||
console.error(chalk.red(`✗ Cannot apply: old_text not found in ${edit.file} (may have been modified by a previous edit)`));
|
||||
return false;
|
||||
}
|
||||
content = content.replace(edit.old_text, edit.new_text);
|
||||
|
||||
// Post-apply char count check
|
||||
if (content.length > CHAR_LIMIT) {
|
||||
console.error(chalk.red(`✗ Cannot apply: would exceed ${CHAR_LIMIT} char limit (${content.length} chars)`));
|
||||
return false;
|
||||
}
|
||||
|
||||
fs.writeFileSync(filePath, content, 'utf8');
|
||||
console.log(chalk.green(`✓ Applied edit to ${edit.file} (now ${content.length} chars)`));
|
||||
return true;
|
||||
} catch (err) {
|
||||
console.error(chalk.red(`✗ Failed to apply edit: ${err instanceof Error ? err.message : String(err)}`));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Main applier ──────────────────────────────────────────────────────────────
|
||||
|
||||
export interface ApplierOptions {
|
||||
proposalPath: string;
|
||||
plan2codeRoot: string;
|
||||
proposalsDir: string;
|
||||
}
|
||||
|
||||
export interface ApplierResult {
|
||||
approved: number;
|
||||
rejected: number;
|
||||
applied: number;
|
||||
failed: number;
|
||||
}
|
||||
|
||||
export async function reviewAndApply(opts: ApplierOptions): Promise<ApplierResult> {
|
||||
const { proposalPath, plan2codeRoot, proposalsDir } = opts;
|
||||
|
||||
// Load proposal
|
||||
let proposal: PromptProposal;
|
||||
try {
|
||||
proposal = JSON.parse(fs.readFileSync(proposalPath, 'utf8')) as PromptProposal;
|
||||
} catch {
|
||||
throw new Error(`Could not read proposal at ${proposalPath}`);
|
||||
}
|
||||
|
||||
if (proposal.proposals.length === 0) {
|
||||
console.log(chalk.yellow('No edits in this proposal.'));
|
||||
return { approved: 0, rejected: 0, applied: 0, failed: 0 };
|
||||
}
|
||||
|
||||
console.log();
|
||||
console.log(chalk.bold.cyan('=== Plan2Code Prompt Improvement Review ==='));
|
||||
console.log(chalk.gray(`Proposal: ${proposal.proposal_id}`));
|
||||
console.log(chalk.gray(`Created: ${proposal.created_at}`));
|
||||
console.log(chalk.gray(`Based on: ${proposal.based_on_runs.length} run(s)`));
|
||||
console.log(chalk.gray(`Edits: ${proposal.proposals.length}`));
|
||||
|
||||
// Re-validate all edits against current file state
|
||||
const promptContents: Record<string, string> = {};
|
||||
const srcDir = path.join(plan2codeRoot, 'src');
|
||||
for (const edit of proposal.proposals) {
|
||||
if (!promptContents[edit.file]) {
|
||||
try {
|
||||
promptContents[edit.file] = fs.readFileSync(path.join(srcDir, edit.file), 'utf8');
|
||||
} catch {
|
||||
promptContents[edit.file] = '';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let approved = 0;
|
||||
let rejected = 0;
|
||||
let applied = 0;
|
||||
let failed = 0;
|
||||
|
||||
for (let i = 0; i < proposal.proposals.length; i++) {
|
||||
const edit = proposal.proposals[i];
|
||||
|
||||
// Re-validate
|
||||
const validation = validateEdit(edit, promptContents);
|
||||
if (!validation.valid) {
|
||||
console.log();
|
||||
console.log(chalk.red(`✗ Edit ${i + 1} is no longer valid (files may have changed):`));
|
||||
for (const err of validation.errors) {
|
||||
console.log(chalk.red(` - ${err}`));
|
||||
}
|
||||
rejected++;
|
||||
continue;
|
||||
}
|
||||
|
||||
displayEditDiff(edit, i, proposal.proposals.length);
|
||||
|
||||
const approve = await confirm({
|
||||
message: `Apply this edit to ${edit.file}?`,
|
||||
default: true,
|
||||
});
|
||||
|
||||
if (!approve) {
|
||||
console.log(chalk.gray('Skipped.'));
|
||||
rejected++;
|
||||
continue;
|
||||
}
|
||||
|
||||
approved++;
|
||||
const success = applyEdit(edit, plan2codeRoot);
|
||||
if (success) {
|
||||
applied++;
|
||||
// Update in-memory content to reflect the edit
|
||||
promptContents[edit.file] = promptContents[edit.file].replace(edit.old_text, edit.new_text);
|
||||
} else {
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
// Update proposal status
|
||||
proposal.status = applied > 0 ? 'applied' : 'rejected';
|
||||
fs.writeFileSync(proposalPath, JSON.stringify(proposal, null, 2), 'utf8');
|
||||
|
||||
// Summary
|
||||
console.log();
|
||||
console.log(chalk.bold('─── Review Complete ──────────────────────────────────'));
|
||||
console.log(`Approved: ${chalk.green(String(approved))} Rejected: ${chalk.red(String(rejected))} Applied: ${chalk.green(String(applied))} Failed: ${chalk.red(String(failed))}`);
|
||||
|
||||
if (applied > 0) {
|
||||
console.log();
|
||||
console.log(chalk.bold.cyan('Next steps — create a PR with your changes:'));
|
||||
console.log(chalk.gray(' git add src/'));
|
||||
console.log(chalk.gray(` git commit -m "metrics: apply prompt improvements (${proposal.proposal_id})"`));
|
||||
console.log(chalk.gray(' git push -u origin HEAD'));
|
||||
console.log(chalk.gray(' gh pr create --title "metrics: apply gen N improvements"'));
|
||||
}
|
||||
|
||||
return { approved, rejected, applied, failed };
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import { runCLI } from '../cli.js';
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
await runCLI();
|
||||
process.exit(0);
|
||||
} catch (err) {
|
||||
if (err instanceof Error && err.message.includes('User force closed')) {
|
||||
// Ctrl+C — exit cleanly
|
||||
process.exit(0);
|
||||
}
|
||||
console.error(err instanceof Error ? err.message : String(err));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,574 @@
|
||||
/**
|
||||
* cli.ts
|
||||
* 100% interactive menu-driven CLI for plan2code-metrics.
|
||||
* No flags — all inputs collected via @inquirer/prompts.
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { select, input, confirm } from '@inquirer/prompts';
|
||||
import chalk from 'chalk';
|
||||
import ora from 'ora';
|
||||
import { collectRun } from './collector.js';
|
||||
import { aggregate, loadAggregated, importRun, loadRunFiles } from './aggregator.js';
|
||||
import { runAnalysis } from './analyzer.js';
|
||||
import { generateImprovement } from './improver.js';
|
||||
import { reviewAndApply } from './applier.js';
|
||||
import type { AggregatedMetrics, CohortMetrics } from './types.js';
|
||||
import { METRIC_TARGETS } from './types.js';
|
||||
import { AGENTS, type AgentType } from './invoke-llm.js';
|
||||
|
||||
// ── Defaults ──────────────────────────────────────────────────────────────────
|
||||
|
||||
const DEFAULT_METRICS_DIR = '.plan2code-metrics';
|
||||
const DEFAULT_RUNS_SUBDIR = 'runs';
|
||||
const DEFAULT_AGGREGATED_FILE = 'aggregated.json';
|
||||
const DEFAULT_PROPOSALS_SUBDIR = 'proposals';
|
||||
|
||||
function getMetricsDirs(baseDir = process.cwd()) {
|
||||
const metricsDir = path.join(baseDir, DEFAULT_METRICS_DIR);
|
||||
return {
|
||||
metricsDir,
|
||||
runsDir: path.join(metricsDir, DEFAULT_RUNS_SUBDIR),
|
||||
aggregatedPath: path.join(metricsDir, DEFAULT_AGGREGATED_FILE),
|
||||
proposalsDir: path.join(metricsDir, DEFAULT_PROPOSALS_SUBDIR),
|
||||
};
|
||||
}
|
||||
|
||||
// Detect plan2code root (either this directory or parent directories)
|
||||
function detectPlan2CodeRoot(): string | null {
|
||||
let dir = process.cwd();
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const srcDir = path.join(dir, 'src');
|
||||
if (
|
||||
fs.existsSync(path.join(srcDir, 'plan2code-1--plan.md')) &&
|
||||
fs.existsSync(path.join(srcDir, 'plan2code-2--document.md'))
|
||||
) {
|
||||
return dir;
|
||||
}
|
||||
const parent = path.dirname(dir);
|
||||
if (parent === dir) break;
|
||||
dir = parent;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Detect plan2code version
|
||||
function detectPlan2CodeVersion(plan2codeRoot: string): string {
|
||||
try {
|
||||
const versionPath = path.join(plan2codeRoot, 'version.json');
|
||||
const content = JSON.parse(fs.readFileSync(versionPath, 'utf8'));
|
||||
return content.version ?? '0.0.0';
|
||||
} catch {
|
||||
try {
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(plan2codeRoot, 'package.json'), 'utf8'));
|
||||
return pkg.version ?? '0.0.0';
|
||||
} catch {
|
||||
return '0.0.0';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Metric health helpers ─────────────────────────────────────────────────────
|
||||
|
||||
function metricStatus(key: string, value: number | null): string {
|
||||
if (value == null) return chalk.gray('N/A');
|
||||
const target = METRIC_TARGETS[key as keyof typeof METRIC_TARGETS];
|
||||
if (!target) return chalk.white(String(Math.round(value * 100) / 100));
|
||||
|
||||
const ok = target.direction === 'gte' ? value >= target.target : value <= target.target;
|
||||
const formatted = Number.isInteger(value) ? String(value) : value.toFixed(2);
|
||||
return ok ? chalk.green(`✓ ${formatted}`) : chalk.red(`✗ ${formatted} (target: ${target.direction === 'gte' ? '≥' : '≤'}${target.target})`);
|
||||
}
|
||||
|
||||
// ── Agent + model selection ───────────────────────────────────────────────────
|
||||
|
||||
async function selectAgentAndModel(): Promise<{ agent: AgentType; model: string }> {
|
||||
const agentChoice = await select({
|
||||
message: 'Which AI agent?',
|
||||
choices: Object.values(AGENTS).map(a => ({ name: a.displayName, value: a.name })),
|
||||
});
|
||||
|
||||
const agentDef = AGENTS[agentChoice];
|
||||
const model = await select({
|
||||
message: 'AI model:',
|
||||
choices: agentDef.models.map(m => ({ name: m.label, value: m.value })),
|
||||
});
|
||||
|
||||
return { agent: agentChoice, model };
|
||||
}
|
||||
|
||||
// ── Flow: Collect metrics ─────────────────────────────────────────────────────
|
||||
|
||||
async function flowCollect(): Promise<void> {
|
||||
console.log();
|
||||
console.log(chalk.bold.cyan('── Collect Metrics ──'));
|
||||
|
||||
const sourceChoice = await select({
|
||||
message: 'Where is the completed project spec?',
|
||||
choices: [
|
||||
{ name: 'Active spec directory (specs/<feature-name>/)', value: 'active' },
|
||||
{ name: 'Archived spec (specs--completed/<feature-name>/)', value: 'archived' },
|
||||
{ name: 'Custom path', value: 'custom' },
|
||||
],
|
||||
});
|
||||
|
||||
let specDir: string;
|
||||
let projectName: string;
|
||||
|
||||
if (sourceChoice === 'active' || sourceChoice === 'archived') {
|
||||
const baseSubdir = sourceChoice === 'active' ? 'specs' : 'specs--completed';
|
||||
const baseDir = path.join(process.cwd(), baseSubdir);
|
||||
|
||||
if (!fs.existsSync(baseDir)) {
|
||||
console.log(chalk.red(`No ${baseSubdir}/ directory found in ${process.cwd()}`));
|
||||
return;
|
||||
}
|
||||
|
||||
const entries = fs.readdirSync(baseDir)
|
||||
.filter(f => fs.statSync(path.join(baseDir, f)).isDirectory());
|
||||
|
||||
if (entries.length === 0) {
|
||||
console.log(chalk.red(`No spec directories found in ${baseSubdir}/`));
|
||||
return;
|
||||
}
|
||||
|
||||
const chosen = await select({
|
||||
message: 'Select spec directory:',
|
||||
choices: entries.map(e => ({ name: e, value: e })),
|
||||
});
|
||||
|
||||
specDir = path.join(baseDir, chosen);
|
||||
projectName = chosen;
|
||||
} else {
|
||||
specDir = await input({
|
||||
message: 'Enter full path to spec directory:',
|
||||
validate: (v) => fs.existsSync(v) ? true : 'Directory not found',
|
||||
});
|
||||
projectName = await input({
|
||||
message: 'Project name (for metrics label):',
|
||||
default: path.basename(specDir),
|
||||
});
|
||||
}
|
||||
|
||||
// Detect plan2code root
|
||||
const plan2codeRoot = detectPlan2CodeRoot() ?? process.cwd();
|
||||
const plan2codeVersion = detectPlan2CodeVersion(plan2codeRoot);
|
||||
|
||||
const { runsDir, aggregatedPath } = getMetricsDirs();
|
||||
|
||||
const spinner = ora('Collecting metrics from project artifacts...').start();
|
||||
try {
|
||||
const metrics = await collectRun({
|
||||
specDir,
|
||||
projectName,
|
||||
plan2codeRoot,
|
||||
plan2codeVersion,
|
||||
outputDir: runsDir,
|
||||
});
|
||||
|
||||
spinner.succeed(`Metrics collected: ${metrics.run_id}`);
|
||||
console.log(chalk.gray(` Saved to: ${runsDir}/${metrics.run_id}.json`));
|
||||
|
||||
// Re-aggregate
|
||||
aggregate(runsDir, aggregatedPath);
|
||||
console.log(chalk.gray(' Aggregated metrics updated.'));
|
||||
|
||||
// Show summary
|
||||
console.log();
|
||||
console.log(chalk.bold('Collected:'));
|
||||
console.log(` Step 1 (Plan): ${metrics.step1_plan.present ? chalk.green('✓') : chalk.gray('—')}`);
|
||||
console.log(` Step 2 (Document): ${metrics.step2_document.present ? chalk.green('✓') : chalk.gray('—')}`);
|
||||
console.log(` Step 3 (Implement): ${metrics.step3_implement.present ? (metrics.step3_implement.used_loop_mode ? chalk.green('✓ (loop)') : chalk.yellow('✓ (manual)')) : chalk.gray('—')}`);
|
||||
console.log(` Step 4 (Finalize): ${metrics.step4_finalize.present ? chalk.green('✓') : chalk.gray('—')}`);
|
||||
} catch (err) {
|
||||
spinner.fail(`Collection failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Flow: Import run data ─────────────────────────────────────────────────────
|
||||
|
||||
async function flowImport(): Promise<void> {
|
||||
console.log();
|
||||
console.log(chalk.bold.cyan('── Import Run Data ──'));
|
||||
console.log(chalk.gray('Copy a run JSON from another project into this repo for aggregation.'));
|
||||
console.log();
|
||||
|
||||
const sourcePath = await input({
|
||||
message: 'Path to run JSON file (e.g., /path/to/project/.plan2code-metrics/runs/run-xxx.json):',
|
||||
validate: (v) => {
|
||||
if (!fs.existsSync(v)) return 'File not found';
|
||||
if (!v.endsWith('.json')) return 'Must be a .json file';
|
||||
return true;
|
||||
},
|
||||
});
|
||||
|
||||
const { runsDir, aggregatedPath } = getMetricsDirs();
|
||||
|
||||
try {
|
||||
const imported = importRun(sourcePath, runsDir);
|
||||
if (!imported) {
|
||||
console.log(chalk.yellow('Run already imported (same run_id exists).'));
|
||||
} else {
|
||||
console.log(chalk.green(`✓ Imported ${path.basename(sourcePath)}`));
|
||||
}
|
||||
|
||||
const spinner = ora('Re-aggregating...').start();
|
||||
const aggregated = aggregate(runsDir, aggregatedPath);
|
||||
spinner.succeed(`Aggregated metrics updated (${aggregated.total_runs} total runs)`);
|
||||
} catch (err) {
|
||||
console.error(chalk.red(`Import failed: ${err instanceof Error ? err.message : String(err)}`));
|
||||
}
|
||||
}
|
||||
|
||||
// ── Flow: View metrics status ─────────────────────────────────────────────────
|
||||
|
||||
async function flowViewStatus(): Promise<void> {
|
||||
console.log();
|
||||
console.log(chalk.bold.cyan('── Metrics Status & History ──'));
|
||||
|
||||
const { runsDir, aggregatedPath } = getMetricsDirs();
|
||||
const aggregated = loadAggregated(aggregatedPath);
|
||||
|
||||
if (!aggregated || aggregated.total_runs === 0) {
|
||||
console.log(chalk.yellow('No metrics collected yet. Use "Collect metrics" to get started.'));
|
||||
return;
|
||||
}
|
||||
|
||||
console.log();
|
||||
console.log(chalk.bold(`Total runs: ${aggregated.total_runs} | Generations: ${aggregated.cohorts.length}`));
|
||||
console.log(chalk.gray(`Last updated: ${aggregated.last_updated}`));
|
||||
|
||||
for (let i = 0; i < aggregated.cohorts.length; i++) {
|
||||
const cohort = aggregated.cohorts[i];
|
||||
const isCurrent = cohort.cohort_key === aggregated.current_cohort_key;
|
||||
const label = isCurrent ? chalk.bold.green('[CURRENT]') : '';
|
||||
|
||||
console.log();
|
||||
console.log(chalk.bold(`Generation ${i + 1} (sha:${cohort.cohort_key}) — ${cohort.run_count} run(s) ${label}`));
|
||||
console.log(chalk.gray(` Period: ${cohort.first_seen?.slice(0, 10)} → ${cohort.last_seen?.slice(0, 10)}`));
|
||||
|
||||
// Step 1 metrics
|
||||
if (cohort.avg_confidence != null || cohort.avg_clarification_rounds != null) {
|
||||
console.log(chalk.bold(' Step 1 (Plan):'));
|
||||
if (cohort.avg_confidence != null)
|
||||
console.log(` avg_confidence: ${metricStatus('avg_confidence', cohort.avg_confidence)}`);
|
||||
if (cohort.avg_clarification_rounds != null)
|
||||
console.log(` avg_clarification_rounds: ${metricStatus('avg_clarification_rounds', cohort.avg_clarification_rounds)}`);
|
||||
if (cohort.avg_verification_gaps_found != null)
|
||||
console.log(` avg_verification_gaps: ${metricStatus('avg_verification_gaps_found', cohort.avg_verification_gaps_found)}`);
|
||||
}
|
||||
|
||||
// Step 2 metrics
|
||||
if (cohort.avg_total_tasks != null || cohort.avg_parallel_groups != null) {
|
||||
console.log(chalk.bold(' Step 2 (Document):'));
|
||||
if (cohort.avg_total_tasks != null)
|
||||
console.log(` avg_total_tasks: ${chalk.white(cohort.avg_total_tasks.toFixed(1))}`);
|
||||
if (cohort.avg_parallel_groups != null)
|
||||
console.log(` avg_parallel_groups: ${metricStatus('avg_parallel_groups', cohort.avg_parallel_groups)}`);
|
||||
if (cohort.avg_verification_items_added != null)
|
||||
console.log(` avg_verification_items: ${metricStatus('avg_verification_items_added', cohort.avg_verification_items_added)}`);
|
||||
}
|
||||
|
||||
// Step 3 metrics (loop only)
|
||||
if (cohort.loop_run_count > 0) {
|
||||
console.log(chalk.bold(` Step 3 (Implement) — ${cohort.loop_run_count} loop run(s):`));
|
||||
if (cohort.avg_task_completion_rate != null)
|
||||
console.log(` avg_task_completion_rate: ${metricStatus('avg_task_completion_rate', cohort.avg_task_completion_rate)}`);
|
||||
if (cohort.avg_blocker_count != null)
|
||||
console.log(` avg_blocker_count: ${metricStatus('avg_blocker_count', cohort.avg_blocker_count)}`);
|
||||
if (cohort.avg_completion_marker_success_rate != null)
|
||||
console.log(` avg_marker_success_rate: ${metricStatus('avg_completion_marker_success_rate', cohort.avg_completion_marker_success_rate)}`);
|
||||
}
|
||||
|
||||
// Step 4 metrics
|
||||
if (cohort.avg_completion_rate_at_audit != null || cohort.archival_success_rate != null) {
|
||||
console.log(chalk.bold(' Step 4 (Finalize):'));
|
||||
if (cohort.avg_completion_rate_at_audit != null)
|
||||
console.log(` avg_completion_at_audit: ${chalk.white(cohort.avg_completion_rate_at_audit.toFixed(2))}`);
|
||||
if (cohort.avg_verification_failures_found != null)
|
||||
console.log(` avg_verif_failures: ${metricStatus('avg_verification_failures_found', cohort.avg_verification_failures_found)}`);
|
||||
if (cohort.archival_success_rate != null)
|
||||
console.log(` archival_success_rate: ${metricStatus('archival_success_rate', cohort.archival_success_rate)}`);
|
||||
}
|
||||
|
||||
// Compare with previous generation
|
||||
if (i > 0) {
|
||||
const prev = aggregated.cohorts[i - 1];
|
||||
const deltas: string[] = [];
|
||||
if (cohort.avg_confidence != null && prev.avg_confidence != null) {
|
||||
const d = cohort.avg_confidence - prev.avg_confidence;
|
||||
deltas.push(`confidence ${d >= 0 ? chalk.green(`▲${d.toFixed(1)}`) : chalk.red(`▼${Math.abs(d).toFixed(1)}`)}`);
|
||||
}
|
||||
if (cohort.avg_task_completion_rate != null && prev.avg_task_completion_rate != null) {
|
||||
const d = cohort.avg_task_completion_rate - prev.avg_task_completion_rate;
|
||||
deltas.push(`completion ${d >= 0 ? chalk.green(`▲${(d * 100).toFixed(1)}%`) : chalk.red(`▼${(Math.abs(d) * 100).toFixed(1)}%`)}`);
|
||||
}
|
||||
if (deltas.length > 0) {
|
||||
console.log(chalk.gray(` vs Gen ${i}: ${deltas.join(' ')}`));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run list
|
||||
const runs = loadRunFiles(runsDir);
|
||||
if (runs.length > 0) {
|
||||
console.log();
|
||||
console.log(chalk.bold(`Run files (${runs.length}):`));
|
||||
for (const run of runs.slice(-10)) {
|
||||
console.log(chalk.gray(` ${run.run_id} ${run.project.name} v${run.plan2code_version}`));
|
||||
}
|
||||
if (runs.length > 10) {
|
||||
console.log(chalk.gray(` ... and ${runs.length - 10} more`));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Flow: Run analysis ────────────────────────────────────────────────────────
|
||||
|
||||
async function flowRunAnalysis(): Promise<string | null> {
|
||||
console.log();
|
||||
console.log(chalk.bold.cyan('── Run Analysis (Diagnose Weak Steps) ──'));
|
||||
|
||||
const { aggregatedPath, proposalsDir } = getMetricsDirs();
|
||||
const plan2codeRoot = detectPlan2CodeRoot() ?? process.cwd();
|
||||
|
||||
const aggregated = loadAggregated(aggregatedPath);
|
||||
if (!aggregated || aggregated.total_runs === 0) {
|
||||
console.log(chalk.yellow('No aggregated metrics found. Collect and import runs first.'));
|
||||
return null;
|
||||
}
|
||||
|
||||
if (aggregated.total_runs < 3) {
|
||||
const proceed = await confirm({
|
||||
message: `Only ${aggregated.total_runs} run(s) available (≥3 recommended for reliable analysis). Proceed anyway?`,
|
||||
default: false,
|
||||
});
|
||||
if (!proceed) return null;
|
||||
}
|
||||
|
||||
const { agent, model } = await selectAgentAndModel();
|
||||
|
||||
try {
|
||||
const diagPath = await runAnalysis({
|
||||
aggregatedPath,
|
||||
plan2codeRoot,
|
||||
proposalsDir,
|
||||
model,
|
||||
agent,
|
||||
});
|
||||
|
||||
console.log();
|
||||
console.log(chalk.green('✓ Analysis complete'));
|
||||
console.log(chalk.gray(`Diagnosis: ${diagPath}`));
|
||||
|
||||
const viewNow = await confirm({
|
||||
message: 'Open diagnosis in console?',
|
||||
default: true,
|
||||
});
|
||||
|
||||
if (viewNow) {
|
||||
const content = fs.readFileSync(diagPath, 'utf8');
|
||||
console.log();
|
||||
console.log(chalk.dim('─'.repeat(60)));
|
||||
console.log(content);
|
||||
console.log(chalk.dim('─'.repeat(60)));
|
||||
}
|
||||
|
||||
return diagPath;
|
||||
} catch (err) {
|
||||
console.error(chalk.red(`Analysis failed: ${err instanceof Error ? err.message : String(err)}`));
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Flow: Generate improvement proposal ──────────────────────────────────────
|
||||
|
||||
async function flowGenerateProposal(): Promise<void> {
|
||||
console.log();
|
||||
console.log(chalk.bold.cyan('── Generate Improvement Proposal ──'));
|
||||
|
||||
const { aggregatedPath, proposalsDir, runsDir } = getMetricsDirs();
|
||||
const plan2codeRoot = detectPlan2CodeRoot() ?? process.cwd();
|
||||
|
||||
// Find diagnosis files
|
||||
let diagFiles: string[] = [];
|
||||
if (fs.existsSync(proposalsDir)) {
|
||||
diagFiles = fs.readdirSync(proposalsDir)
|
||||
.filter(f => f.endsWith('-diagnosis.md'))
|
||||
.sort()
|
||||
.reverse(); // most recent first
|
||||
}
|
||||
|
||||
let diagnosisPath: string;
|
||||
|
||||
if (diagFiles.length === 0) {
|
||||
console.log(chalk.yellow('No diagnosis files found. Running analysis first...'));
|
||||
const diagPath = await flowRunAnalysis();
|
||||
if (!diagPath) return;
|
||||
diagnosisPath = diagPath;
|
||||
} else {
|
||||
const choice = await select({
|
||||
message: 'Select diagnosis to base proposal on:',
|
||||
choices: [
|
||||
...diagFiles.map(f => ({ name: f, value: path.join(proposalsDir, f) })),
|
||||
{ name: '(run new analysis first)', value: '__new__' },
|
||||
],
|
||||
});
|
||||
|
||||
if (choice === '__new__') {
|
||||
const diagPath = await flowRunAnalysis();
|
||||
if (!diagPath) return;
|
||||
diagnosisPath = diagPath;
|
||||
} else {
|
||||
diagnosisPath = choice;
|
||||
}
|
||||
}
|
||||
|
||||
const { agent, model } = await selectAgentAndModel();
|
||||
|
||||
try {
|
||||
const result = await generateImprovement({
|
||||
diagnosisPath,
|
||||
plan2codeRoot,
|
||||
proposalsDir,
|
||||
runsDir,
|
||||
model,
|
||||
agent,
|
||||
});
|
||||
|
||||
console.log();
|
||||
console.log(chalk.green(`✓ Proposal generated: ${result.proposal.proposal_id}`));
|
||||
console.log(chalk.gray(` Valid edits: ${result.validEditCount}`));
|
||||
console.log(chalk.gray(` Invalid edits: ${result.invalidEditCount} (rejected)`));
|
||||
console.log(chalk.gray(` Saved to: ${result.proposalPath}`));
|
||||
|
||||
if (result.validEditCount > 0) {
|
||||
const reviewNow = await confirm({
|
||||
message: 'Review and apply edits now?',
|
||||
default: true,
|
||||
});
|
||||
|
||||
if (reviewNow) {
|
||||
await reviewAndApply({
|
||||
proposalPath: result.proposalPath,
|
||||
plan2codeRoot,
|
||||
proposalsDir,
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(chalk.red(`Proposal generation failed: ${err instanceof Error ? err.message : String(err)}`));
|
||||
}
|
||||
}
|
||||
|
||||
// ── Flow: Review and apply proposal ──────────────────────────────────────────
|
||||
|
||||
async function flowReviewAndApply(): Promise<void> {
|
||||
console.log();
|
||||
console.log(chalk.bold.cyan('── Review and Apply a Proposal ──'));
|
||||
|
||||
const { proposalsDir } = getMetricsDirs();
|
||||
const plan2codeRoot = detectPlan2CodeRoot() ?? process.cwd();
|
||||
|
||||
if (!fs.existsSync(proposalsDir)) {
|
||||
console.log(chalk.yellow('No proposals directory found. Generate a proposal first.'));
|
||||
return;
|
||||
}
|
||||
|
||||
const proposalFiles = fs.readdirSync(proposalsDir)
|
||||
.filter(f => /^prop-\d+\.json$/.test(f))
|
||||
.sort()
|
||||
.reverse(); // most recent first
|
||||
|
||||
if (proposalFiles.length === 0) {
|
||||
console.log(chalk.yellow('No proposal files found. Generate a proposal first.'));
|
||||
return;
|
||||
}
|
||||
|
||||
const chosen = await select({
|
||||
message: 'Select proposal to review:',
|
||||
choices: proposalFiles.map(f => {
|
||||
try {
|
||||
const p = JSON.parse(fs.readFileSync(path.join(proposalsDir, f), 'utf8'));
|
||||
const label = `${f} [${p.status}] ${p.proposals?.length ?? 0} edits`;
|
||||
return { name: label, value: path.join(proposalsDir, f) };
|
||||
} catch {
|
||||
return { name: f, value: path.join(proposalsDir, f) };
|
||||
}
|
||||
}),
|
||||
});
|
||||
|
||||
await reviewAndApply({
|
||||
proposalPath: chosen,
|
||||
plan2codeRoot,
|
||||
proposalsDir,
|
||||
});
|
||||
}
|
||||
|
||||
// ── Main menu ─────────────────────────────────────────────────────────────────
|
||||
|
||||
export async function runCLI(): Promise<void> {
|
||||
console.log();
|
||||
console.log(chalk.bold.white('plan2code-metrics'));
|
||||
console.log(chalk.gray('Recursive self-improvement toolchain for plan2code contributors'));
|
||||
console.log();
|
||||
|
||||
// Check if we're in (or near) a plan2code repo
|
||||
const plan2codeRoot = detectPlan2CodeRoot();
|
||||
if (!plan2codeRoot) {
|
||||
console.log(chalk.yellow('⚠ Could not detect plan2code repository (src/plan2code-*.md not found nearby).'));
|
||||
console.log(chalk.gray(' Prompt version hashing and analysis will be limited.'));
|
||||
console.log();
|
||||
} else {
|
||||
const version = detectPlan2CodeVersion(plan2codeRoot);
|
||||
console.log(chalk.gray(`plan2code root: ${plan2codeRoot} (v${version})`));
|
||||
console.log();
|
||||
}
|
||||
|
||||
let continueLoop = true;
|
||||
while (continueLoop) {
|
||||
const action = await select({
|
||||
message: 'What would you like to do?',
|
||||
choices: [
|
||||
{ name: 'Collect metrics for a completed project', value: 'collect' },
|
||||
{ name: 'Import run data from another project', value: 'import' },
|
||||
{ name: 'View metrics status and history', value: 'view' },
|
||||
{ name: 'Run analysis (diagnose weak steps)', value: 'analyze' },
|
||||
{ name: 'Generate improvement proposal', value: 'propose' },
|
||||
{ name: 'Review and apply a proposal', value: 'apply' },
|
||||
{ name: 'Exit', value: 'exit' },
|
||||
],
|
||||
});
|
||||
|
||||
switch (action) {
|
||||
case 'collect':
|
||||
await flowCollect();
|
||||
break;
|
||||
case 'import':
|
||||
await flowImport();
|
||||
break;
|
||||
case 'view':
|
||||
await flowViewStatus();
|
||||
break;
|
||||
case 'analyze':
|
||||
await flowRunAnalysis();
|
||||
break;
|
||||
case 'propose':
|
||||
await flowGenerateProposal();
|
||||
break;
|
||||
case 'apply':
|
||||
await flowReviewAndApply();
|
||||
break;
|
||||
case 'exit':
|
||||
continueLoop = false;
|
||||
break;
|
||||
}
|
||||
|
||||
if (continueLoop && action !== 'exit') {
|
||||
console.log();
|
||||
}
|
||||
}
|
||||
|
||||
console.log(chalk.gray('Goodbye.'));
|
||||
}
|
||||
@@ -0,0 +1,474 @@
|
||||
/**
|
||||
* collector.ts
|
||||
* Reads finished project artifacts and writes a RunMetrics JSON file.
|
||||
* Zero dependency on plan2code-loop internals — reads files directly.
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import crypto from 'crypto';
|
||||
import type {
|
||||
RunMetrics,
|
||||
PromptVersions,
|
||||
Step1PlanMetrics,
|
||||
Step2DocumentMetrics,
|
||||
Step3ImplementMetrics,
|
||||
Step4FinalizeMetrics,
|
||||
} from './types.js';
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function sha256File(filePath: string): string {
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, 'utf8');
|
||||
return 'sha256:' + crypto.createHash('sha256').update(content).digest('hex');
|
||||
} catch {
|
||||
return 'sha256:missing';
|
||||
}
|
||||
}
|
||||
|
||||
function readFileSafe(filePath: string): string | null {
|
||||
try {
|
||||
return fs.readFileSync(filePath, 'utf8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function generateRunId(): string {
|
||||
const now = new Date();
|
||||
const ts = now.toISOString().replace(/[-:T]/g, '').slice(0, 14);
|
||||
const rand = crypto.randomBytes(2).toString('hex');
|
||||
return `run-${ts.slice(0, 8)}-${ts.slice(8, 14)}-${rand}`;
|
||||
}
|
||||
|
||||
// ── Prompt version hashing ────────────────────────────────────────────────────
|
||||
|
||||
export function collectPromptVersions(plan2codeRoot: string): PromptVersions {
|
||||
const srcDir = path.join(plan2codeRoot, 'src');
|
||||
return {
|
||||
plan: sha256File(path.join(srcDir, 'plan2code-1--plan.md')),
|
||||
revise_plan: sha256File(path.join(srcDir, 'plan2code-1b--revise-plan.md')),
|
||||
document: sha256File(path.join(srcDir, 'plan2code-2--document.md')),
|
||||
implement: sha256File(path.join(srcDir, 'plan2code-3--implement.md')),
|
||||
finalize: sha256File(path.join(srcDir, 'plan2code-4--finalize.md')),
|
||||
init: sha256File(path.join(srcDir, 'plan2code---init.md')),
|
||||
init_update: sha256File(path.join(srcDir, 'plan2code---init-update.md')),
|
||||
quick_task: sha256File(path.join(srcDir, 'plan2code---quick-task.md')),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Step 1: Plan ──────────────────────────────────────────────────────────────
|
||||
|
||||
export function collectStep1(specDir: string): Step1PlanMetrics {
|
||||
// Find PLAN-DRAFT-*.md files
|
||||
let draftFiles: string[] = [];
|
||||
let convFiles: string[] = [];
|
||||
try {
|
||||
const files = fs.readdirSync(specDir);
|
||||
draftFiles = files
|
||||
.filter(f => f.startsWith('PLAN-DRAFT-') && f.endsWith('.md'))
|
||||
.map(f => path.join(specDir, f));
|
||||
convFiles = files
|
||||
.filter(f => f.startsWith('PLAN-CONVERSATION-') && f.endsWith('.md'))
|
||||
.map(f => path.join(specDir, f));
|
||||
} catch {
|
||||
return { present: false, final_confidence: null, confidence_breakdown: null,
|
||||
clarification_rounds: null, tech_stack_revision_rounds: null,
|
||||
verification_gaps_found: null, functional_requirements_count: null,
|
||||
non_functional_requirements_count: null, risk_count: null, phase_count: null };
|
||||
}
|
||||
|
||||
if (draftFiles.length === 0) {
|
||||
return { present: false, final_confidence: null, confidence_breakdown: null,
|
||||
clarification_rounds: null, tech_stack_revision_rounds: null,
|
||||
verification_gaps_found: null, functional_requirements_count: null,
|
||||
non_functional_requirements_count: null, risk_count: null, phase_count: null };
|
||||
}
|
||||
|
||||
// Use the latest draft file
|
||||
draftFiles.sort();
|
||||
const latestDraft = readFileSafe(draftFiles[draftFiles.length - 1]) ?? '';
|
||||
|
||||
// Parse "Confidence: XX%" — look for overall or total confidence
|
||||
let finalConfidence: number | null = null;
|
||||
const confMatch = latestDraft.match(/(?:Overall|Final|Total)?\s*[Cc]onfidence[:\s]+(\d{1,3})%/);
|
||||
if (confMatch) {
|
||||
finalConfidence = parseInt(confMatch[1], 10);
|
||||
} else {
|
||||
// Try table format: | Confidence | 92 |
|
||||
const tableMatch = latestDraft.match(/[|]\s*[Cc]onfidence\s*[|]\s*(\d{1,3})/);
|
||||
if (tableMatch) finalConfidence = parseInt(tableMatch[1], 10);
|
||||
}
|
||||
|
||||
// Confidence breakdown (requirements, feasibility, integration, risk)
|
||||
let confidenceBreakdown: Step1PlanMetrics['confidence_breakdown'] = null;
|
||||
const reqMatch = latestDraft.match(/[Rr]equirements?[:\s|]+(\d{1,2})/);
|
||||
const feasMatch = latestDraft.match(/[Ff]easibility[:\s|]+(\d{1,2})/);
|
||||
const intMatch = latestDraft.match(/[Ii]ntegration[:\s|]+(\d{1,2})/);
|
||||
const riskMatch = latestDraft.match(/[Rr]isk[:\s|]+(\d{1,2})/);
|
||||
if (reqMatch || feasMatch || intMatch || riskMatch) {
|
||||
confidenceBreakdown = {
|
||||
requirements: reqMatch ? parseInt(reqMatch[1], 10) : null,
|
||||
feasibility: feasMatch ? parseInt(feasMatch[1], 10) : null,
|
||||
integration: intMatch ? parseInt(intMatch[1], 10) : null,
|
||||
risk: riskMatch ? parseInt(riskMatch[1], 10) : null,
|
||||
};
|
||||
}
|
||||
|
||||
// Count ### FR- / ### NFR- headings
|
||||
const frCount = (latestDraft.match(/###\s+FR-/g) ?? []).length;
|
||||
const nfrCount = (latestDraft.match(/###\s+NFR-/g) ?? []).length;
|
||||
|
||||
// Count risk table rows (lines starting with | that contain risk-level keywords)
|
||||
const riskRows = latestDraft.match(/^\s*[|][^|]*(?:High|Medium|Low|Critical)[^|]*[|]/gm) ?? [];
|
||||
const riskCount = riskRows.length || null;
|
||||
|
||||
// Count ## Phase headings
|
||||
const phaseCount = (latestDraft.match(/^##\s+Phase\s+\d/gm) ?? []).length || null;
|
||||
|
||||
// Clarification rounds from conversation file
|
||||
let clarificationRounds: number | null = null;
|
||||
let techStackRevisions: number | null = null;
|
||||
let verificationGaps: number | null = null;
|
||||
|
||||
if (convFiles.length > 0) {
|
||||
convFiles.sort();
|
||||
const latestConv = readFileSafe(convFiles[convFiles.length - 1]) ?? '';
|
||||
// Count heading repetitions as clarification rounds (## Clarification or ## Round)
|
||||
const clarRounds = (latestConv.match(/^##\s+(?:Clarification|Round)\s+\d/gm) ?? []).length;
|
||||
clarificationRounds = clarRounds || null;
|
||||
// Tech stack revision rounds
|
||||
const techRounds = (latestConv.match(/^##\s+(?:Tech\s+Stack|Technology)\s+Revision/gmi) ?? []).length;
|
||||
techStackRevisions = techRounds || null;
|
||||
// Verification gaps found
|
||||
const gapMatches = latestConv.match(/(?:verification\s+gap|gap\s+found|missing\s+requirement)/gi) ?? [];
|
||||
verificationGaps = gapMatches.length || null;
|
||||
}
|
||||
|
||||
return {
|
||||
present: true,
|
||||
final_confidence: finalConfidence,
|
||||
confidence_breakdown: confidenceBreakdown,
|
||||
clarification_rounds: clarificationRounds,
|
||||
tech_stack_revision_rounds: techStackRevisions,
|
||||
verification_gaps_found: verificationGaps,
|
||||
functional_requirements_count: frCount || null,
|
||||
non_functional_requirements_count: nfrCount || null,
|
||||
risk_count: riskCount,
|
||||
phase_count: phaseCount,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Step 2: Document ──────────────────────────────────────────────────────────
|
||||
|
||||
export function collectStep2(specDir: string): Step2DocumentMetrics {
|
||||
const overviewPath = path.join(specDir, 'overview.md');
|
||||
const overview = readFileSafe(overviewPath);
|
||||
|
||||
if (!overview) {
|
||||
return { present: false, total_tasks: null, tasks_per_phase: null,
|
||||
phase_count: null, parallel_groups_identified: null,
|
||||
requirement_coverage_percent: null, verification_items_added: null };
|
||||
}
|
||||
|
||||
// Count all checkbox tasks: - [ ], - [x], - [!], - [/]
|
||||
const allTasks = (overview.match(/^\s*-\s+\[[ x!/?]\]/gm) ?? []).length;
|
||||
|
||||
// Detect Parallel Execution Groups table
|
||||
const parallelGroups = (overview.match(/Parallel\s+Execution\s+Group/gi) ?? []).length;
|
||||
|
||||
// Count phase files
|
||||
let phaseCount = 0;
|
||||
let tasksPerPhase: number[] = [];
|
||||
try {
|
||||
const files = fs.readdirSync(specDir);
|
||||
const phaseFiles = files
|
||||
.filter(f => /^phase-\d+\.md$/i.test(f))
|
||||
.sort();
|
||||
phaseCount = phaseFiles.length;
|
||||
for (const pf of phaseFiles) {
|
||||
const content = readFileSafe(path.join(specDir, pf)) ?? '';
|
||||
const count = (content.match(/^\s*-\s+\[[ x!/?]\]/gm) ?? []).length;
|
||||
tasksPerPhase.push(count);
|
||||
}
|
||||
} catch {
|
||||
// leave empty
|
||||
}
|
||||
|
||||
// Requirement coverage: look for coverage percentage in overview
|
||||
let reqCoverage: number | null = null;
|
||||
const covMatch = overview.match(/[Cc]overage[:\s]+(\d{1,3})%/);
|
||||
if (covMatch) reqCoverage = parseInt(covMatch[1], 10);
|
||||
|
||||
// Verification items added (look for verification checklist items)
|
||||
const verifItems = (overview.match(/(?:verify|verification|test|check):\s*\[[ x]\]/gi) ?? []).length;
|
||||
|
||||
return {
|
||||
present: true,
|
||||
total_tasks: allTasks || null,
|
||||
tasks_per_phase: tasksPerPhase.length > 0 ? tasksPerPhase : null,
|
||||
phase_count: phaseCount || null,
|
||||
parallel_groups_identified: parallelGroups || 0,
|
||||
requirement_coverage_percent: reqCoverage,
|
||||
verification_items_added: verifItems || null,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Step 3: Implement (loop data) ─────────────────────────────────────────────
|
||||
|
||||
export function collectStep3(specDir: string): Step3ImplementMetrics {
|
||||
const loopDir = path.join(specDir, '.plan2code-loop');
|
||||
const iterLogPath = path.join(loopDir, 'iteration.log');
|
||||
const configPath = path.join(loopDir, 'config.json');
|
||||
|
||||
if (!fs.existsSync(loopDir)) {
|
||||
return { present: true, used_loop_mode: false, loop_mode: null,
|
||||
task_completion_rate: null, tasks_completed: null, tasks_total: null,
|
||||
blocker_count: null, blocker_categories: null, total_iterations: null,
|
||||
avg_iteration_duration_ms: null, exit_code_distribution: null,
|
||||
completion_marker_success_rate: null };
|
||||
}
|
||||
|
||||
// Parse config.json for loop mode
|
||||
let loopMode: 'task' | 'phase' | null = null;
|
||||
const configContent = readFileSafe(configPath);
|
||||
if (configContent) {
|
||||
try {
|
||||
const config = JSON.parse(configContent);
|
||||
loopMode = config.loopMode ?? null;
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
// Parse NDJSON iteration.log
|
||||
const iterLogContent = readFileSafe(iterLogPath);
|
||||
if (!iterLogContent) {
|
||||
return { present: true, used_loop_mode: true, loop_mode: loopMode,
|
||||
task_completion_rate: null, tasks_completed: null, tasks_total: null,
|
||||
blocker_count: null, blocker_categories: null, total_iterations: null,
|
||||
avg_iteration_duration_ms: null, exit_code_distribution: null,
|
||||
completion_marker_success_rate: null };
|
||||
}
|
||||
|
||||
interface IterEntry {
|
||||
iteration: number;
|
||||
timestamp: string;
|
||||
duration: number;
|
||||
exitCode: number;
|
||||
status: string;
|
||||
completionMarker?: string;
|
||||
}
|
||||
|
||||
const entries: IterEntry[] = [];
|
||||
for (const line of iterLogContent.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
try {
|
||||
entries.push(JSON.parse(trimmed));
|
||||
} catch {
|
||||
// skip malformed lines
|
||||
}
|
||||
}
|
||||
|
||||
const totalIterations = entries.length;
|
||||
if (totalIterations === 0) {
|
||||
return { present: true, used_loop_mode: true, loop_mode: loopMode,
|
||||
task_completion_rate: null, tasks_completed: null, tasks_total: null,
|
||||
blocker_count: null, blocker_categories: null, total_iterations: 0,
|
||||
avg_iteration_duration_ms: null, exit_code_distribution: null,
|
||||
completion_marker_success_rate: null };
|
||||
}
|
||||
|
||||
// Exit code distribution
|
||||
const exitCodes: Record<string, number> = {};
|
||||
for (const e of entries) {
|
||||
const key = String(e.exitCode ?? 'other');
|
||||
exitCodes[key] = (exitCodes[key] ?? 0) + 1;
|
||||
}
|
||||
|
||||
// Average duration
|
||||
const durations = entries.map(e => e.duration).filter(d => d != null && d > 0);
|
||||
const avgDuration = durations.length > 0
|
||||
? Math.round(durations.reduce((a, b) => a + b, 0) / durations.length)
|
||||
: null;
|
||||
|
||||
// Completion markers
|
||||
const markerEntries = entries.filter(e => e.completionMarker && e.completionMarker.trim() !== '');
|
||||
const taskCompleteEntries = markerEntries.filter(e =>
|
||||
e.completionMarker?.startsWith('TASK_COMPLETE')
|
||||
);
|
||||
const blockedEntries = markerEntries.filter(e =>
|
||||
e.completionMarker?.startsWith('TASK_BLOCKED')
|
||||
);
|
||||
|
||||
// Completion marker success rate: entries where a valid marker was detected vs total
|
||||
const validMarkerCount = markerEntries.length;
|
||||
const markerSuccessRate = totalIterations > 0
|
||||
? validMarkerCount / totalIterations
|
||||
: null;
|
||||
|
||||
// Task counts from markers (used for completion_marker_success_rate)
|
||||
const blockerCount = blockedEntries.length;
|
||||
|
||||
// Blocker categories (parse from "TASK_BLOCKED: X.Y - reason")
|
||||
const blockerCategories: string[] = [];
|
||||
for (const e of blockedEntries) {
|
||||
const match = e.completionMarker?.match(/TASK_BLOCKED:\s*[\d.]+\s*-\s*(.+)/);
|
||||
if (match) {
|
||||
// Normalize to snake_case category
|
||||
const reason = match[1].toLowerCase().replace(/\s+/g, '_').replace(/[^a-z0-9_]/g, '');
|
||||
if (!blockerCategories.includes(reason)) {
|
||||
blockerCategories.push(reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Count tasks from overview.md checkboxes (consistent numerator and denominator)
|
||||
const overviewPath = path.join(specDir, 'overview.md');
|
||||
const overview = readFileSafe(overviewPath);
|
||||
let tasksTotal: number | null = null;
|
||||
let tasksCompleted: number | null = null;
|
||||
if (overview) {
|
||||
tasksTotal = (overview.match(/^\s*-\s+\[[ x!/?]\]/gm) ?? []).length || null;
|
||||
tasksCompleted = (overview.match(/^\s*-\s+\[x\]/gm) ?? []).length || null;
|
||||
}
|
||||
|
||||
const taskCompletionRate = tasksTotal && tasksTotal > 0 && tasksCompleted != null
|
||||
? tasksCompleted / tasksTotal
|
||||
: null;
|
||||
|
||||
return {
|
||||
present: true,
|
||||
used_loop_mode: true,
|
||||
loop_mode: loopMode,
|
||||
task_completion_rate: taskCompletionRate,
|
||||
tasks_completed: tasksCompleted || null,
|
||||
tasks_total: tasksTotal,
|
||||
blocker_count: blockerCount,
|
||||
blocker_categories: blockerCategories.length > 0 ? blockerCategories : null,
|
||||
total_iterations: totalIterations,
|
||||
avg_iteration_duration_ms: avgDuration,
|
||||
exit_code_distribution: exitCodes,
|
||||
completion_marker_success_rate: markerSuccessRate,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Step 4: Finalize ──────────────────────────────────────────────────────────
|
||||
|
||||
export function collectStep4(specDir: string): Step4FinalizeMetrics {
|
||||
// Check if spec was archived (specs--completed exists at parent level)
|
||||
const specDirName = path.basename(specDir);
|
||||
const parentDir = path.dirname(specDir);
|
||||
const completedPath = path.join(parentDir, '..', 'specs--completed', specDirName);
|
||||
const archivalSucceeded = fs.existsSync(completedPath);
|
||||
|
||||
// Read overview for completion metrics
|
||||
const overviewPath = path.join(specDir, 'overview.md');
|
||||
|
||||
// If archived, try from archived location
|
||||
const effectiveOverviewPath = archivalSucceeded
|
||||
? path.join(completedPath, 'overview.md')
|
||||
: overviewPath;
|
||||
|
||||
const overview = readFileSafe(effectiveOverviewPath) ?? readFileSafe(overviewPath);
|
||||
if (!overview) {
|
||||
return { present: false, completion_rate_at_audit: null,
|
||||
verification_failures_found: null, documentation_updates_needed: null,
|
||||
archival_succeeded: archivalSucceeded };
|
||||
}
|
||||
|
||||
// Completion rate: completed tasks / total tasks
|
||||
const totalTasks = (overview.match(/^\s*-\s+\[[ x!/?]\]/gm) ?? []).length;
|
||||
const completedTasks = (overview.match(/^\s*-\s+\[x\]/gm) ?? []).length;
|
||||
const completionRate = totalTasks > 0 ? completedTasks / totalTasks : null;
|
||||
|
||||
// Verification failures: look for [!] tasks or "verification failed" text
|
||||
const blockedTasks = (overview.match(/^\s*-\s+\[!\]/gm) ?? []).length;
|
||||
|
||||
// Documentation updates needed (look for TODO or "update" markers)
|
||||
const docUpdates = (overview.match(/(?:TODO|FIXME|update\s+(?:README|CHANGELOG|docs))/gi) ?? []).length;
|
||||
|
||||
return {
|
||||
present: true,
|
||||
completion_rate_at_audit: completionRate,
|
||||
verification_failures_found: blockedTasks || 0,
|
||||
documentation_updates_needed: docUpdates || null,
|
||||
archival_succeeded: archivalSucceeded,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Main collector ────────────────────────────────────────────────────────────
|
||||
|
||||
export interface CollectorOptions {
|
||||
specDir: string; // Path to the spec directory (specs/<feature-name>)
|
||||
projectName: string; // Human-readable project name
|
||||
plan2codeRoot: string; // Path to the plan2code repo (for prompt hashing)
|
||||
plan2codeVersion: string; // e.g. "1.7.0"
|
||||
outputDir: string; // Where to write <run-id>.json
|
||||
}
|
||||
|
||||
export async function collectRun(opts: CollectorOptions): Promise<RunMetrics> {
|
||||
const {
|
||||
specDir,
|
||||
projectName,
|
||||
plan2codeRoot,
|
||||
plan2codeVersion,
|
||||
outputDir,
|
||||
} = opts;
|
||||
|
||||
const runId = generateRunId();
|
||||
|
||||
// Get started_at / completed_at from spec directory mtime / iteration.log
|
||||
let startedAt: string | null = null;
|
||||
let completedAt: string | null = null;
|
||||
try {
|
||||
const stat = fs.statSync(specDir);
|
||||
startedAt = stat.birthtime.toISOString();
|
||||
completedAt = stat.mtime.toISOString();
|
||||
} catch {
|
||||
// leave null
|
||||
}
|
||||
|
||||
// Try to get started_at from earliest iteration log entry
|
||||
const loopDir = path.join(specDir, '.plan2code-loop');
|
||||
const iterLogPath = path.join(loopDir, 'iteration.log');
|
||||
const iterLogContent = readFileSafe(iterLogPath);
|
||||
if (iterLogContent) {
|
||||
const lines = iterLogContent.split('\n').filter(l => l.trim());
|
||||
if (lines.length > 0) {
|
||||
try {
|
||||
const first = JSON.parse(lines[0]);
|
||||
if (first.timestamp) startedAt = first.timestamp;
|
||||
} catch { /* ignore */ }
|
||||
try {
|
||||
const last = JSON.parse(lines[lines.length - 1]);
|
||||
if (last.timestamp) completedAt = last.timestamp;
|
||||
} catch { /* ignore */ }
|
||||
}
|
||||
}
|
||||
|
||||
const metrics: RunMetrics = {
|
||||
schema_version: '1.0',
|
||||
run_id: runId,
|
||||
plan2code_version: plan2codeVersion,
|
||||
prompt_versions: collectPromptVersions(plan2codeRoot),
|
||||
project: {
|
||||
name: projectName,
|
||||
started_at: startedAt,
|
||||
completed_at: completedAt,
|
||||
},
|
||||
step1_plan: collectStep1(specDir),
|
||||
step2_document: collectStep2(specDir),
|
||||
step3_implement: collectStep3(specDir),
|
||||
step4_finalize: collectStep4(specDir),
|
||||
};
|
||||
|
||||
// Write to output dir
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
const outPath = path.join(outputDir, `${runId}.json`);
|
||||
fs.writeFileSync(outPath, JSON.stringify(metrics, null, 2), 'utf8');
|
||||
|
||||
return metrics;
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { validateEdit, parseProposalFromResponse } from './improver.js';
|
||||
import type { PromptEdit } from './types.js';
|
||||
|
||||
// ── Shared fixtures ───────────────────────────────────────────────────────────
|
||||
|
||||
const PROMPT_CONTENTS: Record<string, string> = {
|
||||
'plan2code-1--plan.md': 'This is the plan prompt content. It has some text here.',
|
||||
'plan2code-2--document.md': 'Document prompt with repeated text. repeated text. Done.',
|
||||
'plan2code-3--implement.md': 'Implement prompt content.',
|
||||
};
|
||||
|
||||
function makeEdit(overrides: Partial<PromptEdit> = {}): PromptEdit {
|
||||
return {
|
||||
file: 'plan2code-1--plan.md',
|
||||
rationale: 'test rationale',
|
||||
expected_metric_impact: 'test impact',
|
||||
char_count_before: PROMPT_CONTENTS['plan2code-1--plan.md'].length,
|
||||
char_count_after: PROMPT_CONTENTS['plan2code-1--plan.md'].length,
|
||||
char_count_delta: 0,
|
||||
old_text: 'some text',
|
||||
new_text: 'better text',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
// ── validateEdit() ────────────────────────────────────────────────────────────
|
||||
|
||||
describe('validateEdit', () => {
|
||||
it('valid edit passes with no errors', () => {
|
||||
const result = validateEdit(makeEdit(), PROMPT_CONTENTS);
|
||||
expect(result.valid).toBe(true);
|
||||
expect(result.errors).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('rejects path traversal (../ in file path)', () => {
|
||||
const result = validateEdit(makeEdit({ file: '../etc/passwd' }), PROMPT_CONTENTS);
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.errors[0]).toMatch(/path traversal/i);
|
||||
});
|
||||
|
||||
it('rejects absolute paths', () => {
|
||||
const result = validateEdit(makeEdit({ file: '/etc/passwd' }), PROMPT_CONTENTS);
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.errors[0]).toMatch(/path traversal|absolute/i);
|
||||
});
|
||||
|
||||
it('errors when file not found in promptContents', () => {
|
||||
const result = validateEdit(makeEdit({ file: 'nonexistent.md' }), PROMPT_CONTENTS);
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.errors[0]).toMatch(/not found/i);
|
||||
});
|
||||
|
||||
it('errors when old_text not found in file content', () => {
|
||||
const result = validateEdit(makeEdit({ old_text: 'hallucinated text' }), PROMPT_CONTENTS);
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.errors[0]).toMatch(/not found verbatim/i);
|
||||
});
|
||||
|
||||
it('warns when old_text appears multiple times', () => {
|
||||
const edit = makeEdit({
|
||||
file: 'plan2code-2--document.md',
|
||||
old_text: 'repeated text',
|
||||
char_count_before: PROMPT_CONTENTS['plan2code-2--document.md'].length,
|
||||
char_count_after: PROMPT_CONTENTS['plan2code-2--document.md'].length,
|
||||
});
|
||||
const result = validateEdit(edit, PROMPT_CONTENTS);
|
||||
expect(result.valid).toBe(true);
|
||||
expect(result.warnings.some(w => /appears.*times/i.test(w))).toBe(true);
|
||||
});
|
||||
|
||||
it('errors when edit would exceed 11,000 char limit', () => {
|
||||
const bigText = 'x'.repeat(12_000);
|
||||
const edit = makeEdit({ new_text: bigText });
|
||||
const result = validateEdit(edit, PROMPT_CONTENTS);
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.errors.some(e => /exceed.*11.?000/i.test(e))).toBe(true);
|
||||
});
|
||||
|
||||
it('warns when reported char counts diverge from actual (>10 chars off)', () => {
|
||||
const edit = makeEdit({
|
||||
char_count_before: 999,
|
||||
char_count_after: 999,
|
||||
});
|
||||
const result = validateEdit(edit, PROMPT_CONTENTS);
|
||||
expect(result.valid).toBe(true);
|
||||
expect(result.warnings.some(w => /char_count_before.*differs/i.test(w))).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ── parseProposalFromResponse() ───────────────────────────────────────────────
|
||||
|
||||
describe('parseProposalFromResponse', () => {
|
||||
const sampleEdit = {
|
||||
file: 'test.md',
|
||||
rationale: 'r',
|
||||
expected_metric_impact: 'e',
|
||||
char_count_before: 100,
|
||||
char_count_after: 110,
|
||||
char_count_delta: 10,
|
||||
old_text: 'old',
|
||||
new_text: 'new',
|
||||
};
|
||||
|
||||
it('parses JSON from markdown code block (```json ... ```)', () => {
|
||||
const response = `Here is my proposal:\n\n\`\`\`json\n${JSON.stringify([sampleEdit])}\n\`\`\`\n\nDone.`;
|
||||
const result = parseProposalFromResponse(response);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result![0].file).toBe('test.md');
|
||||
});
|
||||
|
||||
it('parses JSON from bare code block (``` ... ```)', () => {
|
||||
const response = `Proposal:\n\n\`\`\`\n${JSON.stringify([sampleEdit])}\n\`\`\``;
|
||||
const result = parseProposalFromResponse(response);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result![0].old_text).toBe('old');
|
||||
});
|
||||
|
||||
it('parses bare JSON array with old_text field', () => {
|
||||
const response = `Some preamble\n${JSON.stringify([sampleEdit])}\nSome postamble`;
|
||||
const result = parseProposalFromResponse(response);
|
||||
expect(result).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('returns null for non-JSON response', () => {
|
||||
const result = parseProposalFromResponse('No changes needed at this time.');
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for malformed JSON', () => {
|
||||
const result = parseProposalFromResponse('```json\n{broken json]\n```');
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null for empty response', () => {
|
||||
const result = parseProposalFromResponse('');
|
||||
expect(result).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,288 @@
|
||||
/**
|
||||
* improver.ts
|
||||
* Reads diagnosis + prompt files, invokes AI, parses PromptEdit[] from response.
|
||||
* Validates: old_text verbatim match, char count limits.
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import type { PromptEdit, PromptProposal } from './types.js';
|
||||
import { invokeLLM, type AgentType } from './invoke-llm.js';
|
||||
|
||||
const CHAR_LIMIT = 11_000;
|
||||
const IMPROVE_PROMPT_PATH = new URL('../src/prompts/improve.md', import.meta.url).pathname
|
||||
.replace(/^\/([A-Za-z]:)/, '$1'); // Fix Windows path
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
function interpolate(template: string, vars: Record<string, string>): string {
|
||||
let result = template;
|
||||
for (const [key, value] of Object.entries(vars)) {
|
||||
result = result.replaceAll(`{{${key}}}`, value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function generateProposalId(): string {
|
||||
const now = new Date();
|
||||
const ts = now.toISOString().replace(/[-:T.Z]/g, '').slice(0, 14);
|
||||
return `prop-${ts}`;
|
||||
}
|
||||
|
||||
function readPromptFiles(plan2codeRoot: string): Record<string, string> {
|
||||
const srcDir = path.join(plan2codeRoot, 'src');
|
||||
const promptFiles = [
|
||||
'plan2code-1--plan.md',
|
||||
'plan2code-1b--revise-plan.md',
|
||||
'plan2code-2--document.md',
|
||||
'plan2code-3--implement.md',
|
||||
'plan2code-4--finalize.md',
|
||||
'plan2code---init.md',
|
||||
'plan2code---init-update.md',
|
||||
'plan2code---quick-task.md',
|
||||
];
|
||||
|
||||
const contents: Record<string, string> = {};
|
||||
for (const file of promptFiles) {
|
||||
try {
|
||||
contents[file] = fs.readFileSync(path.join(srcDir, file), 'utf8');
|
||||
} catch {
|
||||
contents[file] = '';
|
||||
}
|
||||
}
|
||||
return contents;
|
||||
}
|
||||
|
||||
// ── Edit validation ───────────────────────────────────────────────────────────
|
||||
|
||||
export interface ValidationResult {
|
||||
valid: boolean;
|
||||
errors: string[];
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
export function validateEdit(
|
||||
edit: PromptEdit,
|
||||
promptContents: Record<string, string>,
|
||||
): ValidationResult {
|
||||
const errors: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
|
||||
// Reject path traversal attempts
|
||||
if (edit.file.includes('..') || path.isAbsolute(edit.file)) {
|
||||
errors.push(`Rejected: "${edit.file}" contains path traversal or absolute path.`);
|
||||
return { valid: false, errors, warnings };
|
||||
}
|
||||
|
||||
// Check target file exists
|
||||
const fileContent = promptContents[edit.file];
|
||||
if (fileContent === undefined) {
|
||||
errors.push(`Target file "${edit.file}" not found. Valid files: ${Object.keys(promptContents).join(', ')}`);
|
||||
return { valid: false, errors, warnings };
|
||||
}
|
||||
|
||||
// Check old_text exists verbatim in the file
|
||||
if (!fileContent.includes(edit.old_text)) {
|
||||
errors.push(`old_text not found verbatim in "${edit.file}". The AI may have hallucinated text.`);
|
||||
} else {
|
||||
// Warn if old_text appears more than once (ambiguous match)
|
||||
const occurrences = fileContent.split(edit.old_text).length - 1;
|
||||
if (occurrences > 1) {
|
||||
warnings.push(`old_text appears ${occurrences} times in "${edit.file}". Only the first occurrence will be replaced.`);
|
||||
}
|
||||
}
|
||||
|
||||
// Check char count after edit
|
||||
const afterContent = fileContent.replace(edit.old_text, edit.new_text);
|
||||
if (afterContent.length > CHAR_LIMIT) {
|
||||
errors.push(`Edit would cause "${edit.file}" to exceed ${CHAR_LIMIT} char limit (would be ${afterContent.length} chars).`);
|
||||
}
|
||||
|
||||
// Verify reported char counts match reality
|
||||
const actualBefore = fileContent.length;
|
||||
const actualAfter = afterContent.length;
|
||||
if (Math.abs(edit.char_count_before - actualBefore) > 10) {
|
||||
warnings.push(`Reported char_count_before (${edit.char_count_before}) differs from actual (${actualBefore}).`);
|
||||
}
|
||||
if (Math.abs(edit.char_count_after - actualAfter) > 10) {
|
||||
warnings.push(`Reported char_count_after (${edit.char_count_after}) differs from actual (${actualAfter}).`);
|
||||
}
|
||||
|
||||
return { valid: errors.length === 0, errors, warnings };
|
||||
}
|
||||
|
||||
// ── AI response parsing ───────────────────────────────────────────────────────
|
||||
|
||||
export function parseProposalFromResponse(response: string): PromptEdit[] | null {
|
||||
// Look for JSON code block containing PromptEdit[]
|
||||
const jsonBlockMatch = response.match(/```(?:json)?\s*(\[[\s\S]*?\])\s*```/);
|
||||
if (!jsonBlockMatch) {
|
||||
// Try bare JSON array
|
||||
const bareMatch = response.match(/(\[[\s\S]*"old_text"[\s\S]*\])/);
|
||||
if (!bareMatch) return null;
|
||||
try {
|
||||
return JSON.parse(bareMatch[1]) as PromptEdit[];
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(jsonBlockMatch[1]) as PromptEdit[];
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Main improver ─────────────────────────────────────────────────────────────
|
||||
|
||||
export interface ImproverOptions {
|
||||
diagnosisPath: string; // Path to diagnosis markdown file
|
||||
plan2codeRoot: string; // Path to plan2code repo root
|
||||
proposalsDir: string; // Where to save proposal JSON
|
||||
runsDir: string; // For tracking which runs this is based on
|
||||
model?: string;
|
||||
agent?: AgentType; // Agent to use (default: claude-code)
|
||||
}
|
||||
|
||||
export interface ImproverResult {
|
||||
proposalPath: string;
|
||||
proposal: PromptProposal;
|
||||
validationResults: Array<{ edit: PromptEdit; result: ValidationResult }>;
|
||||
validEditCount: number;
|
||||
invalidEditCount: number;
|
||||
}
|
||||
|
||||
export async function generateImprovement(opts: ImproverOptions): Promise<ImproverResult> {
|
||||
const { diagnosisPath, plan2codeRoot, proposalsDir, runsDir, model = 'claude-opus-4-6', agent = 'claude-code' } = opts;
|
||||
|
||||
// Load diagnosis
|
||||
let diagnosisContent: string;
|
||||
try {
|
||||
diagnosisContent = fs.readFileSync(diagnosisPath, 'utf8');
|
||||
} catch {
|
||||
throw new Error(`Could not read diagnosis file at ${diagnosisPath}`);
|
||||
}
|
||||
|
||||
// Read prompt files
|
||||
const promptContents = readPromptFiles(plan2codeRoot);
|
||||
const srcDir = path.join(plan2codeRoot, 'src');
|
||||
|
||||
// Build char counts for each file
|
||||
const charCounts = Object.entries(promptContents)
|
||||
.map(([file, content]) => `| ${file} | ${content.length} | ${CHAR_LIMIT} | ${CHAR_LIMIT - content.length} headroom |`)
|
||||
.join('\n');
|
||||
|
||||
const promptContentsStr = Object.entries(promptContents)
|
||||
.map(([file, content]) => `## ${file} (${content.length} chars)\n\n${content}`)
|
||||
.join('\n\n---\n\n');
|
||||
|
||||
// Load improve prompt template
|
||||
let improveTemplate: string;
|
||||
try {
|
||||
improveTemplate = fs.readFileSync(IMPROVE_PROMPT_PATH, 'utf8');
|
||||
} catch {
|
||||
const altPath = path.join(process.cwd(), 'src', 'prompts', 'improve.md');
|
||||
improveTemplate = fs.readFileSync(altPath, 'utf8');
|
||||
}
|
||||
|
||||
const fullPrompt = interpolate(improveTemplate, {
|
||||
diagnosisContent,
|
||||
promptContents: promptContentsStr,
|
||||
charCounts: `| File | Current Chars | Limit | Headroom |\n|------|--------------|-------|----------|\n${charCounts}`,
|
||||
});
|
||||
|
||||
// Invoke Claude
|
||||
console.log(`\nInvoking AI improvement proposal (model: ${model})...`);
|
||||
console.log('This may take a minute...\n');
|
||||
|
||||
let aiResponse: string;
|
||||
try {
|
||||
aiResponse = await invokeLLM({
|
||||
prompt: fullPrompt,
|
||||
model,
|
||||
agent,
|
||||
timeout: 300_000,
|
||||
});
|
||||
} catch (err) {
|
||||
throw new Error(`AI invocation failed: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
// Parse edits
|
||||
const rawEdits = parseProposalFromResponse(aiResponse);
|
||||
if (!rawEdits || rawEdits.length === 0) {
|
||||
throw new Error('Could not parse PromptEdit[] from AI response. The AI may not have produced a valid JSON block.');
|
||||
}
|
||||
|
||||
// Enforce max edits per cycle
|
||||
const MAX_EDITS = 5;
|
||||
if (rawEdits.length > MAX_EDITS) {
|
||||
console.warn(`\n⚠ AI generated ${rawEdits.length} edits (max is ${MAX_EDITS}). Truncating to first ${MAX_EDITS}.`);
|
||||
rawEdits.length = MAX_EDITS;
|
||||
}
|
||||
|
||||
// Validate each edit
|
||||
const validationResults: ImproverResult['validationResults'] = [];
|
||||
const validEdits: PromptEdit[] = [];
|
||||
|
||||
for (const edit of rawEdits) {
|
||||
const result = validateEdit(edit, promptContents);
|
||||
validationResults.push({ edit, result });
|
||||
|
||||
if (result.valid) {
|
||||
// Compute accurate char counts
|
||||
const fileContent = promptContents[edit.file] ?? '';
|
||||
const afterContent = fileContent.replace(edit.old_text, edit.new_text);
|
||||
edit.char_count_before = fileContent.length;
|
||||
edit.char_count_after = afterContent.length;
|
||||
edit.char_count_delta = afterContent.length - fileContent.length;
|
||||
validEdits.push(edit);
|
||||
} else {
|
||||
console.warn(`\n⚠ Edit rejected for "${edit.file}":`);
|
||||
for (const err of result.errors) {
|
||||
console.warn(` - ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const warn of result.warnings) {
|
||||
console.warn(` Warning: ${warn}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Get run IDs that contributed to this analysis
|
||||
const runIds: string[] = [];
|
||||
try {
|
||||
const files = fs.readdirSync(runsDir)
|
||||
.filter(f => f.startsWith('run-') && f.endsWith('.json'));
|
||||
runIds.push(...files.map(f => f.replace('.json', '')));
|
||||
} catch { /* no runs dir */ }
|
||||
|
||||
// Build proposal
|
||||
const proposalId = generateProposalId();
|
||||
const proposal: PromptProposal = {
|
||||
proposal_id: proposalId,
|
||||
created_at: new Date().toISOString(),
|
||||
based_on_runs: runIds,
|
||||
analyst_model: model,
|
||||
proposals: validEdits,
|
||||
status: 'pending',
|
||||
diagnosis_file: path.basename(diagnosisPath),
|
||||
};
|
||||
|
||||
// Save proposal JSON
|
||||
fs.mkdirSync(proposalsDir, { recursive: true });
|
||||
const proposalPath = path.join(proposalsDir, `${proposalId}.json`);
|
||||
fs.writeFileSync(proposalPath, JSON.stringify(proposal, null, 2), 'utf8');
|
||||
|
||||
// Also save raw AI response alongside
|
||||
const rawPath = path.join(proposalsDir, `${proposalId}-raw.md`);
|
||||
fs.writeFileSync(rawPath, aiResponse, 'utf8');
|
||||
|
||||
return {
|
||||
proposalPath,
|
||||
proposal,
|
||||
validationResults,
|
||||
validEditCount: validEdits.length,
|
||||
invalidEditCount: rawEdits.length - validEdits.length,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
// Public API for plan2code-metrics
|
||||
export { collectRun, collectPromptVersions } from './collector.js';
|
||||
export { aggregate, loadAggregated, loadRunFiles, importRun } from './aggregator.js';
|
||||
export { runAnalysis } from './analyzer.js';
|
||||
export { generateImprovement, validateEdit, parseProposalFromResponse } from './improver.js';
|
||||
export { reviewAndApply } from './applier.js';
|
||||
export { invokeLLM, AGENTS } from './invoke-llm.js';
|
||||
export type { AgentType, InvokeLLMOptions } from './invoke-llm.js';
|
||||
export { runCLI } from './cli.js';
|
||||
export { METRIC_TARGETS } from './types.js';
|
||||
export type {
|
||||
RunMetrics,
|
||||
PromptVersions,
|
||||
PromptEdit,
|
||||
PromptProposal,
|
||||
AggregatedMetrics,
|
||||
CohortMetrics,
|
||||
} from './types.js';
|
||||
@@ -0,0 +1,98 @@
|
||||
/**
|
||||
* invoke-llm.ts
|
||||
* Unified LLM invocation for plan2code-metrics.
|
||||
* Supports Claude Code (temp file → stdin) and Copilot CLI (stdin string).
|
||||
* Mirrors the agent pattern from plan2code-loop.
|
||||
*/
|
||||
|
||||
import { execa } from 'execa';
|
||||
import { writeFileSync, unlinkSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
|
||||
// ── Agent definitions ────────────────────────────────────────────────────────
|
||||
|
||||
export type AgentType = 'claude-code' | 'copilot-cli';
|
||||
|
||||
export interface AgentDef {
|
||||
name: AgentType;
|
||||
displayName: string;
|
||||
command: string;
|
||||
models: Array<{ value: string; label: string }>;
|
||||
}
|
||||
|
||||
export const AGENTS: Record<AgentType, AgentDef> = {
|
||||
'claude-code': {
|
||||
name: 'claude-code',
|
||||
displayName: 'Claude Code',
|
||||
command: 'claude',
|
||||
models: [
|
||||
{ value: 'claude-opus-4-6', label: 'Claude Opus 4.6 (Recommended)' },
|
||||
{ value: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6 (faster)' },
|
||||
],
|
||||
},
|
||||
'copilot-cli': {
|
||||
name: 'copilot-cli',
|
||||
displayName: 'GitHub Copilot CLI',
|
||||
command: 'copilot',
|
||||
models: [
|
||||
{ value: 'claude-sonnet-4', label: 'Claude Sonnet 4 (Default)' },
|
||||
{ value: 'claude-sonnet-4.5', label: 'Claude Sonnet 4.5' },
|
||||
{ value: 'claude-opus-4.5', label: 'Claude Opus 4.5' },
|
||||
{ value: 'gpt-5', label: 'GPT-5' },
|
||||
{ value: 'gpt-5-mini', label: 'GPT-5 Mini' },
|
||||
{ value: 'gemini-3-pro-preview', label: 'Gemini 3 Pro' },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
// ── Invocation ───────────────────────────────────────────────────────────────
|
||||
|
||||
export interface InvokeLLMOptions {
|
||||
prompt: string;
|
||||
model: string;
|
||||
agent: AgentType;
|
||||
timeout?: number;
|
||||
}
|
||||
|
||||
export async function invokeLLM(opts: InvokeLLMOptions): Promise<string> {
|
||||
const { prompt, model, agent, timeout = 300_000 } = opts;
|
||||
const def = AGENTS[agent];
|
||||
|
||||
if (agent === 'claude-code') {
|
||||
// Write prompt to temp file — more reliable than stdin on Windows
|
||||
const tempFile = join(tmpdir(), `plan2code-metrics-prompt-${Date.now()}.txt`);
|
||||
writeFileSync(tempFile, prompt, 'utf-8');
|
||||
|
||||
try {
|
||||
const args: string[] = [
|
||||
'--print',
|
||||
'--dangerously-skip-permissions',
|
||||
];
|
||||
if (model) {
|
||||
args.push('--model', model);
|
||||
}
|
||||
|
||||
const result = await execa(def.command, args, {
|
||||
inputFile: tempFile,
|
||||
timeout,
|
||||
});
|
||||
return result.stdout;
|
||||
} finally {
|
||||
try { unlinkSync(tempFile); } catch { /* ignore cleanup errors */ }
|
||||
}
|
||||
} else {
|
||||
// Copilot CLI: pipe prompt via stdin string
|
||||
const args: string[] = [];
|
||||
if (model) {
|
||||
args.push('--model', model);
|
||||
}
|
||||
args.push('--allow-all-tools', '-s');
|
||||
|
||||
const result = await execa(def.command, args, {
|
||||
input: prompt,
|
||||
timeout,
|
||||
});
|
||||
return result.stdout;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
# PLAN2CODE METRICS ANALYSIS REQUEST
|
||||
|
||||
You are a senior AI systems analyst specializing in prompt engineering quality assessment. Your role is to diagnose weaknesses in the plan2code workflow prompts by examining aggregated run metrics.
|
||||
|
||||
**IMPORTANT:** Do NOT propose specific edits in this response. Diagnosis only. The improvement step is separate.
|
||||
|
||||
---
|
||||
|
||||
## Aggregated Run Metrics
|
||||
|
||||
The following JSON contains metrics aggregated from real plan2code project runs, grouped by prompt "generation" (a cohort is identified by the SHA fingerprint of the src/plan2code-*.md prompt files at collection time):
|
||||
|
||||
```json
|
||||
{{aggregatedMetrics}}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Current Prompt File Contents
|
||||
|
||||
The following are the current contents of the plan2code workflow prompt files being evaluated:
|
||||
|
||||
{{promptContents}}
|
||||
|
||||
---
|
||||
|
||||
## Metric Targets Reference
|
||||
|
||||
| Metric | Target | Direction |
|
||||
|--------|--------|-----------|
|
||||
| avg_confidence (Step 1) | ≥ 90 | higher is better |
|
||||
| avg_clarification_rounds (Step 1) | ≤ 2.0 | lower is better |
|
||||
| avg_verification_gaps_found (Step 1) | ≤ 2.0 | lower is better |
|
||||
| avg_parallel_groups (Step 2) | ≥ 0.5 | higher is better |
|
||||
| avg_verification_items_added (Step 2) | ≤ 1.5 | lower is better |
|
||||
| avg_task_completion_rate (Step 3) | ≥ 0.95 | higher is better |
|
||||
| avg_blocker_count (Step 3) | ≤ 1.5 | lower is better |
|
||||
| avg_completion_marker_success_rate (Step 3) | ≥ 0.95 | higher is better |
|
||||
| avg_verification_failures_found (Step 4) | ≤ 1.0 | lower is better |
|
||||
| archival_success_rate (Step 4) | ≥ 0.99 | higher is better |
|
||||
|
||||
---
|
||||
|
||||
## Analysis Instructions
|
||||
|
||||
1. Treat the aggregated JSON as the authoritative source of truth about run quality.
|
||||
2. Compare each metric against its target. Calculate delta (actual − target).
|
||||
3. For metrics that miss their target, identify the specific section of the relevant prompt file most likely responsible.
|
||||
4. Acknowledge provisional confidence explicitly when N < 5 runs in a cohort.
|
||||
5. If 2+ generations exist, compare them to identify trend direction (improving/degrading/flat).
|
||||
6. Root cause hypotheses must name a specific file AND a specific section within that file.
|
||||
7. Do not invent metrics not present in the JSON. If a metric is null, note it as "insufficient data."
|
||||
|
||||
---
|
||||
|
||||
## Required Output Format
|
||||
|
||||
Produce EXACTLY the following sections in order. Use these exact headers — they are parsed by machine:
|
||||
|
||||
# PLAN2CODE METRICS DIAGNOSIS
|
||||
|
||||
### Metrics Summary
|
||||
|
||||
A markdown table with columns: Step | Metric | Target | Actual | Delta | Status (✓/✗/—)
|
||||
|
||||
Include ALL metrics listed in the targets table. Use "—" for null values.
|
||||
|
||||
### Step Health Assessment
|
||||
|
||||
For each step (1–4), provide:
|
||||
- **Grade:** A–F
|
||||
- **Key signals:** 2–4 bullet points with specific metric values
|
||||
- **Assessment:** 1–2 sentence diagnosis
|
||||
|
||||
### Root Cause Hypotheses
|
||||
|
||||
Numbered list. For each underperforming metric:
|
||||
1. **Metric:** [metric name] | **Value:** [actual] | **Target:** [target]
|
||||
- **File:** [plan2code-X--name.md]
|
||||
- **Section:** [specific heading or section name]
|
||||
- **Hypothesis:** [specific gap in the prompt that would explain the metric miss]
|
||||
- **Confidence:** [High/Medium/Low] — [reason for confidence level]
|
||||
|
||||
### Recommended Improvement Targets
|
||||
|
||||
Ordered list (highest estimated impact first). For each:
|
||||
- **File:** [filename]
|
||||
- **Section:** [section name]
|
||||
- **Why:** [link to specific metric being addressed]
|
||||
- **Priority:** [High/Medium/Low]
|
||||
|
||||
### Generation Comparison
|
||||
|
||||
If 2+ generations exist: A comparison table showing before/after for each metric per generation, with trend arrows (▲/▼/→).
|
||||
|
||||
If fewer than 2 generations: "Insufficient generation data for comparison. Current generation: [cohort_key], [N] runs."
|
||||
|
||||
---
|
||||
|
||||
End of analysis request.
|
||||
@@ -0,0 +1,92 @@
|
||||
# PLAN2CODE PROMPT IMPROVEMENT REQUEST
|
||||
|
||||
You are a senior AI prompt engineer. Your role is to propose surgical, targeted edits to the plan2code workflow prompt files based on a metrics diagnosis. You make precise, minimal changes — NOT rewrites.
|
||||
|
||||
---
|
||||
|
||||
## Metrics Diagnosis
|
||||
|
||||
The following diagnosis was produced by the analysis step:
|
||||
|
||||
{{diagnosisContent}}
|
||||
|
||||
---
|
||||
|
||||
## Current Prompt File Contents (with char counts)
|
||||
|
||||
{{promptContents}}
|
||||
|
||||
---
|
||||
|
||||
## Character Count Status
|
||||
|
||||
{{charCounts}}
|
||||
|
||||
---
|
||||
|
||||
## Hard Constraints — ALL must be satisfied:
|
||||
|
||||
1. **Char limit:** Each target file MUST stay under 11,000 characters after your edit is applied. This is enforced by code — edits that violate it will be automatically rejected.
|
||||
2. **Maximum 5 edits per cycle.** Focus on the highest-impact changes only.
|
||||
3. **Each edit must cite a specific metric** in its `expected_metric_impact` field (e.g., "avg_completion_marker_success_rate", "avg_blocker_count").
|
||||
4. **`old_text` must be verbatim** from the file. Copy-paste exactly — include surrounding whitespace/newlines as they appear. Edits with mismatched old_text will be automatically rejected.
|
||||
5. **Do NOT modify Role sections** (lines starting with "You are" at the top of each file) or step headings (lines starting with `#`).
|
||||
6. **Each edit must be independently applicable** — no edit should depend on another edit being applied first.
|
||||
7. **Prefer additive guidance over deletions.** Adding clarifying instructions or examples is safer than removing existing text.
|
||||
8. **Do not change the overall structure** or flow of any prompt file.
|
||||
|
||||
---
|
||||
|
||||
## Edit Strategy Guidelines
|
||||
|
||||
- Target the specific sections identified in "Recommended Improvement Targets" from the diagnosis.
|
||||
- For high `avg_clarification_rounds`: Add more upfront specification examples or decision criteria to Step 1.
|
||||
- For low `avg_completion_marker_success_rate`: Clarify or simplify the completion marker format in Step 3.
|
||||
- For high `avg_blocker_count`: Add blocker-recovery guidance or prerequisite check instructions.
|
||||
- For low `avg_confidence`: Strengthen the confidence calculation instructions with clearer rubrics.
|
||||
- For low `avg_parallel_groups`: Add explicit guidance for identifying parallel tasks in Step 2.
|
||||
- Keep each `new_text` as short as possible while still addressing the root cause.
|
||||
|
||||
---
|
||||
|
||||
## Required Output Format
|
||||
|
||||
Produce EXACTLY the following sections. The JSON block is parsed by machine — it must be syntactically valid.
|
||||
|
||||
# PLAN2CODE PROMPT IMPROVEMENT PROPOSAL
|
||||
|
||||
### Improvement Rationale
|
||||
|
||||
2–3 paragraphs explaining:
|
||||
1. Which metrics are being addressed and why they matter
|
||||
2. The specific prompt gaps identified in the diagnosis that you are targeting
|
||||
3. Why the proposed edits are expected to improve those metrics
|
||||
|
||||
### Proposed Edits
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"file": "plan2code-X--name.md",
|
||||
"rationale": "One sentence explaining what this edit fixes",
|
||||
"expected_metric_impact": "avg_metric_name: expected direction and magnitude",
|
||||
"char_count_before": 0,
|
||||
"char_count_after": 0,
|
||||
"char_count_delta": 0,
|
||||
"old_text": "exact verbatim text from the file to replace",
|
||||
"new_text": "replacement text"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Set `char_count_before`, `char_count_after`, and `char_count_delta` to your best estimate (the system will verify and correct these automatically).
|
||||
|
||||
### Character Count Verification
|
||||
|
||||
A table with columns: File | Before | Projected After | Delta | Limit | Status (✓/✗)
|
||||
|
||||
Verify that NO file exceeds 11,000 characters after your proposed edits.
|
||||
|
||||
---
|
||||
|
||||
End of improvement request.
|
||||
@@ -0,0 +1,162 @@
|
||||
// All TypeScript interfaces for plan2code-metrics
|
||||
|
||||
export interface PromptVersions {
|
||||
plan: string; // sha256:... plan2code-1--plan.md
|
||||
revise_plan: string; // plan2code-1b--revise-plan.md
|
||||
document: string; // plan2code-2--document.md
|
||||
implement: string; // plan2code-3--implement.md
|
||||
finalize: string; // plan2code-4--finalize.md
|
||||
init: string; // plan2code---init.md
|
||||
init_update: string; // plan2code---init-update.md
|
||||
quick_task: string; // plan2code---quick-task.md
|
||||
}
|
||||
|
||||
export interface Step1PlanMetrics {
|
||||
present: boolean;
|
||||
final_confidence: number | null;
|
||||
confidence_breakdown: {
|
||||
requirements: number | null;
|
||||
feasibility: number | null;
|
||||
integration: number | null;
|
||||
risk: number | null;
|
||||
} | null;
|
||||
clarification_rounds: number | null;
|
||||
tech_stack_revision_rounds: number | null;
|
||||
verification_gaps_found: number | null;
|
||||
functional_requirements_count: number | null;
|
||||
non_functional_requirements_count: number | null;
|
||||
risk_count: number | null;
|
||||
phase_count: number | null;
|
||||
}
|
||||
|
||||
export interface Step2DocumentMetrics {
|
||||
present: boolean;
|
||||
total_tasks: number | null;
|
||||
tasks_per_phase: number[] | null;
|
||||
phase_count: number | null;
|
||||
parallel_groups_identified: number | null;
|
||||
requirement_coverage_percent: number | null;
|
||||
verification_items_added: number | null;
|
||||
}
|
||||
|
||||
export interface Step3ImplementMetrics {
|
||||
present: boolean;
|
||||
used_loop_mode: boolean;
|
||||
loop_mode: 'task' | 'phase' | null;
|
||||
task_completion_rate: number | null;
|
||||
tasks_completed: number | null;
|
||||
tasks_total: number | null;
|
||||
blocker_count: number | null;
|
||||
blocker_categories: string[] | null;
|
||||
total_iterations: number | null;
|
||||
avg_iteration_duration_ms: number | null;
|
||||
exit_code_distribution: Record<string, number> | null;
|
||||
completion_marker_success_rate: number | null;
|
||||
}
|
||||
|
||||
export interface Step4FinalizeMetrics {
|
||||
present: boolean;
|
||||
completion_rate_at_audit: number | null;
|
||||
verification_failures_found: number | null;
|
||||
documentation_updates_needed: number | null;
|
||||
archival_succeeded: boolean | null;
|
||||
}
|
||||
|
||||
export interface RunMetrics {
|
||||
schema_version: '1.0';
|
||||
run_id: string;
|
||||
plan2code_version: string;
|
||||
prompt_versions: PromptVersions;
|
||||
project: {
|
||||
name: string;
|
||||
started_at: string | null;
|
||||
completed_at: string | null;
|
||||
};
|
||||
step1_plan: Step1PlanMetrics;
|
||||
step2_document: Step2DocumentMetrics;
|
||||
step3_implement: Step3ImplementMetrics;
|
||||
step4_finalize: Step4FinalizeMetrics;
|
||||
}
|
||||
|
||||
export interface PromptEdit {
|
||||
file: string;
|
||||
rationale: string;
|
||||
expected_metric_impact: string;
|
||||
char_count_before: number;
|
||||
char_count_after: number;
|
||||
char_count_delta: number;
|
||||
old_text: string;
|
||||
new_text: string;
|
||||
}
|
||||
|
||||
export interface PromptProposal {
|
||||
proposal_id: string;
|
||||
created_at: string;
|
||||
based_on_runs: string[];
|
||||
analyst_model: string;
|
||||
proposals: PromptEdit[];
|
||||
status: 'pending' | 'applied' | 'rejected';
|
||||
diagnosis_file: string | null;
|
||||
}
|
||||
|
||||
// Aggregated metrics schema
|
||||
export interface CohortMetrics {
|
||||
cohort_key: string; // hash of sorted prompt_versions
|
||||
prompt_versions: PromptVersions;
|
||||
run_count: number;
|
||||
run_ids: string[];
|
||||
first_seen: string;
|
||||
last_seen: string;
|
||||
|
||||
// Step 1 averages
|
||||
avg_confidence: number | null;
|
||||
avg_clarification_rounds: number | null;
|
||||
avg_verification_gaps_found: number | null;
|
||||
avg_functional_requirements_count: number | null;
|
||||
avg_non_functional_requirements_count: number | null;
|
||||
avg_risk_count: number | null;
|
||||
avg_phase_count_step1: number | null;
|
||||
|
||||
// Step 2 averages
|
||||
avg_total_tasks: number | null;
|
||||
avg_phase_count_step2: number | null;
|
||||
avg_parallel_groups: number | null;
|
||||
avg_requirement_coverage_percent: number | null;
|
||||
avg_verification_items_added: number | null;
|
||||
|
||||
// Step 3 averages (loop only)
|
||||
loop_run_count: number;
|
||||
avg_task_completion_rate: number | null;
|
||||
avg_blocker_count: number | null;
|
||||
avg_total_iterations: number | null;
|
||||
avg_iteration_duration_ms: number | null;
|
||||
avg_completion_marker_success_rate: number | null;
|
||||
|
||||
// Step 4 averages
|
||||
avg_completion_rate_at_audit: number | null;
|
||||
avg_verification_failures_found: number | null;
|
||||
avg_documentation_updates_needed: number | null;
|
||||
archival_success_rate: number | null;
|
||||
}
|
||||
|
||||
export interface AggregatedMetrics {
|
||||
schema_version: '1.0';
|
||||
last_updated: string;
|
||||
total_runs: number;
|
||||
cohorts: CohortMetrics[];
|
||||
current_cohort_key: string | null;
|
||||
}
|
||||
|
||||
// Metric targets for health assessment
|
||||
export const METRIC_TARGETS = {
|
||||
avg_confidence: { target: 90, direction: 'gte' as const },
|
||||
avg_clarification_rounds: { target: 2.0, direction: 'lte' as const },
|
||||
avg_verification_gaps_found: { target: 2.0, direction: 'lte' as const },
|
||||
avg_parallel_groups: { target: 0.5, direction: 'gte' as const },
|
||||
avg_verification_items_added: { target: 1.5, direction: 'lte' as const },
|
||||
avg_task_completion_rate: { target: 0.95, direction: 'gte' as const },
|
||||
avg_blocker_count: { target: 1.5, direction: 'lte' as const },
|
||||
avg_completion_marker_success_rate: { target: 0.95, direction: 'gte' as const },
|
||||
avg_verification_failures_found: { target: 1.0, direction: 'lte' as const },
|
||||
archival_success_rate: { target: 0.99, direction: 'gte' as const },
|
||||
} as const;
|
||||
Reference in New Issue
Block a user