mirror of
https://github.com/jparkerweb/plan2code.git
synced 2026-07-21 18:33:22 -07:00
v1.8.0 — add plan2code-metrics recursive self-improvement toolchain
New package for contributors to collect, aggregate, analyze, and improve workflow prompts based on real project data. Includes unit tests (vitest), installer integration, and loop/finalize hints.
This commit is contained in:
@@ -0,0 +1,219 @@
|
||||
/**
|
||||
* aggregator.ts
|
||||
* Merges all run JSON files into aggregated.json, grouped by prompt generation (SHA fingerprint).
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import crypto from 'crypto';
|
||||
import type { RunMetrics, AggregatedMetrics, CohortMetrics, PromptVersions } from './types.js';
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export function avg(values: (number | null | undefined)[]): number | null {
|
||||
const valid = values.filter((v): v is number => v != null && !isNaN(v));
|
||||
if (valid.length === 0) return null;
|
||||
return valid.reduce((a, b) => a + b, 0) / valid.length;
|
||||
}
|
||||
|
||||
export function rate(values: (boolean | null | undefined)[]): number | null {
|
||||
const valid = values.filter((v): v is boolean => v != null);
|
||||
if (valid.length === 0) return null;
|
||||
return valid.filter(v => v).length / valid.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a stable cohort key from the sorted prompt_versions object.
|
||||
* Two runs with identical prompt files get the same cohort key.
|
||||
*/
|
||||
export function buildCohortKey(promptVersions: PromptVersions): string {
|
||||
// Filter out missing sentinels so old runs (4 fields) keep their original key
|
||||
const entries = Object.entries(promptVersions)
|
||||
.filter(([, v]) => v !== 'sha256:missing')
|
||||
.sort(([a], [b]) => a.localeCompare(b));
|
||||
const str = JSON.stringify(Object.fromEntries(entries));
|
||||
return crypto.createHash('sha256').update(str).digest('hex').slice(0, 12);
|
||||
}
|
||||
|
||||
/**
|
||||
* Backfill missing PromptVersions fields for old run files (pre-v1.1).
|
||||
*/
|
||||
export function backfillPromptVersions(pv: PromptVersions): PromptVersions {
|
||||
return {
|
||||
plan: pv.plan ?? 'sha256:missing',
|
||||
revise_plan: pv.revise_plan ?? 'sha256:missing',
|
||||
document: pv.document ?? 'sha256:missing',
|
||||
implement: pv.implement ?? 'sha256:missing',
|
||||
finalize: pv.finalize ?? 'sha256:missing',
|
||||
init: pv.init ?? 'sha256:missing',
|
||||
init_update: pv.init_update ?? 'sha256:missing',
|
||||
quick_task: pv.quick_task ?? 'sha256:missing',
|
||||
};
|
||||
}
|
||||
|
||||
// ── Load run files ────────────────────────────────────────────────────────────
|
||||
|
||||
export function loadRunFiles(runsDir: string): RunMetrics[] {
|
||||
if (!fs.existsSync(runsDir)) return [];
|
||||
|
||||
const files = fs.readdirSync(runsDir)
|
||||
.filter(f => f.endsWith('.json') && f.startsWith('run-'))
|
||||
.sort();
|
||||
|
||||
const runs: RunMetrics[] = [];
|
||||
for (const file of files) {
|
||||
try {
|
||||
const content = fs.readFileSync(path.join(runsDir, file), 'utf8');
|
||||
const data = JSON.parse(content) as RunMetrics;
|
||||
data.prompt_versions = backfillPromptVersions(data.prompt_versions);
|
||||
runs.push(data);
|
||||
} catch (err) {
|
||||
console.warn(`Warning: could not parse ${file}: ${err}`);
|
||||
}
|
||||
}
|
||||
return runs;
|
||||
}
|
||||
|
||||
// ── Build cohort metrics ──────────────────────────────────────────────────────
|
||||
|
||||
function buildCohort(runs: RunMetrics[], cohortKey: string): CohortMetrics {
|
||||
const runIds = runs.map(r => r.run_id);
|
||||
const timestamps = runs
|
||||
.flatMap(r => [r.project.started_at, r.project.completed_at])
|
||||
.filter((t): t is string => t != null)
|
||||
.sort();
|
||||
|
||||
// Step 1 averages
|
||||
const avgConfidence = avg(runs.map(r => r.step1_plan.final_confidence));
|
||||
const avgClarification = avg(runs.map(r => r.step1_plan.clarification_rounds));
|
||||
const avgVerifGaps = avg(runs.map(r => r.step1_plan.verification_gaps_found));
|
||||
const avgFRCount = avg(runs.map(r => r.step1_plan.functional_requirements_count));
|
||||
const avgNFRCount = avg(runs.map(r => r.step1_plan.non_functional_requirements_count));
|
||||
const avgRiskCount = avg(runs.map(r => r.step1_plan.risk_count));
|
||||
const avgPhaseStep1 = avg(runs.map(r => r.step1_plan.phase_count));
|
||||
|
||||
// Step 2 averages
|
||||
const avgTotalTasks = avg(runs.map(r => r.step2_document.total_tasks));
|
||||
const avgPhaseStep2 = avg(runs.map(r => r.step2_document.phase_count));
|
||||
const avgParallelGroups = avg(runs.map(r => r.step2_document.parallel_groups_identified));
|
||||
const avgReqCoverage = avg(runs.map(r => r.step2_document.requirement_coverage_percent));
|
||||
const avgVerifItems = avg(runs.map(r => r.step2_document.verification_items_added));
|
||||
|
||||
// Step 3 (loop only)
|
||||
const loopRuns = runs.filter(r => r.step3_implement.used_loop_mode);
|
||||
const avgCompletionRate = avg(loopRuns.map(r => r.step3_implement.task_completion_rate));
|
||||
const avgBlockerCount = avg(loopRuns.map(r => r.step3_implement.blocker_count));
|
||||
const avgTotalIter = avg(loopRuns.map(r => r.step3_implement.total_iterations));
|
||||
const avgIterDuration = avg(loopRuns.map(r => r.step3_implement.avg_iteration_duration_ms));
|
||||
const avgMarkerSuccess = avg(loopRuns.map(r => r.step3_implement.completion_marker_success_rate));
|
||||
|
||||
// Step 4 averages
|
||||
const avgCompletionAtAudit = avg(runs.map(r => r.step4_finalize.completion_rate_at_audit));
|
||||
const avgVerifFailures = avg(runs.map(r => r.step4_finalize.verification_failures_found));
|
||||
const avgDocUpdates = avg(runs.map(r => r.step4_finalize.documentation_updates_needed));
|
||||
const archivalRate = rate(runs.map(r => r.step4_finalize.archival_succeeded));
|
||||
|
||||
return {
|
||||
cohort_key: cohortKey,
|
||||
prompt_versions: runs[0].prompt_versions,
|
||||
run_count: runs.length,
|
||||
run_ids: runIds,
|
||||
first_seen: timestamps[0] ?? runs[0].run_id,
|
||||
last_seen: timestamps[timestamps.length - 1] ?? runs[runs.length - 1].run_id,
|
||||
|
||||
avg_confidence: avgConfidence,
|
||||
avg_clarification_rounds: avgClarification,
|
||||
avg_verification_gaps_found: avgVerifGaps,
|
||||
avg_functional_requirements_count: avgFRCount,
|
||||
avg_non_functional_requirements_count: avgNFRCount,
|
||||
avg_risk_count: avgRiskCount,
|
||||
avg_phase_count_step1: avgPhaseStep1,
|
||||
|
||||
avg_total_tasks: avgTotalTasks,
|
||||
avg_phase_count_step2: avgPhaseStep2,
|
||||
avg_parallel_groups: avgParallelGroups,
|
||||
avg_requirement_coverage_percent: avgReqCoverage,
|
||||
avg_verification_items_added: avgVerifItems,
|
||||
|
||||
loop_run_count: loopRuns.length,
|
||||
avg_task_completion_rate: avgCompletionRate,
|
||||
avg_blocker_count: avgBlockerCount,
|
||||
avg_total_iterations: avgTotalIter,
|
||||
avg_iteration_duration_ms: avgIterDuration,
|
||||
avg_completion_marker_success_rate: avgMarkerSuccess,
|
||||
|
||||
avg_completion_rate_at_audit: avgCompletionAtAudit,
|
||||
avg_verification_failures_found: avgVerifFailures,
|
||||
avg_documentation_updates_needed: avgDocUpdates,
|
||||
archival_success_rate: archivalRate,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Main aggregator ───────────────────────────────────────────────────────────
|
||||
|
||||
export function aggregate(runsDir: string, outputPath: string): AggregatedMetrics {
|
||||
const runs = loadRunFiles(runsDir);
|
||||
|
||||
// Group by cohort key
|
||||
const cohortMap = new Map<string, RunMetrics[]>();
|
||||
for (const run of runs) {
|
||||
const key = buildCohortKey(run.prompt_versions);
|
||||
if (!cohortMap.has(key)) cohortMap.set(key, []);
|
||||
cohortMap.get(key)!.push(run);
|
||||
}
|
||||
|
||||
// Sort cohorts by first seen
|
||||
const cohorts: CohortMetrics[] = [];
|
||||
for (const [key, cohortRuns] of cohortMap) {
|
||||
cohorts.push(buildCohort(cohortRuns, key));
|
||||
}
|
||||
cohorts.sort((a, b) => a.first_seen.localeCompare(b.first_seen));
|
||||
|
||||
// Determine current cohort (most recent)
|
||||
const currentCohortKey = cohorts.length > 0
|
||||
? cohorts[cohorts.length - 1].cohort_key
|
||||
: null;
|
||||
|
||||
const aggregated: AggregatedMetrics = {
|
||||
schema_version: '1.0',
|
||||
last_updated: new Date().toISOString(),
|
||||
total_runs: runs.length,
|
||||
cohorts,
|
||||
current_cohort_key: currentCohortKey,
|
||||
};
|
||||
|
||||
// Write to output path
|
||||
const dir = path.dirname(outputPath);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(outputPath, JSON.stringify(aggregated, null, 2), 'utf8');
|
||||
|
||||
return aggregated;
|
||||
}
|
||||
|
||||
export function loadAggregated(outputPath: string): AggregatedMetrics | null {
|
||||
try {
|
||||
const content = fs.readFileSync(outputPath, 'utf8');
|
||||
return JSON.parse(content) as AggregatedMetrics;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Import a single run JSON from another project into the local runs dir.
|
||||
* Returns true if imported, false if already present.
|
||||
*/
|
||||
export function importRun(runJsonPath: string, runsDir: string): boolean {
|
||||
const content = fs.readFileSync(runJsonPath, 'utf8');
|
||||
const run = JSON.parse(content) as RunMetrics;
|
||||
run.prompt_versions = backfillPromptVersions(run.prompt_versions);
|
||||
const destPath = path.join(runsDir, `${run.run_id}.json`);
|
||||
|
||||
if (fs.existsSync(destPath)) {
|
||||
return false; // Already imported
|
||||
}
|
||||
|
||||
fs.mkdirSync(runsDir, { recursive: true });
|
||||
fs.writeFileSync(destPath, JSON.stringify(run, null, 2), 'utf8');
|
||||
return true;
|
||||
}
|
||||
Reference in New Issue
Block a user