Harden the shared console install, launcher, Loop staging, spec state and prompts

AI Assisted
This commit is contained in:
jparkerweb committed 2026-11-08 13:48:28 -08:00
1 parent 120c259d9c
commit 3ee96834bf
130 files changed
+5988 -1434

No files matched your search

+16 -9
View File
@@ -56,18 +56,25 @@ export function cohortKeyForRun(run: RunMetrics): string {
}
/**
* Backfill missing PromptVersions fields for old run files (pre-v1.1).
* Backfill missing PromptVersions fields for old run files (pre-v1.1): the
* eight prompts every run once recorded are padded with the missing sentinel
* (buildCohortKey ignores it), and any other prompt a run recorded is kept.
*
* One-time regrouping: a run recorded before the collector fingerprinted every
* prompt has fewer entries than a new one, so its cohort key differs and it
* groups as the previous prompt generation.
*/
export function backfillPromptVersions(pv: PromptVersions): PromptVersions {
return {
plan: pv.plan ?? 'sha256:missing',
revise_plan: pv.revise_plan ?? 'sha256:missing',
document: pv.document ?? 'sha256:missing',
implement: pv.implement ?? 'sha256:missing',
finalize: pv.finalize ?? 'sha256:missing',
init: pv.init ?? 'sha256:missing',
init_update: pv.init_update ?? 'sha256:missing',
quick_task: pv.quick_task ?? 'sha256:missing',
plan: 'sha256:missing',
revise_plan: 'sha256:missing',
document: 'sha256:missing',
implement: 'sha256:missing',
finalize: 'sha256:missing',
init: 'sha256:missing',
init_update: 'sha256:missing',
quick_task: 'sha256:missing',
...(pv ?? {}),
};
}
+101
View File
@@ -0,0 +1,101 @@
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import fs from 'fs';
import os from 'os';
import path from 'path';
import { collectPromptVersions, collectStep1, extractMetricsJson, promptKey } from './collector.js';
import { backfillPromptVersions, buildCohortKey } from './aggregator.js';
let root: string;
beforeEach(() => {
root = fs.mkdtempSync(path.join(os.tmpdir(), 'p2c-collector-'));
fs.mkdirSync(path.join(root, 'src'));
});
afterEach(() => {
fs.rmSync(root, { recursive: true, force: true });
});
function prompt(name: string, body: string) {
fs.writeFileSync(path.join(root, 'src', name), body);
}
describe('collectPromptVersions', () => {
it('fingerprints every src/plan2code-*.md prompt in the folder', () => {
prompt('plan2code-1-plan.md', 'plan');
prompt('plan2code-3-implement-review.md', 'implement review');
prompt('plan2code-git.md', 'git');
const versions = collectPromptVersions(root);
expect(Object.keys(versions).sort()).toEqual(['3_implement_review', 'git', 'plan']);
for (const v of Object.values(versions)) expect(v).toMatch(/^sha256:[0-9a-f]{64}$/);
});
it('covers a newly added prompt without a code change', () => {
prompt('plan2code-1-plan.md', 'plan');
const before = collectPromptVersions(root);
prompt('plan2code-brand-new.md', 'a new skill');
const after = collectPromptVersions(root);
expect(before).not.toHaveProperty('brand_new');
expect(after).toHaveProperty('brand_new');
expect(buildCohortKey(after)).not.toBe(buildCohortKey(before));
});
it('ignores files and folders that are not top-level plan2code-*.md prompts', () => {
prompt('plan2code-1-plan.md', 'plan');
prompt('README.md', 'not a prompt');
prompt('plan2code-notes.txt', 'not markdown');
fs.mkdirSync(path.join(root, 'src', 'plan2code-0-pathfinder-references.md'));
expect(Object.keys(collectPromptVersions(root))).toEqual(['plan']);
});
it('keeps the old keys for the eight prompts recorded before, so their hashes stay comparable', () => {
expect(promptKey('plan2code-1-plan.md')).toBe('plan');
expect(promptKey('plan2code-1b-revise-plan.md')).toBe('revise_plan');
expect(promptKey('plan2code-2-document.md')).toBe('document');
expect(promptKey('plan2code-3-implement.md')).toBe('implement');
expect(promptKey('plan2code-4-finalize.md')).toBe('finalize');
expect(promptKey('plan2code-init.md')).toBe('init');
expect(promptKey('plan2code-init-update.md')).toBe('init_update');
expect(promptKey('plan2code-quick-task.md')).toBe('quick_task');
});
it('returns nothing when there is no src folder', () => {
expect(collectPromptVersions(path.join(root, 'nowhere'))).toEqual({});
});
it('groups a run recorded before this change as the previous prompt generation', () => {
prompt('plan2code-1-plan.md', 'plan');
prompt('plan2code-git.md', 'git');
const now = collectPromptVersions(root);
const old = backfillPromptVersions({ plan: now.plan });
expect(buildCohortKey(old)).not.toBe(buildCohortKey(now));
});
});
describe('the shared scrape pattern', () => {
function draft(body: string) {
const specDir = path.join(root, 'spec');
fs.mkdirSync(specDir);
fs.writeFileSync(path.join(specDir, 'PLAN-DRAFT-1.md'), body);
return collectStep1(specDir);
}
it('reads the METRICS_JSON comment', () => {
expect(extractMetricsJson('x <!-- METRICS_JSON {"confidence": 95} --> y')).toEqual({ confidence: 95 });
});
it('reads a breakdown written as label, colon, score', () => {
const m = draft('Requirements: 24\nFeasibility: 23\nIntegration: 22\nRisk: 21\n');
expect(m.confidence_breakdown).toEqual({ requirements: 24, feasibility: 23, integration: 22, risk: 21 });
});
it('reads a breakdown table whose score ends the row', () => {
const m = draft('| Requirements | 24 |\n| Risk | 21/25 |\n');
expect(m.confidence_breakdown).toEqual({ requirements: 24, feasibility: null, integration: null, risk: 21 });
});
it('leaves an ordinary risk-register row alone', () => {
const m = draft('| Risk | 3 | High | Mitigation |\n');
expect(m.confidence_breakdown).toBeNull();
});
});
+49 -18
View File
@@ -16,6 +16,7 @@ import type {
Step4FinalizeMetrics,
UserFeedback,
} from './types.js';
import metricsPattern from './metrics-pattern.json' with { type: 'json' };
// ── Helpers ──────────────────────────────────────────────────────────────────
@@ -44,7 +45,7 @@ function readFileSafe(filePath: string): string | null {
* Returns parsed object or null if not found / invalid.
*/
export function extractMetricsJson(content: string, stepFilter?: string): Record<string, unknown> | null {
const re = /<!--\s*METRICS_JSON\s+(\{[\s\S]*?\})\s*-->/g;
const re = new RegExp(metricsPattern.metricsComment, 'g');
let match: RegExpExecArray | null;
while ((match = re.exec(content)) !== null) {
try {
@@ -96,18 +97,38 @@ function generateRunId(): string {
// ── Prompt version hashing ────────────────────────────────────────────────────
// The prompts recorded before the collector read the folder keep their keys.
const LEGACY_PROMPT_KEYS: Record<string, string> = {
'plan2code-1-plan.md': 'plan',
'plan2code-1b-revise-plan.md': 'revise_plan',
'plan2code-2-document.md': 'document',
'plan2code-3-implement.md': 'implement',
'plan2code-4-finalize.md': 'finalize',
'plan2code-init.md': 'init',
'plan2code-init-update.md': 'init_update',
'plan2code-quick-task.md': 'quick_task',
};
/** The PromptVersions key for a prompt file: its legacy key, else its name without "plan2code-" and ".md", dashes as underscores. */
export function promptKey(fileName: string): string {
return LEGACY_PROMPT_KEYS[fileName]
?? fileName.replace(/^plan2code-/, '').replace(/\.md$/, '').replace(/-/g, '_');
}
/** Fingerprint every src/plan2code-*.md prompt, read from the folder so a new skill is covered at once. */
export function collectPromptVersions(plan2codeRoot: string): PromptVersions {
const srcDir = path.join(plan2codeRoot, 'src');
return {
plan: sha256File(path.join(srcDir, 'plan2code-1-plan.md')),
revise_plan: sha256File(path.join(srcDir, 'plan2code-1b-revise-plan.md')),
document: sha256File(path.join(srcDir, 'plan2code-2-document.md')),
implement: sha256File(path.join(srcDir, 'plan2code-3-implement.md')),
finalize: sha256File(path.join(srcDir, 'plan2code-4-finalize.md')),
init: sha256File(path.join(srcDir, 'plan2code-init.md')),
init_update: sha256File(path.join(srcDir, 'plan2code-init-update.md')),
quick_task: sha256File(path.join(srcDir, 'plan2code-quick-task.md')),
};
let files: string[] = [];
try {
files = fs.readdirSync(srcDir)
.filter(f => /^plan2code-.+\.md$/.test(f) && fs.statSync(path.join(srcDir, f)).isFile())
.sort();
} catch {
return {};
}
const versions: PromptVersions = {};
for (const f of files) versions[promptKey(f)] = sha256File(path.join(srcDir, f));
return versions;
}
// ── Step 1: Plan ──────────────────────────────────────────────────────────────
@@ -184,24 +205,34 @@ export function collectStep1(specDir: string): Step1PlanMetrics {
};
// ── Confidence ──
// The patterns are the shared ones in metrics-pattern.json, by name, like the breakdown below.
const confidenceMatch = (name: string) => {
const p = metricsPattern.confidence.find(c => c.name === name);
return p ? latestDraft.match(new RegExp(p.source, p.flags)) : null;
};
let finalConfidence = parseMetricField('confidence');
if (finalConfidence === null) {
// Fallback: prose scraping — handle bold markdown like **Confidence:** 95%
const confMatch = latestDraft.match(/(?:Overall|Final|Total)?\s*\**[Cc]onfidence\**[:\s*]+(\d{1,3})%/);
const confMatch = confidenceMatch('overall confidence with %');
if (confMatch) {
finalConfidence = parseInt(confMatch[1], 10);
} else {
const tableMatch = latestDraft.match(/[|]\s*[Cc]onfidence\s*[|]\s*(\d{1,3})/);
const tableMatch = confidenceMatch('confidence table cell');
if (tableMatch) finalConfidence = parseInt(tableMatch[1], 10);
}
}
// Confidence breakdown (requirements, feasibility, integration, risk)
let confidenceBreakdown: Step1PlanMetrics['confidence_breakdown'] = null;
const reqMatch = latestDraft.match(/[Rr]equirements?[:\s|]+(\d{1,2})/);
const feasMatch = latestDraft.match(/[Ff]easibility[:\s|]+(\d{1,2})/);
const intMatch = latestDraft.match(/[Ii]ntegration[:\s|]+(\d{1,2})/);
const riskMatch = latestDraft.match(/[Rr]isk[:\s|]+(\d{1,2})/);
// The patterns are the shared ones in metrics-pattern.json (the console's bait scanner reads the same).
const breakdownMatch = (key: string): RegExpMatchArray | null => {
const p = metricsPattern.breakdown.find(b => b.key === key);
return p ? latestDraft.match(new RegExp(p.source, p.flags)) : null;
};
const reqMatch = breakdownMatch('requirements');
const feasMatch = breakdownMatch('feasibility');
const intMatch = breakdownMatch('integration');
const riskMatch = breakdownMatch('risk');
if (reqMatch || feasMatch || intMatch || riskMatch) {
confidenceBreakdown = {
requirements: reqMatch ? parseInt(reqMatch[1], 10) : null,
@@ -596,7 +627,7 @@ export async function collectRun(opts: CollectorOptions): Promise<RunMetrics> {
const runId = generateRunId();
// Get started_at / completed_at from spec directory mtime / iteration.log
// Get started_at / completed_at from the spec directory's birth and modified times
let startedAt: string | null = null;
let completedAt: string | null = null;
try {
@@ -0,0 +1,42 @@
{
"_note": "The shapes plan2code-metrics scrapes from a PLAN-DRAFT: one source for the console's bait scanner (src/web-console/patches.mjs), the skill scripts' SCRAPE (src/skill-scripts/common.mjs) and the collector's copy (plan2code-metrics/src/metrics-pattern.json). scripts/test-web-console-hardening.mjs fails when the copies drift. A label then a number in a table row counts only when the number ends the row.",
"metricsComment": "<!--\\s*METRICS_JSON\\s+(\\{[\\s\\S]*?\\})\\s*-->",
"confidence": [
{
"name": "overall confidence with %",
"source": "(?:Overall|Final|Total)?\\s*\\**[Cc]onfidence\\**[:\\s*]+(\\d{1,3})%",
"flags": ""
},
{
"name": "confidence table cell",
"source": "[|]\\s*[Cc]onfidence\\s*[|]\\s*(\\d{1,3})",
"flags": ""
}
],
"breakdown": [
{
"key": "requirements",
"name": "Requirements",
"source": "[Rr]equirements?(?:[:\\s]+|[ \\t:]*\\|[ \\t:|]*(?=\\d{1,2}(?:\\/\\d+)?[ \\t]*\\|?[ \\t]*$))(\\d{1,2})",
"flags": "m"
},
{
"key": "feasibility",
"name": "Feasibility",
"source": "[Ff]easibility(?:[:\\s]+|[ \\t:]*\\|[ \\t:|]*(?=\\d{1,2}(?:\\/\\d+)?[ \\t]*\\|?[ \\t]*$))(\\d{1,2})",
"flags": "m"
},
{
"key": "integration",
"name": "Integration",
"source": "[Ii]ntegration(?:[:\\s]+|[ \\t:]*\\|[ \\t:|]*(?=\\d{1,2}(?:\\/\\d+)?[ \\t]*\\|?[ \\t]*$))(\\d{1,2})",
"flags": "m"
},
{
"key": "risk",
"name": "Risk",
"source": "[Rr]isk(?:[:\\s]+|[ \\t:]*\\|[ \\t:|]*(?=\\d{1,2}(?:\\/\\d+)?[ \\t]*\\|?[ \\t]*$))(\\d{1,2})",
"flags": "m"
}
]
}
+7 -10
View File
@@ -1,15 +1,12 @@
// All TypeScript interfaces for plan2code-metrics
export interface PromptVersions {
plan: string; // sha256:... plan2code-1-plan.md
revise_plan: string; // plan2code-1b-revise-plan.md
document: string; // plan2code-2-document.md
implement: string; // plan2code-3-implement.md
finalize: string; // plan2code-4-finalize.md
init: string; // plan2code-init.md
init_update: string; // plan2code-init-update.md
quick_task: string; // plan2code-quick-task.md
}
/**
* SHA-256 fingerprint of every src/plan2code-*.md prompt, keyed by name
* (promptKey in collector.ts). The eight prompts recorded before the
* collector read the folder keep their old keys: plan, revise_plan, document,
* implement, finalize, init, init_update, quick_task.
*/
export type PromptVersions = Record<string, string>;
export interface Step1PlanMetrics {
present: boolean;