mirror of
https://github.com/jparkerweb/plan2code.git
synced 2026-11-16 20:45:33 -08:00
Harden the shared console install, launcher, Loop staging, spec state and prompts
AI Assisted
This commit is contained in:
1 parent
120c259d9c
commit
3ee96834bf
130 files changed
+5988
-1434
No files matched your search
@@ -56,18 +56,25 @@ export function cohortKeyForRun(run: RunMetrics): string {
|
||||
}
|
||||
|
||||
/**
|
||||
* Backfill missing PromptVersions fields for old run files (pre-v1.1).
|
||||
* Backfill missing PromptVersions fields for old run files (pre-v1.1): the
|
||||
* eight prompts every run once recorded are padded with the missing sentinel
|
||||
* (buildCohortKey ignores it), and any other prompt a run recorded is kept.
|
||||
*
|
||||
* One-time regrouping: a run recorded before the collector fingerprinted every
|
||||
* prompt has fewer entries than a new one, so its cohort key differs and it
|
||||
* groups as the previous prompt generation.
|
||||
*/
|
||||
export function backfillPromptVersions(pv: PromptVersions): PromptVersions {
|
||||
return {
|
||||
plan: pv.plan ?? 'sha256:missing',
|
||||
revise_plan: pv.revise_plan ?? 'sha256:missing',
|
||||
document: pv.document ?? 'sha256:missing',
|
||||
implement: pv.implement ?? 'sha256:missing',
|
||||
finalize: pv.finalize ?? 'sha256:missing',
|
||||
init: pv.init ?? 'sha256:missing',
|
||||
init_update: pv.init_update ?? 'sha256:missing',
|
||||
quick_task: pv.quick_task ?? 'sha256:missing',
|
||||
plan: 'sha256:missing',
|
||||
revise_plan: 'sha256:missing',
|
||||
document: 'sha256:missing',
|
||||
implement: 'sha256:missing',
|
||||
finalize: 'sha256:missing',
|
||||
init: 'sha256:missing',
|
||||
init_update: 'sha256:missing',
|
||||
quick_task: 'sha256:missing',
|
||||
...(pv ?? {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { collectPromptVersions, collectStep1, extractMetricsJson, promptKey } from './collector.js';
|
||||
import { backfillPromptVersions, buildCohortKey } from './aggregator.js';
|
||||
|
||||
let root: string;
|
||||
|
||||
beforeEach(() => {
|
||||
root = fs.mkdtempSync(path.join(os.tmpdir(), 'p2c-collector-'));
|
||||
fs.mkdirSync(path.join(root, 'src'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function prompt(name: string, body: string) {
|
||||
fs.writeFileSync(path.join(root, 'src', name), body);
|
||||
}
|
||||
|
||||
describe('collectPromptVersions', () => {
|
||||
it('fingerprints every src/plan2code-*.md prompt in the folder', () => {
|
||||
prompt('plan2code-1-plan.md', 'plan');
|
||||
prompt('plan2code-3-implement-review.md', 'implement review');
|
||||
prompt('plan2code-git.md', 'git');
|
||||
const versions = collectPromptVersions(root);
|
||||
expect(Object.keys(versions).sort()).toEqual(['3_implement_review', 'git', 'plan']);
|
||||
for (const v of Object.values(versions)) expect(v).toMatch(/^sha256:[0-9a-f]{64}$/);
|
||||
});
|
||||
|
||||
it('covers a newly added prompt without a code change', () => {
|
||||
prompt('plan2code-1-plan.md', 'plan');
|
||||
const before = collectPromptVersions(root);
|
||||
prompt('plan2code-brand-new.md', 'a new skill');
|
||||
const after = collectPromptVersions(root);
|
||||
expect(before).not.toHaveProperty('brand_new');
|
||||
expect(after).toHaveProperty('brand_new');
|
||||
expect(buildCohortKey(after)).not.toBe(buildCohortKey(before));
|
||||
});
|
||||
|
||||
it('ignores files and folders that are not top-level plan2code-*.md prompts', () => {
|
||||
prompt('plan2code-1-plan.md', 'plan');
|
||||
prompt('README.md', 'not a prompt');
|
||||
prompt('plan2code-notes.txt', 'not markdown');
|
||||
fs.mkdirSync(path.join(root, 'src', 'plan2code-0-pathfinder-references.md'));
|
||||
expect(Object.keys(collectPromptVersions(root))).toEqual(['plan']);
|
||||
});
|
||||
|
||||
it('keeps the old keys for the eight prompts recorded before, so their hashes stay comparable', () => {
|
||||
expect(promptKey('plan2code-1-plan.md')).toBe('plan');
|
||||
expect(promptKey('plan2code-1b-revise-plan.md')).toBe('revise_plan');
|
||||
expect(promptKey('plan2code-2-document.md')).toBe('document');
|
||||
expect(promptKey('plan2code-3-implement.md')).toBe('implement');
|
||||
expect(promptKey('plan2code-4-finalize.md')).toBe('finalize');
|
||||
expect(promptKey('plan2code-init.md')).toBe('init');
|
||||
expect(promptKey('plan2code-init-update.md')).toBe('init_update');
|
||||
expect(promptKey('plan2code-quick-task.md')).toBe('quick_task');
|
||||
});
|
||||
|
||||
it('returns nothing when there is no src folder', () => {
|
||||
expect(collectPromptVersions(path.join(root, 'nowhere'))).toEqual({});
|
||||
});
|
||||
|
||||
it('groups a run recorded before this change as the previous prompt generation', () => {
|
||||
prompt('plan2code-1-plan.md', 'plan');
|
||||
prompt('plan2code-git.md', 'git');
|
||||
const now = collectPromptVersions(root);
|
||||
const old = backfillPromptVersions({ plan: now.plan });
|
||||
expect(buildCohortKey(old)).not.toBe(buildCohortKey(now));
|
||||
});
|
||||
});
|
||||
|
||||
describe('the shared scrape pattern', () => {
|
||||
function draft(body: string) {
|
||||
const specDir = path.join(root, 'spec');
|
||||
fs.mkdirSync(specDir);
|
||||
fs.writeFileSync(path.join(specDir, 'PLAN-DRAFT-1.md'), body);
|
||||
return collectStep1(specDir);
|
||||
}
|
||||
|
||||
it('reads the METRICS_JSON comment', () => {
|
||||
expect(extractMetricsJson('x <!-- METRICS_JSON {"confidence": 95} --> y')).toEqual({ confidence: 95 });
|
||||
});
|
||||
|
||||
it('reads a breakdown written as label, colon, score', () => {
|
||||
const m = draft('Requirements: 24\nFeasibility: 23\nIntegration: 22\nRisk: 21\n');
|
||||
expect(m.confidence_breakdown).toEqual({ requirements: 24, feasibility: 23, integration: 22, risk: 21 });
|
||||
});
|
||||
|
||||
it('reads a breakdown table whose score ends the row', () => {
|
||||
const m = draft('| Requirements | 24 |\n| Risk | 21/25 |\n');
|
||||
expect(m.confidence_breakdown).toEqual({ requirements: 24, feasibility: null, integration: null, risk: 21 });
|
||||
});
|
||||
|
||||
it('leaves an ordinary risk-register row alone', () => {
|
||||
const m = draft('| Risk | 3 | High | Mitigation |\n');
|
||||
expect(m.confidence_breakdown).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -16,6 +16,7 @@ import type {
|
||||
Step4FinalizeMetrics,
|
||||
UserFeedback,
|
||||
} from './types.js';
|
||||
import metricsPattern from './metrics-pattern.json' with { type: 'json' };
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -44,7 +45,7 @@ function readFileSafe(filePath: string): string | null {
|
||||
* Returns parsed object or null if not found / invalid.
|
||||
*/
|
||||
export function extractMetricsJson(content: string, stepFilter?: string): Record<string, unknown> | null {
|
||||
const re = /<!--\s*METRICS_JSON\s+(\{[\s\S]*?\})\s*-->/g;
|
||||
const re = new RegExp(metricsPattern.metricsComment, 'g');
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = re.exec(content)) !== null) {
|
||||
try {
|
||||
@@ -96,18 +97,38 @@ function generateRunId(): string {
|
||||
|
||||
// ── Prompt version hashing ────────────────────────────────────────────────────
|
||||
|
||||
// The prompts recorded before the collector read the folder keep their keys.
|
||||
const LEGACY_PROMPT_KEYS: Record<string, string> = {
|
||||
'plan2code-1-plan.md': 'plan',
|
||||
'plan2code-1b-revise-plan.md': 'revise_plan',
|
||||
'plan2code-2-document.md': 'document',
|
||||
'plan2code-3-implement.md': 'implement',
|
||||
'plan2code-4-finalize.md': 'finalize',
|
||||
'plan2code-init.md': 'init',
|
||||
'plan2code-init-update.md': 'init_update',
|
||||
'plan2code-quick-task.md': 'quick_task',
|
||||
};
|
||||
|
||||
/** The PromptVersions key for a prompt file: its legacy key, else its name without "plan2code-" and ".md", dashes as underscores. */
|
||||
export function promptKey(fileName: string): string {
|
||||
return LEGACY_PROMPT_KEYS[fileName]
|
||||
?? fileName.replace(/^plan2code-/, '').replace(/\.md$/, '').replace(/-/g, '_');
|
||||
}
|
||||
|
||||
/** Fingerprint every src/plan2code-*.md prompt, read from the folder so a new skill is covered at once. */
|
||||
export function collectPromptVersions(plan2codeRoot: string): PromptVersions {
|
||||
const srcDir = path.join(plan2codeRoot, 'src');
|
||||
return {
|
||||
plan: sha256File(path.join(srcDir, 'plan2code-1-plan.md')),
|
||||
revise_plan: sha256File(path.join(srcDir, 'plan2code-1b-revise-plan.md')),
|
||||
document: sha256File(path.join(srcDir, 'plan2code-2-document.md')),
|
||||
implement: sha256File(path.join(srcDir, 'plan2code-3-implement.md')),
|
||||
finalize: sha256File(path.join(srcDir, 'plan2code-4-finalize.md')),
|
||||
init: sha256File(path.join(srcDir, 'plan2code-init.md')),
|
||||
init_update: sha256File(path.join(srcDir, 'plan2code-init-update.md')),
|
||||
quick_task: sha256File(path.join(srcDir, 'plan2code-quick-task.md')),
|
||||
};
|
||||
let files: string[] = [];
|
||||
try {
|
||||
files = fs.readdirSync(srcDir)
|
||||
.filter(f => /^plan2code-.+\.md$/.test(f) && fs.statSync(path.join(srcDir, f)).isFile())
|
||||
.sort();
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
const versions: PromptVersions = {};
|
||||
for (const f of files) versions[promptKey(f)] = sha256File(path.join(srcDir, f));
|
||||
return versions;
|
||||
}
|
||||
|
||||
// ── Step 1: Plan ──────────────────────────────────────────────────────────────
|
||||
@@ -184,24 +205,34 @@ export function collectStep1(specDir: string): Step1PlanMetrics {
|
||||
};
|
||||
|
||||
// ── Confidence ──
|
||||
// The patterns are the shared ones in metrics-pattern.json, by name, like the breakdown below.
|
||||
const confidenceMatch = (name: string) => {
|
||||
const p = metricsPattern.confidence.find(c => c.name === name);
|
||||
return p ? latestDraft.match(new RegExp(p.source, p.flags)) : null;
|
||||
};
|
||||
let finalConfidence = parseMetricField('confidence');
|
||||
if (finalConfidence === null) {
|
||||
// Fallback: prose scraping — handle bold markdown like **Confidence:** 95%
|
||||
const confMatch = latestDraft.match(/(?:Overall|Final|Total)?\s*\**[Cc]onfidence\**[:\s*]+(\d{1,3})%/);
|
||||
const confMatch = confidenceMatch('overall confidence with %');
|
||||
if (confMatch) {
|
||||
finalConfidence = parseInt(confMatch[1], 10);
|
||||
} else {
|
||||
const tableMatch = latestDraft.match(/[|]\s*[Cc]onfidence\s*[|]\s*(\d{1,3})/);
|
||||
const tableMatch = confidenceMatch('confidence table cell');
|
||||
if (tableMatch) finalConfidence = parseInt(tableMatch[1], 10);
|
||||
}
|
||||
}
|
||||
|
||||
// Confidence breakdown (requirements, feasibility, integration, risk)
|
||||
let confidenceBreakdown: Step1PlanMetrics['confidence_breakdown'] = null;
|
||||
const reqMatch = latestDraft.match(/[Rr]equirements?[:\s|]+(\d{1,2})/);
|
||||
const feasMatch = latestDraft.match(/[Ff]easibility[:\s|]+(\d{1,2})/);
|
||||
const intMatch = latestDraft.match(/[Ii]ntegration[:\s|]+(\d{1,2})/);
|
||||
const riskMatch = latestDraft.match(/[Rr]isk[:\s|]+(\d{1,2})/);
|
||||
// The patterns are the shared ones in metrics-pattern.json (the console's bait scanner reads the same).
|
||||
const breakdownMatch = (key: string): RegExpMatchArray | null => {
|
||||
const p = metricsPattern.breakdown.find(b => b.key === key);
|
||||
return p ? latestDraft.match(new RegExp(p.source, p.flags)) : null;
|
||||
};
|
||||
const reqMatch = breakdownMatch('requirements');
|
||||
const feasMatch = breakdownMatch('feasibility');
|
||||
const intMatch = breakdownMatch('integration');
|
||||
const riskMatch = breakdownMatch('risk');
|
||||
if (reqMatch || feasMatch || intMatch || riskMatch) {
|
||||
confidenceBreakdown = {
|
||||
requirements: reqMatch ? parseInt(reqMatch[1], 10) : null,
|
||||
@@ -596,7 +627,7 @@ export async function collectRun(opts: CollectorOptions): Promise<RunMetrics> {
|
||||
|
||||
const runId = generateRunId();
|
||||
|
||||
// Get started_at / completed_at from spec directory mtime / iteration.log
|
||||
// Get started_at / completed_at from the spec directory's birth and modified times
|
||||
let startedAt: string | null = null;
|
||||
let completedAt: string | null = null;
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"_note": "The shapes plan2code-metrics scrapes from a PLAN-DRAFT: one source for the console's bait scanner (src/web-console/patches.mjs), the skill scripts' SCRAPE (src/skill-scripts/common.mjs) and the collector's copy (plan2code-metrics/src/metrics-pattern.json). scripts/test-web-console-hardening.mjs fails when the copies drift. A label then a number in a table row counts only when the number ends the row.",
|
||||
"metricsComment": "<!--\\s*METRICS_JSON\\s+(\\{[\\s\\S]*?\\})\\s*-->",
|
||||
"confidence": [
|
||||
{
|
||||
"name": "overall confidence with %",
|
||||
"source": "(?:Overall|Final|Total)?\\s*\\**[Cc]onfidence\\**[:\\s*]+(\\d{1,3})%",
|
||||
"flags": ""
|
||||
},
|
||||
{
|
||||
"name": "confidence table cell",
|
||||
"source": "[|]\\s*[Cc]onfidence\\s*[|]\\s*(\\d{1,3})",
|
||||
"flags": ""
|
||||
}
|
||||
],
|
||||
"breakdown": [
|
||||
{
|
||||
"key": "requirements",
|
||||
"name": "Requirements",
|
||||
"source": "[Rr]equirements?(?:[:\\s]+|[ \\t:]*\\|[ \\t:|]*(?=\\d{1,2}(?:\\/\\d+)?[ \\t]*\\|?[ \\t]*$))(\\d{1,2})",
|
||||
"flags": "m"
|
||||
},
|
||||
{
|
||||
"key": "feasibility",
|
||||
"name": "Feasibility",
|
||||
"source": "[Ff]easibility(?:[:\\s]+|[ \\t:]*\\|[ \\t:|]*(?=\\d{1,2}(?:\\/\\d+)?[ \\t]*\\|?[ \\t]*$))(\\d{1,2})",
|
||||
"flags": "m"
|
||||
},
|
||||
{
|
||||
"key": "integration",
|
||||
"name": "Integration",
|
||||
"source": "[Ii]ntegration(?:[:\\s]+|[ \\t:]*\\|[ \\t:|]*(?=\\d{1,2}(?:\\/\\d+)?[ \\t]*\\|?[ \\t]*$))(\\d{1,2})",
|
||||
"flags": "m"
|
||||
},
|
||||
{
|
||||
"key": "risk",
|
||||
"name": "Risk",
|
||||
"source": "[Rr]isk(?:[:\\s]+|[ \\t:]*\\|[ \\t:|]*(?=\\d{1,2}(?:\\/\\d+)?[ \\t]*\\|?[ \\t]*$))(\\d{1,2})",
|
||||
"flags": "m"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,15 +1,12 @@
|
||||
// All TypeScript interfaces for plan2code-metrics
|
||||
|
||||
export interface PromptVersions {
|
||||
plan: string; // sha256:... plan2code-1-plan.md
|
||||
revise_plan: string; // plan2code-1b-revise-plan.md
|
||||
document: string; // plan2code-2-document.md
|
||||
implement: string; // plan2code-3-implement.md
|
||||
finalize: string; // plan2code-4-finalize.md
|
||||
init: string; // plan2code-init.md
|
||||
init_update: string; // plan2code-init-update.md
|
||||
quick_task: string; // plan2code-quick-task.md
|
||||
}
|
||||
/**
|
||||
* SHA-256 fingerprint of every src/plan2code-*.md prompt, keyed by name
|
||||
* (promptKey in collector.ts). The eight prompts recorded before the
|
||||
* collector read the folder keep their old keys: plan, revise_plan, document,
|
||||
* implement, finalize, init, init_update, quick_task.
|
||||
*/
|
||||
export type PromptVersions = Record<string, string>;
|
||||
|
||||
export interface Step1PlanMetrics {
|
||||
present: boolean;
|
||||
|
||||
Reference in new issue
Block a user