From 348c8ed7b24bff0165c67d96791914c60c9df947 Mon Sep 17 00:00:00 2001 From: Justin Parker Date: Sun, 22 Feb 2026 21:38:58 -0800 Subject: [PATCH] feat: add user feedback collection to plan2code-metrics - Add UserFeedback type (rating 1-10, reason, went well, went poorly) - Collector parses ## User Feedback table from overview.md - Aggregator computes avg_user_rating and feedback_count per cohort - CLI offers interactive feedback collection with pipe-safe escaping - Finalize prompt collects optional feedback as Step 5 (before archival) - Analysis/improvement prompts reference avg_user_rating metric target --- plan2code-metrics/src/aggregator.ts | 8 +++ plan2code-metrics/src/cli.ts | 71 ++++++++++++++++++++++++ plan2code-metrics/src/collector.ts | 46 +++++++++++++++ plan2code-metrics/src/index.ts | 1 + plan2code-metrics/src/prompts/analyze.md | 1 + plan2code-metrics/src/prompts/improve.md | 1 + plan2code-metrics/src/types.ts | 13 +++++ src/plan2code-4--finalize.md | 43 ++++++++++++-- 8 files changed, 178 insertions(+), 6 deletions(-) diff --git a/plan2code-metrics/src/aggregator.ts b/plan2code-metrics/src/aggregator.ts index 96dfa53..6b6f9b5 100644 --- a/plan2code-metrics/src/aggregator.ts +++ b/plan2code-metrics/src/aggregator.ts @@ -113,6 +113,11 @@ function buildCohort(runs: RunMetrics[], cohortKey: string): CohortMetrics { const avgDocUpdates = avg(runs.map(r => r.step4_finalize.documentation_updates_needed)); const archivalRate = rate(runs.map(r => r.step4_finalize.archival_succeeded)); + // User feedback + const feedbackRuns = runs.filter(r => r.user_feedback != null); + const avgUserRating = avg(feedbackRuns.map(r => r.user_feedback!.overall_rating)); + const feedbackCount = feedbackRuns.length; + return { cohort_key: cohortKey, prompt_versions: runs[0].prompt_versions, @@ -146,6 +151,9 @@ function buildCohort(runs: RunMetrics[], cohortKey: string): CohortMetrics { avg_verification_failures_found: avgVerifFailures, avg_documentation_updates_needed: avgDocUpdates, archival_success_rate: archivalRate, + + avg_user_rating: avgUserRating, + feedback_count: feedbackCount, }; } diff --git a/plan2code-metrics/src/cli.ts b/plan2code-metrics/src/cli.ts index d34e0c3..64f652a 100644 --- a/plan2code-metrics/src/cli.ts +++ b/plan2code-metrics/src/cli.ts @@ -181,6 +181,70 @@ async function flowCollect(): Promise { console.log(` Step 2 (Document): ${metrics.step2_document.present ? chalk.green('✓') : chalk.gray('—')}`); console.log(` Step 3 (Implement): ${metrics.step3_implement.present ? (metrics.step3_implement.used_loop_mode ? chalk.green('✓ (loop)') : chalk.yellow('✓ (manual)')) : chalk.gray('—')}`); console.log(` Step 4 (Finalize): ${metrics.step4_finalize.present ? chalk.green('✓') : chalk.gray('—')}`); + console.log(` User Feedback: ${metrics.user_feedback ? chalk.green(`✓ (${metrics.user_feedback.overall_rating}/10)`) : chalk.gray('—')}`); + + // Offer to collect feedback interactively if not present + if (!metrics.user_feedback) { + console.log(); + const wantFeedback = await confirm({ + message: 'No user feedback found in overview.md. Would you like to provide feedback now?', + default: false, + }); + + if (wantFeedback) { + const rating = await input({ + message: 'Overall rating (1-10):', + validate: (v) => { + const n = parseInt(v, 10); + return (n >= 1 && n <= 10) ? true : 'Must be a number between 1 and 10'; + }, + }); + const reason = await input({ message: 'Rating reason:' }); + const wentWell = await input({ message: 'What went well?' }); + const wentPoorly = await input({ message: 'What went poorly?' }); + + // Write the feedback table into overview.md + const overviewPath = path.join(specDir, 'overview.md'); + let overviewContent = ''; + try { + overviewContent = fs.readFileSync(overviewPath, 'utf8'); + } catch { /* empty */ } + + // Escape pipe characters in user text to prevent table parsing issues + const esc = (s: string) => s.replace(/\|/g, '\\|'); + + const feedbackTable = [ + '', + '## User Feedback', + '| Field | Value |', + '|-------|-------|', + `| Rating | ${rating} |`, + `| Reason | ${esc(reason)} |`, + `| Went Well | ${esc(wentWell)} |`, + `| Went Poorly | ${esc(wentPoorly)} |`, + '', + ].join('\n'); + + fs.writeFileSync(overviewPath, overviewContent + feedbackTable, 'utf8'); + console.log(chalk.green('✓ Feedback written to overview.md')); + + // Delete the original run (without feedback) before re-collecting + const originalRunPath = path.join(runsDir, `${metrics.run_id}.json`); + try { fs.unlinkSync(originalRunPath); } catch { /* ignore */ } + + // Re-collect and re-aggregate + const reSpinner = ora('Re-collecting metrics with feedback...').start(); + const updatedMetrics = await collectRun({ + specDir, + projectName, + plan2codeRoot, + plan2codeVersion, + outputDir: runsDir, + }); + aggregate(runsDir, aggregatedPath); + reSpinner.succeed(`Updated: ${updatedMetrics.run_id} (rating: ${updatedMetrics.user_feedback?.overall_rating}/10)`); + } + } } catch (err) { spinner.fail(`Collection failed: ${err instanceof Error ? err.message : String(err)}`); } @@ -292,6 +356,13 @@ async function flowViewStatus(): Promise { console.log(` archival_success_rate: ${metricStatus('archival_success_rate', cohort.archival_success_rate)}`); } + // User feedback + if (cohort.feedback_count > 0) { + console.log(chalk.bold(' User Feedback:')); + console.log(` avg_user_rating: ${metricStatus('avg_user_rating', cohort.avg_user_rating)}`); + console.log(` feedback_count: ${chalk.white(String(cohort.feedback_count))}`); + } + // Compare with previous generation if (i > 0) { const prev = aggregated.cohorts[i - 1]; diff --git a/plan2code-metrics/src/collector.ts b/plan2code-metrics/src/collector.ts index bf76702..4b01f58 100644 --- a/plan2code-metrics/src/collector.ts +++ b/plan2code-metrics/src/collector.ts @@ -14,6 +14,7 @@ import type { Step2DocumentMetrics, Step3ImplementMetrics, Step4FinalizeMetrics, + UserFeedback, } from './types.js'; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -399,6 +400,50 @@ export function collectStep4(specDir: string): Step4FinalizeMetrics { }; } +// ── User Feedback ───────────────────────────────────────────────────────── + +export function collectUserFeedback(specDir: string): UserFeedback | null { + // Try both active and archived locations for overview.md + const overviewPath = path.join(specDir, 'overview.md'); + const specDirName = path.basename(specDir); + const parentDir = path.dirname(specDir); + const completedPath = path.join(parentDir, '..', 'specs--completed', specDirName, 'overview.md'); + + const overview = readFileSafe(overviewPath) ?? readFileSafe(completedPath); + if (!overview) return null; + + // Look for ## User Feedback section with a markdown table + const feedbackMatch = overview.match( + /## User Feedback\s*\n\s*\|[^\n]*\|\s*\n\s*\|[-| ]+\|\s*\n([\s\S]*?)(?=\n##\s|\n*$)/ + ); + if (!feedbackMatch) return null; + + const tableBody = feedbackMatch[1]; + + // Parse table rows: | Field | Value | + // Use regex to split only on unescaped pipes, preserving whitespace in values + const rows = tableBody.split('\n').filter(l => l.trim().startsWith('|')); + const fields: Record = {}; + for (const row of rows) { + // Split on unescaped pipes (not preceded by backslash) + const cells = row.split(/(? c.trim()).filter(c => c.length > 0); + if (cells.length >= 2) { + const value = cells.slice(1).join('|').replace(/\\\|/g, '|'); + fields[cells[0].toLowerCase()] = value; + } + } + + const rating = parseInt(fields['rating'] ?? '', 10); + if (isNaN(rating) || rating < 1 || rating > 10) return null; + + return { + overall_rating: rating, + rating_reason: fields['reason'] ?? '', + what_went_well: fields['went well'] ?? '', + what_went_poorly: fields['went poorly'] ?? '', + }; +} + // ── Main collector ──────────────────────────────────────────────────────────── export interface CollectorOptions { @@ -463,6 +508,7 @@ export async function collectRun(opts: CollectorOptions): Promise { step2_document: collectStep2(specDir), step3_implement: collectStep3(specDir), step4_finalize: collectStep4(specDir), + user_feedback: collectUserFeedback(specDir), }; // Write to output dir diff --git a/plan2code-metrics/src/index.ts b/plan2code-metrics/src/index.ts index 3ae9726..be0e1bf 100644 --- a/plan2code-metrics/src/index.ts +++ b/plan2code-metrics/src/index.ts @@ -10,6 +10,7 @@ export { runCLI } from './cli.js'; export { METRIC_TARGETS } from './types.js'; export type { RunMetrics, + UserFeedback, PromptVersions, PromptEdit, PromptProposal, diff --git a/plan2code-metrics/src/prompts/analyze.md b/plan2code-metrics/src/prompts/analyze.md index eb5f886..70ba08a 100644 --- a/plan2code-metrics/src/prompts/analyze.md +++ b/plan2code-metrics/src/prompts/analyze.md @@ -38,6 +38,7 @@ The following are the current contents of the plan2code workflow prompt files be | avg_completion_marker_success_rate (Step 3) | ≥ 0.95 | higher is better | | avg_verification_failures_found (Step 4) | ≤ 1.0 | lower is better | | archival_success_rate (Step 4) | ≥ 0.99 | higher is better | +| avg_user_rating (Feedback) | ≥ 7.0 | higher is better (1-10 scale, null if no feedback) | --- diff --git a/plan2code-metrics/src/prompts/improve.md b/plan2code-metrics/src/prompts/improve.md index 08f9901..6576dbd 100644 --- a/plan2code-metrics/src/prompts/improve.md +++ b/plan2code-metrics/src/prompts/improve.md @@ -45,6 +45,7 @@ The following diagnosis was produced by the analysis step: - For high `avg_blocker_count`: Add blocker-recovery guidance or prerequisite check instructions. - For low `avg_confidence`: Strengthen the confidence calculation instructions with clearer rubrics. - For low `avg_parallel_groups`: Add explicit guidance for identifying parallel tasks in Step 2. +- For low `avg_user_rating`: Review user feedback themes (what_went_well, what_went_poorly) for systemic issues. - Keep each `new_text` as short as possible while still addressing the root cause. --- diff --git a/plan2code-metrics/src/types.ts b/plan2code-metrics/src/types.ts index afa8403..1dc5fe4 100644 --- a/plan2code-metrics/src/types.ts +++ b/plan2code-metrics/src/types.ts @@ -62,6 +62,13 @@ export interface Step4FinalizeMetrics { archival_succeeded: boolean | null; } +export interface UserFeedback { + overall_rating: number; // 1-10 + rating_reason: string; + what_went_well: string; + what_went_poorly: string; +} + export interface RunMetrics { schema_version: '1.0'; run_id: string; @@ -76,6 +83,7 @@ export interface RunMetrics { step2_document: Step2DocumentMetrics; step3_implement: Step3ImplementMetrics; step4_finalize: Step4FinalizeMetrics; + user_feedback: UserFeedback | null; } export interface PromptEdit { @@ -137,6 +145,10 @@ export interface CohortMetrics { avg_verification_failures_found: number | null; avg_documentation_updates_needed: number | null; archival_success_rate: number | null; + + // User feedback + avg_user_rating: number | null; + feedback_count: number; } export interface AggregatedMetrics { @@ -159,4 +171,5 @@ export const METRIC_TARGETS = { avg_completion_marker_success_rate: { target: 0.95, direction: 'gte' as const }, avg_verification_failures_found: { target: 1.0, direction: 'lte' as const }, archival_success_rate: { target: 0.99, direction: 'gte' as const }, + avg_user_rating: { target: 7.0, direction: 'gte' as const }, } as const; diff --git a/src/plan2code-4--finalize.md b/src/plan2code-4--finalize.md index 258fed9..ecdfc7a 100644 --- a/src/plan2code-4--finalize.md +++ b/src/plan2code-4--finalize.md @@ -197,9 +197,39 @@ Do NOT make documentation changes without user approval. --- -### STEP 5: Spec Cleanup +### STEP 5: User Feedback (Optional) -`🧹 [FINALIZATION STEP 5: Spec Cleanup]` +`🧹 [FINALIZATION STEP 5: User Feedback]` + +**Objective:** Collect optional user feedback before archival. + +Ask the user: "Would you like to provide feedback on this workflow run? (optional)" + +If yes, collect: +1. **Rating** (1-10): "How would you rate this workflow run overall?" +2. **Reason**: "Brief reason for your rating?" +3. **Went Well**: "What went well?" +4. **Went Poorly**: "What went poorly or could improve?" + +Append to `overview.md` (in the active spec directory): + +```markdown +## User Feedback +| Field | Value | +|-------|-------| +| Rating | [1-10] | +| Reason | [response] | +| Went Well | [response] | +| Went Poorly | [response] | +``` + +If the user declines, skip and proceed to Step 6 (Spec Cleanup). + +--- + +### STEP 6: Spec Cleanup + +`🧹 [FINALIZATION STEP 6: Spec Cleanup]` **Objective:** Archive completed specifications. @@ -226,9 +256,9 @@ specs--completed/ --- -### STEP 6: Final Confirmation +### STEP 7: Final Confirmation -`🧹 [FINALIZATION STEP 6: Final Confirmation]` +`🧹 [FINALIZATION STEP 7: Final Confirmation]` **Objective:** Confirm all finalization steps complete. @@ -246,8 +276,9 @@ specs--completed/ - [x] Step 2: Implementation Verification - [x] Step 3: Implementation Summary - [x] Step 4: Documentation Review -- [x] Step 5: Spec Cleanup -- [x] Step 6: Final Confirmation +- [x] Step 5: User Feedback (Optional) +- [x] Step 6: Spec Cleanup +- [x] Step 7: Final Confirmation ### Files Created/Modified During Finalization - `specs//overview.md` - Added completion summary