diff --git a/plan2code-metrics/README.md b/plan2code-metrics/README.md index b69c577..de5d197 100644 --- a/plan2code-metrics/README.md +++ b/plan2code-metrics/README.md @@ -116,9 +116,10 @@ plan2code-metrics | **Collect metrics** | Parse a completed project spec and extract step-by-step metrics into a run JSON | | **Import run data** | Copy a run JSON from another project into the local metrics store | | **View metrics status** | Show aggregated metrics with health indicators and generation deltas | -| **Run analysis** | AI-powered diagnosis of weak steps (requires Claude Code or Copilot CLI) | +| **Run analysis** | AI-powered diagnosis of weak steps (requires Claude Code, GitHub Copilot CLI, or Devin CLI) | | **Generate improvement proposal** | AI generates surgical prompt edits based on a diagnosis | | **Review and apply** | Interactive diff review to accept/reject individual edits | +| **Fetch community submissions** | List open community-feedback GitHub issues, parse + validate their METRICS_JSON payload, import into the local run store, and close them | ### Each Time You Finish a Spec @@ -256,6 +257,7 @@ The analysis and improvement steps require an AI agent. Two backends are support |---------|---------|-------| | **Claude Code** | `claude` | Uses `--print` mode. Recommended. | | **GitHub Copilot CLI** | `copilot` | Uses stdin piping with `--allow-all-tools -s`. | +| **Devin CLI** | `devin` | Uses `--print --prompt-file --permission-mode dangerous`. | Model selection is interactive — choose from available models when prompted. @@ -271,7 +273,8 @@ src/ ├── analyzer.ts # AI diagnosis via LLM invocation ├── improver.ts # AI improvement proposal + validation ├── applier.ts # Interactive diff review + file patching -├── invoke-llm.ts # Unified LLM invocation (Claude Code / Copilot CLI) +├── community.ts # Community-feedback issue parsing + ingestion +├── invoke-llm.ts # Unified LLM invocation (Claude / Copilot / Devin) ├── index.ts # Public API exports └── prompts/ ├── analyze.md # AI prompt template for diagnosis diff --git a/plan2code-metrics/src/aggregator.test.ts b/plan2code-metrics/src/aggregator.test.ts index e629a13..1933994 100644 --- a/plan2code-metrics/src/aggregator.test.ts +++ b/plan2code-metrics/src/aggregator.test.ts @@ -1,6 +1,9 @@ -import { describe, it, expect } from 'vitest'; -import { avg, rate, buildCohortKey, backfillPromptVersions } from './aggregator.js'; -import type { PromptVersions } from './types.js'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { describe, it, expect, afterEach } from 'vitest'; +import { avg, rate, buildCohortKey, backfillPromptVersions, cohortKeyForRun, aggregate } from './aggregator.js'; +import type { PromptVersions, RunMetrics } from './types.js'; // ── avg() ───────────────────────────────────────────────────────────────────── @@ -152,3 +155,99 @@ describe('buildCohortKey', () => { expect(buildCohortKey(ordered)).toBe(buildCohortKey(reversed)); }); }); + +// ── Run fixtures for cohort keying / aggregation ────────────────────────────── + +function makeRun(overrides: Partial = {}): RunMetrics { + return { + schema_version: '1.0', + run_id: 'run-20260101-000000-0000', + plan2code_version: '1.17.0', + prompt_versions: { ...FULL_VERSIONS }, + project: { name: 'proj', started_at: null, completed_at: null }, + step1_plan: { present: false, final_confidence: null, confidence_breakdown: null, clarification_rounds: null, tech_stack_revision_rounds: null, verification_gaps_found: null, functional_requirements_count: null, non_functional_requirements_count: null, risk_count: null, phase_count: null }, + step2_document: { present: false, total_tasks: null, tasks_per_phase: null, phase_count: null, parallel_groups_identified: null, requirement_coverage_percent: null, verification_items_added: null }, + step3_implement: { present: false, task_completion_rate: null, tasks_completed: null, tasks_total: null, blocker_count: null }, + step4_finalize: { present: false, completion_rate_at_audit: null, verification_failures_found: null, documentation_updates_needed: null, archival_succeeded: null }, + user_feedback: null, + ...overrides, + }; +} + +// ── cohortKeyForRun() ───────────────────────────────────────────────────────── + +describe('cohortKeyForRun', () => { + it('keys local runs by the prompt-version hash (unchanged from buildCohortKey)', () => { + const run = makeRun({ source: 'local' }); + expect(cohortKeyForRun(run)).toBe(buildCohortKey(run.prompt_versions)); + }); + + it('treats a run with no source as local', () => { + const run = makeRun(); + delete run.source; + expect(cohortKeyForRun(run)).toBe(buildCohortKey(run.prompt_versions)); + }); + + it('keys community runs by plan2code_version, ignoring prompt fingerprints', () => { + const run = makeRun({ source: 'community', plan2code_version: '1.17.0' }); + expect(cohortKeyForRun(run)).toBe('community:v1.17.0'); + }); + + it('groups two community runs of the same version together regardless of prompt fingerprint', () => { + const a = makeRun({ source: 'community', plan2code_version: '1.17.0', prompt_versions: { ...FULL_VERSIONS } }); + const b = makeRun({ source: 'community', plan2code_version: '1.17.0', prompt_versions: backfillPromptVersions({} as PromptVersions) }); + expect(cohortKeyForRun(a)).toBe(cohortKeyForRun(b)); + }); + + it('separates community runs from different versions', () => { + const a = makeRun({ source: 'community', plan2code_version: '1.17.0' }); + const b = makeRun({ source: 'community', plan2code_version: '1.18.0' }); + expect(cohortKeyForRun(a)).not.toBe(cohortKeyForRun(b)); + }); +}); + +// ── aggregate() cohort separation ───────────────────────────────────────────── + +describe('aggregate', () => { + let tmpDir: string; + + afterEach(() => { + if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + function writeRuns(runs: RunMetrics[]): { runsDir: string; outPath: string } { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan2code-agg-test-')); + const runsDir = path.join(tmpDir, 'runs'); + fs.mkdirSync(runsDir, { recursive: true }); + for (const run of runs) { + fs.writeFileSync(path.join(runsDir, `${run.run_id}.json`), JSON.stringify(run), 'utf8'); + } + return { runsDir, outPath: path.join(tmpDir, 'aggregated.json') }; + } + + it('places local and community runs of the same version in separate cohorts', () => { + const local = makeRun({ run_id: 'run-20260101-000001-0001', source: 'local' }); + const community = makeRun({ run_id: 'run-20260101-000002-0002', source: 'community' }); + const { runsDir, outPath } = writeRuns([local, community]); + + const result = aggregate(runsDir, outPath); + + expect(result.total_runs).toBe(2); + expect(result.cohorts).toHaveLength(2); + const communityCohort = result.cohorts.find(c => c.source === 'community'); + const localCohort = result.cohorts.find(c => c.source === 'local'); + expect(communityCohort?.cohort_key).toBe('community:v1.17.0'); + expect(localCohort?.cohort_key).toBe(buildCohortKey(local.prompt_versions)); + }); + + it('never selects a community cohort as current when a local cohort exists', () => { + // Community run sorts last by run_id, but current must stay on the local cohort. + const local = makeRun({ run_id: 'run-20260101-000001-0001', source: 'local' }); + const community = makeRun({ run_id: 'run-29991231-235959-9999', source: 'community' }); + const { runsDir, outPath } = writeRuns([local, community]); + + const result = aggregate(runsDir, outPath); + + expect(result.current_cohort_key).toBe(buildCohortKey(local.prompt_versions)); + }); +}); diff --git a/plan2code-metrics/src/aggregator.ts b/plan2code-metrics/src/aggregator.ts index 9bef031..48593b9 100644 --- a/plan2code-metrics/src/aggregator.ts +++ b/plan2code-metrics/src/aggregator.ts @@ -35,6 +35,26 @@ export function buildCohortKey(promptVersions: PromptVersions): string { return crypto.createHash('sha256').update(str).digest('hex').slice(0, 12); } +/** + * Cohort key for a single run, chosen by the run's origin. + * + * Local runs are keyed by their prompt-file SHA-256 fingerprint, which captures + * in-development prompt edits that share one unreleased version. + * + * Community submissions can't reproduce that byte-exact hash: they carry the + * installed, platform-transformed prompts (not the raw src/*.md the local + * collector hashes), and the payload is LLM-generated. They are keyed instead + * by `plan2code_version` -- a reliable, byte-comparison-free identifier, since + * every released version ships a fixed set of prompts. This keeps community + * cohorts free of the CRLF/whitespace fragility a cross-machine hash would have. + */ +export function cohortKeyForRun(run: RunMetrics): string { + if (run.source === 'community') { + return `community:v${run.plan2code_version}`; + } + return buildCohortKey(run.prompt_versions); +} + /** * Backfill missing PromptVersions fields for old run files (pre-v1.1). */ @@ -116,6 +136,7 @@ function buildCohort(runs: RunMetrics[], cohortKey: string): CohortMetrics { return { cohort_key: cohortKey, + source: runs[0].source ?? 'local', prompt_versions: runs[0].prompt_versions, run_count: runs.length, run_ids: runIds, @@ -157,7 +178,7 @@ export function aggregate(runsDir: string, outputPath: string): AggregatedMetric // Group by cohort key const cohortMap = new Map(); for (const run of runs) { - const key = buildCohortKey(run.prompt_versions); + const key = cohortKeyForRun(run); if (!cohortMap.has(key)) cohortMap.set(key, []); cohortMap.get(key)!.push(run); } @@ -169,9 +190,14 @@ export function aggregate(runsDir: string, outputPath: string): AggregatedMetric } cohorts.sort((a, b) => a.first_seen.localeCompare(b.first_seen)); - // Determine current cohort (most recent) - const currentCohortKey = cohorts.length > 0 - ? cohorts[cohorts.length - 1].cohort_key + // Determine current cohort (most recent). Prefer local cohorts so an + // ingested community submission never becomes the maintainer's "current + // generation" for self-improvement; fall back to all cohorts if there are + // no local runs yet. + const localCohorts = cohorts.filter(c => c.source !== 'community'); + const currentPool = localCohorts.length > 0 ? localCohorts : cohorts; + const currentCohortKey = currentPool.length > 0 + ? currentPool[currentPool.length - 1].cohort_key : null; const aggregated: AggregatedMetrics = { @@ -200,13 +226,10 @@ export function loadAggregated(outputPath: string): AggregatedMetrics | null { } /** - * Import a single run JSON from another project into the local runs dir. - * Returns true if imported, false if already present. + * Write a run to the local runs dir, deduped by run_id filename. + * Returns true if written, false if a file for that run_id already existed. */ -export function importRun(runJsonPath: string, runsDir: string): boolean { - const content = fs.readFileSync(runJsonPath, 'utf8'); - const run = JSON.parse(content) as RunMetrics; - run.prompt_versions = backfillPromptVersions(run.prompt_versions); +export function writeRunFile(run: RunMetrics, runsDir: string): boolean { const destPath = path.join(runsDir, `${run.run_id}.json`); if (fs.existsSync(destPath)) { @@ -217,3 +240,14 @@ export function importRun(runJsonPath: string, runsDir: string): boolean { fs.writeFileSync(destPath, JSON.stringify(run, null, 2), 'utf8'); return true; } + +/** + * Import a single run JSON from another project into the local runs dir. + * Returns true if imported, false if already present. + */ +export function importRun(runJsonPath: string, runsDir: string): boolean { + const content = fs.readFileSync(runJsonPath, 'utf8'); + const run = JSON.parse(content) as RunMetrics; + run.prompt_versions = backfillPromptVersions(run.prompt_versions); + return writeRunFile(run, runsDir); +} diff --git a/plan2code-metrics/src/cli.ts b/plan2code-metrics/src/cli.ts index b9cc0d2..e15fa53 100644 --- a/plan2code-metrics/src/cli.ts +++ b/plan2code-metrics/src/cli.ts @@ -10,14 +10,16 @@ import path from 'path'; import { select, input, confirm } from '@inquirer/prompts'; import chalk from 'chalk'; import ora from 'ora'; +import { execa } from 'execa'; import { collectRun } from './collector.js'; -import { aggregate, loadAggregated, importRun, loadRunFiles } from './aggregator.js'; +import { aggregate, loadAggregated, importRun, loadRunFiles, writeRunFile } from './aggregator.js'; import { runAnalysis } from './analyzer.js'; import { generateImprovement } from './improver.js'; import { reviewAndApply } from './applier.js'; import type { AggregatedMetrics, CohortMetrics } from './types.js'; import { METRIC_TARGETS } from './types.js'; import { AGENTS, type AgentType } from './invoke-llm.js'; +import { listCommunityIssues, closeIssue, ingestCommunityIssues } from './community.js'; // ── Session state (set at startup via interactive prompts) ─────────────────── @@ -321,18 +323,37 @@ async function flowViewStatus(): Promise { return; } + const localCount = aggregated.cohorts.filter(c => c.source !== 'community').length; + const communityCount = aggregated.cohorts.length - localCount; + console.log(); - console.log(chalk.bold(`Total runs: ${aggregated.total_runs} | Generations: ${aggregated.cohorts.length}`)); + console.log(chalk.bold( + `Total runs: ${aggregated.total_runs} | Generations: ${localCount}` + + (communityCount > 0 ? ` | Community cohorts: ${communityCount}` : '') + )); console.log(chalk.gray(`Last updated: ${aggregated.last_updated}`)); + // Community runs carry no wall-clock timestamps, so their first_seen/last_seen + // fall back to the run_id string; only render a Period line for real ISO dates. + const isIsoDate = (s?: string): boolean => !!s && /^\d{4}-\d{2}-\d{2}/.test(s); + for (let i = 0; i < aggregated.cohorts.length; i++) { const cohort = aggregated.cohorts[i]; const isCurrent = cohort.cohort_key === aggregated.current_cohort_key; const label = isCurrent ? chalk.bold.green('[CURRENT]') : ''; + const isCommunity = cohort.source === 'community'; console.log(); - console.log(chalk.bold(`Generation ${i + 1} (sha:${cohort.cohort_key}) — ${cohort.run_count} run(s) ${label}`)); - console.log(chalk.gray(` Period: ${cohort.first_seen?.slice(0, 10)} → ${cohort.last_seen?.slice(0, 10)}`)); + if (isCommunity) { + // cohort_key is already `community:v` — no sha: prefix. + console.log(chalk.bold(`Community feedback (${cohort.cohort_key}) — ${cohort.run_count} run(s) ${label}`)); + } else { + console.log(chalk.bold(`Generation ${i + 1} (sha:${cohort.cohort_key}) — ${cohort.run_count} run(s) ${label}`)); + } + if (isIsoDate(cohort.first_seen)) { + const end = isIsoDate(cohort.last_seen) ? cohort.last_seen.slice(0, 10) : cohort.first_seen.slice(0, 10); + console.log(chalk.gray(` Period: ${cohort.first_seen.slice(0, 10)} → ${end}`)); + } // Step 1 metrics if (cohort.avg_confidence != null || cohort.avg_clarification_rounds != null) { @@ -383,8 +404,10 @@ async function flowViewStatus(): Promise { console.log(` feedback_count: ${chalk.white(String(cohort.feedback_count))}`); } - // Compare with previous generation - if (i > 0) { + // Compare with the previous cohort -- but only within the same population. + // Community and local cohorts measure different things; a cross-source + // delta (e.g. a community cohort vs the last local generation) is noise. + if (i > 0 && aggregated.cohorts[i - 1].source === cohort.source) { const prev = aggregated.cohorts[i - 1]; const deltas: string[] = []; if (cohort.avg_confidence != null && prev.avg_confidence != null) { @@ -396,7 +419,7 @@ async function flowViewStatus(): Promise { deltas.push(`completion ${d >= 0 ? chalk.green(`▲${(d * 100).toFixed(1)}%`) : chalk.red(`▼${(Math.abs(d) * 100).toFixed(1)}%`)}`); } if (deltas.length > 0) { - console.log(chalk.gray(` vs Gen ${i}: ${deltas.join(' ')}`)); + console.log(chalk.gray(` ${isCommunity ? 'vs prior version' : `vs Gen ${i}`}: ${deltas.join(' ')}`)); } } } @@ -729,6 +752,55 @@ async function flowDelete(): Promise { } } +// ── Flow: Fetch community submissions ──────────────────────────────────────── + +const COMMUNITY_REPO = 'jparkerweb/plan2code'; + +async function flowFetchCommunitySubmissions(): Promise { + console.log(); + console.log(chalk.bold.cyan('── Fetch Community Submissions ──')); + + try { + await execa('gh', ['auth', 'status']); + } catch { + console.log(chalk.red('`gh` CLI not found or not authenticated — install/auth `gh` to use this feature.')); + return; + } + + let issues; + try { + issues = await listCommunityIssues(COMMUNITY_REPO); + } catch (err) { + console.log(chalk.red(`Failed to list community-feedback issues: ${err instanceof Error ? err.message : String(err)}`)); + return; + } + + if (issues.length === 0) { + console.log(chalk.yellow('No open community-feedback issues found.')); + return; + } + + const { runsDir, aggregatedPath } = getMetricsDirs(); + + const tally = await ingestCommunityIssues(issues, COMMUNITY_REPO, runsDir, { writeRunFile, closeIssue }); + + for (const num of tally.malformedIssues) { + console.log(chalk.yellow(` Skipping issue #${num}: malformed or missing METRICS_JSON payload.`)); + } + for (const num of tally.closeFailedIssues) { + console.log(chalk.yellow(` Imported issue #${num} but failed to close it (still open; will retry next fetch).`)); + } + + if (tally.imported > 0) { + aggregate(runsDir, aggregatedPath); + } + + console.log(); + console.log(chalk.green( + `✓ Imported: ${tally.imported} Skipped (duplicate): ${tally.skippedDuplicate} Skipped (malformed): ${tally.skippedMalformed} Closed: ${tally.closed} Close failed: ${tally.closeFailed}` + )); +} + // ── Main menu ───────────────────────────────────────────────────────────────── export async function runCLI(): Promise { @@ -800,6 +872,7 @@ export async function runCLI(): Promise { { name: 'Run analysis (diagnose weak steps)', value: 'analyze' }, { name: 'Generate improvement proposal', value: 'propose' }, { name: 'Review and apply a proposal', value: 'apply' }, + { name: 'Fetch community submissions', value: 'fetch-community' }, { name: chalk.red('Delete metrics data'), value: 'delete' }, { name: 'Exit', value: 'exit' }, ], @@ -824,6 +897,9 @@ export async function runCLI(): Promise { case 'apply': await flowReviewAndApply(); break; + case 'fetch-community': + await flowFetchCommunitySubmissions(); + break; case 'delete': await flowDelete(); break; diff --git a/plan2code-metrics/src/collector.ts b/plan2code-metrics/src/collector.ts index 4308477..65db484 100644 --- a/plan2code-metrics/src/collector.ts +++ b/plan2code-metrics/src/collector.ts @@ -43,7 +43,7 @@ function readFileSafe(filePath: string): string | null { * If `stepFilter` is provided, returns only the block with matching "step" field. * Returns parsed object or null if not found / invalid. */ -function extractMetricsJson(content: string, stepFilter?: string): Record | null { +export function extractMetricsJson(content: string, stepFilter?: string): Record | null { const re = //g; let match: RegExpExecArray | null; while ((match = re.exec(content)) !== null) { @@ -611,6 +611,7 @@ export async function collectRun(opts: CollectorOptions): Promise { schema_version: '1.0', run_id: runId, plan2code_version: plan2codeVersion, + source: 'local', prompt_versions: collectPromptVersions(plan2codeRoot), project: { name: projectName, diff --git a/plan2code-metrics/src/community.test.ts b/plan2code-metrics/src/community.test.ts new file mode 100644 index 0000000..23a58a2 --- /dev/null +++ b/plan2code-metrics/src/community.test.ts @@ -0,0 +1,243 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { describe, it, expect, afterEach } from 'vitest'; +import { parseSubmissionPayload, ingestCommunityIssues } from './community.js'; +import { writeRunFile } from './aggregator.js'; +import type { RunMetrics } from './types.js'; + +function issueBody(payload: Record): string { + return `Some issue text.\n\n\n`; +} + +const VALID_PAYLOAD = { + schema_version: '1.0', + run_id: 'run-20260715-143000-a1b2', + plan2code_version: '1.15.3', + prompt_versions_short: { + plan: 'abc123def456', revise_plan: 'a', document: 'b', implement: 'c', + finalize: 'd', init: 'e', init_update: 'f', quick_task: 'g', + }, + step1: { + final_confidence: 95, + confidence_breakdown: { requirements: 24, feasibility: 23, integration: 24, risk: 22 }, + clarification_rounds: 0, + tech_stack_revision_rounds: 0, + verification_gaps_found: 0, + functional_requirements_count: 8, + non_functional_requirements_count: 6, + risk_count: 7, + phase_count: 4, + }, + step2: { + total_tasks: 28, + phase_count: 4, + parallel_groups_identified: 1, + requirement_coverage_percent: 100, + verification_items_added: 3, + }, + step3: { + task_completion_rate: 0.96, + tasks_completed: 27, + tasks_total: 28, + blocker_count: 1, + }, + step4: { + completion_rate_at_audit: 0.96, + verification_failures_found: 1, + documentation_updates_needed: 2, + }, + user_feedback: { + overall_rating: 8, + rating_reason: 'good stuff', + what_went_well: 'well', + what_went_poorly: 'poorly', + }, +}; + +// ── parseSubmissionPayload() ───────────────────────────────────────────────── + +describe('parseSubmissionPayload', () => { + it('parses a fully valid payload into a correctly-shaped RunMetrics', () => { + const result = parseSubmissionPayload(issueBody(VALID_PAYLOAD)); + expect(result).not.toBeNull(); + expect(result!.run_id).toBe('run-20260715-143000-a1b2'); + expect(result!.schema_version).toBe('1.0'); + expect(result!.plan2code_version).toBe('1.15.3'); + expect(result!.source).toBe('community'); + expect(result!.step1_plan.present).toBe(true); + expect(result!.step1_plan.final_confidence).toBe(95); + expect(result!.step2_document.present).toBe(true); + expect(result!.step2_document.total_tasks).toBe(28); + expect(result!.step3_implement.present).toBe(true); + expect(result!.step3_implement.tasks_completed).toBe(27); + expect(result!.step4_finalize.present).toBe(true); + expect(result!.step4_finalize.completion_rate_at_audit).toBe(0.96); + expect(result!.user_feedback).toEqual({ + overall_rating: 8, + rating_reason: 'good stuff', + what_went_well: 'well', + what_went_poorly: 'poorly', + }); + }); + + it('returns null when run_id is missing', () => { + const { run_id, ...withoutRunId } = VALID_PAYLOAD; + const result = parseSubmissionPayload(issueBody(withoutRunId)); + expect(result).toBeNull(); + }); + + it('returns null when user_feedback.overall_rating is a string instead of a number', () => { + const badPayload = { + ...VALID_PAYLOAD, + user_feedback: { ...VALID_PAYLOAD.user_feedback, overall_rating: 'eight' }, + }; + const result = parseSubmissionPayload(issueBody(badPayload)); + expect(result).toBeNull(); + }); + + it('returns null when schema_version is not exactly "1.0"', () => { + const badPayload = { ...VALID_PAYLOAD, schema_version: '2.0' }; + const result = parseSubmissionPayload(issueBody(badPayload)); + expect(result).toBeNull(); + }); + + it('sets all four steps present:false when only user_feedback is included', () => { + const minimalPayload = { + schema_version: '1.0', + run_id: 'run-20260715-150000-c3d4', + plan2code_version: '1.15.3', + user_feedback: VALID_PAYLOAD.user_feedback, + }; + const result = parseSubmissionPayload(issueBody(minimalPayload)); + expect(result).not.toBeNull(); + expect(result!.step1_plan.present).toBe(false); + expect(result!.step2_document.present).toBe(false); + expect(result!.step3_implement.present).toBe(false); + expect(result!.step4_finalize.present).toBe(false); + }); + + it('backfills missing prompt_versions_short keys with the sha256:missing sentinel', () => { + const partialPayload = { + ...VALID_PAYLOAD, + prompt_versions_short: { plan: 'abc123def456' }, + }; + const result = parseSubmissionPayload(issueBody(partialPayload)); + expect(result).not.toBeNull(); + expect(result!.prompt_versions.plan).toBe('abc123def456'); + expect(result!.prompt_versions.revise_plan).toBe('sha256:missing'); + expect(result!.prompt_versions.document).toBe('sha256:missing'); + expect(result!.prompt_versions.implement).toBe('sha256:missing'); + expect(result!.prompt_versions.finalize).toBe('sha256:missing'); + expect(result!.prompt_versions.init).toBe('sha256:missing'); + expect(result!.prompt_versions.init_update).toBe('sha256:missing'); + expect(result!.prompt_versions.quick_task).toBe('sha256:missing'); + }); +}); + +// ── writeRunFile() ──────────────────────────────────────────────────────────── + +describe('writeRunFile', () => { + let tmpDir: string; + + afterEach(() => { + if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + const RUN: RunMetrics = { + schema_version: '1.0', + run_id: 'run-20260715-160000-e5f6', + plan2code_version: '1.15.3', + prompt_versions: { + plan: 'sha256:missing', revise_plan: 'sha256:missing', document: 'sha256:missing', + implement: 'sha256:missing', finalize: 'sha256:missing', init: 'sha256:missing', + init_update: 'sha256:missing', quick_task: 'sha256:missing', + }, + project: { name: '', started_at: null, completed_at: null }, + step1_plan: { present: false, final_confidence: null, confidence_breakdown: null, clarification_rounds: null, tech_stack_revision_rounds: null, verification_gaps_found: null, functional_requirements_count: null, non_functional_requirements_count: null, risk_count: null, phase_count: null }, + step2_document: { present: false, total_tasks: null, tasks_per_phase: null, phase_count: null, parallel_groups_identified: null, requirement_coverage_percent: null, verification_items_added: null }, + step3_implement: { present: false, task_completion_rate: null, tasks_completed: null, tasks_total: null, blocker_count: null }, + step4_finalize: { present: false, completion_rate_at_audit: null, verification_failures_found: null, documentation_updates_needed: null, archival_succeeded: null }, + user_feedback: null, + }; + + it('writes a new run_id to an empty runsDir and returns true', () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan2code-metrics-test-')); + const result = writeRunFile(RUN, tmpDir); + expect(result).toBe(true); + expect(fs.existsSync(path.join(tmpDir, `${RUN.run_id}.json`))).toBe(true); + }); + + it('returns false and does not overwrite when the same run_id already exists', () => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan2code-metrics-test-')); + writeRunFile(RUN, tmpDir); + const modified = { ...RUN, plan2code_version: '9.9.9' }; + const result = writeRunFile(modified, tmpDir); + expect(result).toBe(false); + const onDisk = JSON.parse(fs.readFileSync(path.join(tmpDir, `${RUN.run_id}.json`), 'utf8')); + expect(onDisk.plan2code_version).toBe('1.15.3'); + }); +}); + +// ── ingestCommunityIssues() ─────────────────────────────────────────────────── + +describe('ingestCommunityIssues', () => { + const goodBody = issueBody(VALID_PAYLOAD); + const badBody = 'an issue with no METRICS_JSON payload'; + + it('imports a new run and closes its issue', async () => { + const closed: number[] = []; + const tally = await ingestCommunityIssues( + [{ number: 1, body: goodBody }], + 'owner/repo', + '/runs', + { writeRunFile: () => true, closeIssue: async (_r, n) => { closed.push(n); } }, + ); + expect(tally.imported).toBe(1); + expect(tally.skippedDuplicate).toBe(0); + expect(tally.closed).toBe(1); + expect(closed).toEqual([1]); + }); + + it('closes an already-imported (duplicate) issue instead of skipping the close', async () => { + const closed: number[] = []; + const tally = await ingestCommunityIssues( + [{ number: 7, body: goodBody }], + 'owner/repo', + '/runs', + { writeRunFile: () => false, closeIssue: async (_r, n) => { closed.push(n); } }, + ); + expect(tally.imported).toBe(0); + expect(tally.skippedDuplicate).toBe(1); + expect(tally.closed).toBe(1); // the close-retry: duplicates are still closed + expect(closed).toEqual([7]); + }); + + it('records a close failure without throwing and leaves the issue for a later retry', async () => { + const tally = await ingestCommunityIssues( + [{ number: 9, body: goodBody }], + 'owner/repo', + '/runs', + { writeRunFile: () => true, closeIssue: async () => { throw new Error('network'); } }, + ); + expect(tally.imported).toBe(1); + expect(tally.closed).toBe(0); + expect(tally.closeFailed).toBe(1); + expect(tally.closeFailedIssues).toEqual([9]); + }); + + it('skips and reports a malformed issue without writing or closing it', async () => { + let wrote = false; + let closeCalled = false; + const tally = await ingestCommunityIssues( + [{ number: 3, body: badBody }], + 'owner/repo', + '/runs', + { writeRunFile: () => { wrote = true; return true; }, closeIssue: async () => { closeCalled = true; } }, + ); + expect(tally.skippedMalformed).toBe(1); + expect(tally.malformedIssues).toEqual([3]); + expect(wrote).toBe(false); + expect(closeCalled).toBe(false); + }); +}); diff --git a/plan2code-metrics/src/community.ts b/plan2code-metrics/src/community.ts new file mode 100644 index 0000000..1cee29d --- /dev/null +++ b/plan2code-metrics/src/community.ts @@ -0,0 +1,276 @@ +/** + * community.ts + * Ingestion side of the community feedback flow: list/parse/close + * `community-feedback`-labeled GitHub issues on jparkerweb/plan2code. + */ + +import { execa } from 'execa'; +import type { + RunMetrics, + PromptVersions, + UserFeedback, + Step1PlanMetrics, + Step2DocumentMetrics, + Step3ImplementMetrics, + Step4FinalizeMetrics, +} from './types.js'; +import { extractMetricsJson } from './collector.js'; +import { backfillPromptVersions } from './aggregator.js'; + +// ── GitHub interaction (via gh CLI) ────────────────────────────────────────── + +export interface CommunityIssue { + number: number; + body: string; +} + +interface RawIssue { + number: number; + title: string; + body: string; + labels: { name: string }[]; +} + +const COMMUNITY_LABEL = 'community-feedback'; +const FEEDBACK_TITLE_PREFIX = '[Feedback]'; +const METRICS_JSON_MARKER = /