Ingest community feedback submissions in plan2code-metrics

Adds a CLI flow that lists open community-feedback issues via gh, validates
and parses each METRICS_JSON payload, imports them deduped by run_id,
re-aggregates and closes the issue. Community runs cohort by Plan2Code
version rather than prompt fingerprint, and local cohorts stay preferred so
ingested feedback never displaces the maintainer's current generation. Adds
Devin CLI as an AI backend.

AI Assisted

plan2code-metrics/README.md

@@ -116,9 +116,10 @@ plan2code-metrics
 | **Collect metrics** | Parse a completed project spec and extract step-by-step metrics into a run JSON |
 | **Import run data** | Copy a run JSON from another project into the local metrics store |
 | **View metrics status** | Show aggregated metrics with health indicators and generation deltas |
-| **Run analysis** | AI-powered diagnosis of weak steps (requires Claude Code or Copilot CLI) |
+| **Run analysis** | AI-powered diagnosis of weak steps (requires Claude Code, GitHub Copilot CLI, or Devin CLI) |
 | **Generate improvement proposal** | AI generates surgical prompt edits based on a diagnosis |
 | **Review and apply** | Interactive diff review to accept/reject individual edits |
+| **Fetch community submissions** | List open community-feedback GitHub issues, parse + validate their METRICS_JSON payload, import into the local run store, and close them |
 
 ### Each Time You Finish a Spec
 
@@ -256,6 +257,7 @@ The analysis and improvement steps require an AI agent. Two backends are support
 |---------|---------|-------|
 | **Claude Code** | `claude` | Uses `--print` mode. Recommended. |
 | **GitHub Copilot CLI** | `copilot` | Uses stdin piping with `--allow-all-tools -s`. |
+| **Devin CLI** | `devin` | Uses `--print --prompt-file <file> --permission-mode dangerous`. |
 
 Model selection is interactive — choose from available models when prompted.
 
@@ -271,7 +273,8 @@ src/
 ├── analyzer.ts                # AI diagnosis via LLM invocation
 ├── improver.ts                # AI improvement proposal + validation
 ├── applier.ts                 # Interactive diff review + file patching
-├── invoke-llm.ts              # Unified LLM invocation (Claude Code / Copilot CLI)
+├── community.ts               # Community-feedback issue parsing + ingestion
+├── invoke-llm.ts              # Unified LLM invocation (Claude / Copilot / Devin)
 ├── index.ts                   # Public API exports
 └── prompts/
     ├── analyze.md             # AI prompt template for diagnosis

plan2code-metrics/src/aggregator.test.ts

@@ -1,6 +1,9 @@
-import { describe, it, expect } from 'vitest';
-import { avg, rate, buildCohortKey, backfillPromptVersions } from './aggregator.js';
-import type { PromptVersions } from './types.js';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { describe, it, expect, afterEach } from 'vitest';
+import { avg, rate, buildCohortKey, backfillPromptVersions, cohortKeyForRun, aggregate } from './aggregator.js';
+import type { PromptVersions, RunMetrics } from './types.js';
 
 // ── avg() ─────────────────────────────────────────────────────────────────────
 
@@ -152,3 +155,99 @@ describe('buildCohortKey', () => {
     expect(buildCohortKey(ordered)).toBe(buildCohortKey(reversed));
   });
 });
+
+// ── Run fixtures for cohort keying / aggregation ──────────────────────────────
+
+function makeRun(overrides: Partial<RunMetrics> = {}): RunMetrics {
+  return {
+    schema_version: '1.0',
+    run_id: 'run-20260101-000000-0000',
+    plan2code_version: '1.17.0',
+    prompt_versions: { ...FULL_VERSIONS },
+    project: { name: 'proj', started_at: null, completed_at: null },
+    step1_plan: { present: false, final_confidence: null, confidence_breakdown: null, clarification_rounds: null, tech_stack_revision_rounds: null, verification_gaps_found: null, functional_requirements_count: null, non_functional_requirements_count: null, risk_count: null, phase_count: null },
+    step2_document: { present: false, total_tasks: null, tasks_per_phase: null, phase_count: null, parallel_groups_identified: null, requirement_coverage_percent: null, verification_items_added: null },
+    step3_implement: { present: false, task_completion_rate: null, tasks_completed: null, tasks_total: null, blocker_count: null },
+    step4_finalize: { present: false, completion_rate_at_audit: null, verification_failures_found: null, documentation_updates_needed: null, archival_succeeded: null },
+    user_feedback: null,
+    ...overrides,
+  };
+}
+
+// ── cohortKeyForRun() ─────────────────────────────────────────────────────────
+
+describe('cohortKeyForRun', () => {
+  it('keys local runs by the prompt-version hash (unchanged from buildCohortKey)', () => {
+    const run = makeRun({ source: 'local' });
+    expect(cohortKeyForRun(run)).toBe(buildCohortKey(run.prompt_versions));
+  });
+
+  it('treats a run with no source as local', () => {
+    const run = makeRun();
+    delete run.source;
+    expect(cohortKeyForRun(run)).toBe(buildCohortKey(run.prompt_versions));
+  });
+
+  it('keys community runs by plan2code_version, ignoring prompt fingerprints', () => {
+    const run = makeRun({ source: 'community', plan2code_version: '1.17.0' });
+    expect(cohortKeyForRun(run)).toBe('community:v1.17.0');
+  });
+
+  it('groups two community runs of the same version together regardless of prompt fingerprint', () => {
+    const a = makeRun({ source: 'community', plan2code_version: '1.17.0', prompt_versions: { ...FULL_VERSIONS } });
+    const b = makeRun({ source: 'community', plan2code_version: '1.17.0', prompt_versions: backfillPromptVersions({} as PromptVersions) });
+    expect(cohortKeyForRun(a)).toBe(cohortKeyForRun(b));
+  });
+
+  it('separates community runs from different versions', () => {
+    const a = makeRun({ source: 'community', plan2code_version: '1.17.0' });
+    const b = makeRun({ source: 'community', plan2code_version: '1.18.0' });
+    expect(cohortKeyForRun(a)).not.toBe(cohortKeyForRun(b));
+  });
+});
+
+// ── aggregate() cohort separation ─────────────────────────────────────────────
+
+describe('aggregate', () => {
+  let tmpDir: string;
+
+  afterEach(() => {
+    if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  function writeRuns(runs: RunMetrics[]): { runsDir: string; outPath: string } {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan2code-agg-test-'));
+    const runsDir = path.join(tmpDir, 'runs');
+    fs.mkdirSync(runsDir, { recursive: true });
+    for (const run of runs) {
+      fs.writeFileSync(path.join(runsDir, `${run.run_id}.json`), JSON.stringify(run), 'utf8');
+    }
+    return { runsDir, outPath: path.join(tmpDir, 'aggregated.json') };
+  }
+
+  it('places local and community runs of the same version in separate cohorts', () => {
+    const local = makeRun({ run_id: 'run-20260101-000001-0001', source: 'local' });
+    const community = makeRun({ run_id: 'run-20260101-000002-0002', source: 'community' });
+    const { runsDir, outPath } = writeRuns([local, community]);
+
+    const result = aggregate(runsDir, outPath);
+
+    expect(result.total_runs).toBe(2);
+    expect(result.cohorts).toHaveLength(2);
+    const communityCohort = result.cohorts.find(c => c.source === 'community');
+    const localCohort = result.cohorts.find(c => c.source === 'local');
+    expect(communityCohort?.cohort_key).toBe('community:v1.17.0');
+    expect(localCohort?.cohort_key).toBe(buildCohortKey(local.prompt_versions));
+  });
+
+  it('never selects a community cohort as current when a local cohort exists', () => {
+    // Community run sorts last by run_id, but current must stay on the local cohort.
+    const local = makeRun({ run_id: 'run-20260101-000001-0001', source: 'local' });
+    const community = makeRun({ run_id: 'run-29991231-235959-9999', source: 'community' });
+    const { runsDir, outPath } = writeRuns([local, community]);
+
+    const result = aggregate(runsDir, outPath);
+
+    expect(result.current_cohort_key).toBe(buildCohortKey(local.prompt_versions));
+  });
+});

plan2code-metrics/src/aggregator.ts

@@ -35,6 +35,26 @@ export function buildCohortKey(promptVersions: PromptVersions): string {
   return crypto.createHash('sha256').update(str).digest('hex').slice(0, 12);
 }
 
+/**
+ * Cohort key for a single run, chosen by the run's origin.
+ *
+ * Local runs are keyed by their prompt-file SHA-256 fingerprint, which captures
+ * in-development prompt edits that share one unreleased version.
+ *
+ * Community submissions can't reproduce that byte-exact hash: they carry the
+ * installed, platform-transformed prompts (not the raw src/*.md the local
+ * collector hashes), and the payload is LLM-generated. They are keyed instead
+ * by `plan2code_version` -- a reliable, byte-comparison-free identifier, since
+ * every released version ships a fixed set of prompts. This keeps community
+ * cohorts free of the CRLF/whitespace fragility a cross-machine hash would have.
+ */
+export function cohortKeyForRun(run: RunMetrics): string {
+  if (run.source === 'community') {
+    return `community:v${run.plan2code_version}`;
+  }
+  return buildCohortKey(run.prompt_versions);
+}
+
 /**
  * Backfill missing PromptVersions fields for old run files (pre-v1.1).
  */
@@ -116,6 +136,7 @@ function buildCohort(runs: RunMetrics[], cohortKey: string): CohortMetrics {
 
   return {
     cohort_key: cohortKey,
+    source: runs[0].source ?? 'local',
     prompt_versions: runs[0].prompt_versions,
     run_count: runs.length,
     run_ids: runIds,
@@ -157,7 +178,7 @@ export function aggregate(runsDir: string, outputPath: string): AggregatedMetric
   // Group by cohort key
   const cohortMap = new Map<string, RunMetrics[]>();
   for (const run of runs) {
-    const key = buildCohortKey(run.prompt_versions);
+    const key = cohortKeyForRun(run);
     if (!cohortMap.has(key)) cohortMap.set(key, []);
     cohortMap.get(key)!.push(run);
   }
@@ -169,9 +190,14 @@ export function aggregate(runsDir: string, outputPath: string): AggregatedMetric
   }
   cohorts.sort((a, b) => a.first_seen.localeCompare(b.first_seen));
 
-  // Determine current cohort (most recent)
-  const currentCohortKey = cohorts.length > 0
-    ? cohorts[cohorts.length - 1].cohort_key
+  // Determine current cohort (most recent). Prefer local cohorts so an
+  // ingested community submission never becomes the maintainer's "current
+  // generation" for self-improvement; fall back to all cohorts if there are
+  // no local runs yet.
+  const localCohorts = cohorts.filter(c => c.source !== 'community');
+  const currentPool = localCohorts.length > 0 ? localCohorts : cohorts;
+  const currentCohortKey = currentPool.length > 0
+    ? currentPool[currentPool.length - 1].cohort_key
     : null;
 
   const aggregated: AggregatedMetrics = {
@@ -200,13 +226,10 @@ export function loadAggregated(outputPath: string): AggregatedMetrics | null {
 }
 
 /**
- * Import a single run JSON from another project into the local runs dir.
- * Returns true if imported, false if already present.
+ * Write a run to the local runs dir, deduped by run_id filename.
+ * Returns true if written, false if a file for that run_id already existed.
  */
-export function importRun(runJsonPath: string, runsDir: string): boolean {
-  const content = fs.readFileSync(runJsonPath, 'utf8');
-  const run = JSON.parse(content) as RunMetrics;
-  run.prompt_versions = backfillPromptVersions(run.prompt_versions);
+export function writeRunFile(run: RunMetrics, runsDir: string): boolean {
   const destPath = path.join(runsDir, `${run.run_id}.json`);
 
   if (fs.existsSync(destPath)) {
@@ -217,3 +240,14 @@ export function importRun(runJsonPath: string, runsDir: string): boolean {
   fs.writeFileSync(destPath, JSON.stringify(run, null, 2), 'utf8');
   return true;
 }
+
+/**
+ * Import a single run JSON from another project into the local runs dir.
+ * Returns true if imported, false if already present.
+ */
+export function importRun(runJsonPath: string, runsDir: string): boolean {
+  const content = fs.readFileSync(runJsonPath, 'utf8');
+  const run = JSON.parse(content) as RunMetrics;
+  run.prompt_versions = backfillPromptVersions(run.prompt_versions);
+  return writeRunFile(run, runsDir);
+}

plan2code-metrics/src/cli.ts

@@ -10,14 +10,16 @@ import path from 'path';
 import { select, input, confirm } from '@inquirer/prompts';
 import chalk from 'chalk';
 import ora from 'ora';
+import { execa } from 'execa';
 import { collectRun } from './collector.js';
-import { aggregate, loadAggregated, importRun, loadRunFiles } from './aggregator.js';
+import { aggregate, loadAggregated, importRun, loadRunFiles, writeRunFile } from './aggregator.js';
 import { runAnalysis } from './analyzer.js';
 import { generateImprovement } from './improver.js';
 import { reviewAndApply } from './applier.js';
 import type { AggregatedMetrics, CohortMetrics } from './types.js';
 import { METRIC_TARGETS } from './types.js';
 import { AGENTS, type AgentType } from './invoke-llm.js';
+import { listCommunityIssues, closeIssue, ingestCommunityIssues } from './community.js';
 
 // ── Session state (set at startup via interactive prompts) ───────────────────
 
@@ -321,18 +323,37 @@ async function flowViewStatus(): Promise<void> {
     return;
   }
 
+  const localCount = aggregated.cohorts.filter(c => c.source !== 'community').length;
+  const communityCount = aggregated.cohorts.length - localCount;
+
   console.log();
-  console.log(chalk.bold(`Total runs: ${aggregated.total_runs}  |  Generations: ${aggregated.cohorts.length}`));
+  console.log(chalk.bold(
+    `Total runs: ${aggregated.total_runs}  |  Generations: ${localCount}` +
+    (communityCount > 0 ? `  |  Community cohorts: ${communityCount}` : '')
+  ));
   console.log(chalk.gray(`Last updated: ${aggregated.last_updated}`));
 
+  // Community runs carry no wall-clock timestamps, so their first_seen/last_seen
+  // fall back to the run_id string; only render a Period line for real ISO dates.
+  const isIsoDate = (s?: string): boolean => !!s && /^\d{4}-\d{2}-\d{2}/.test(s);
+
   for (let i = 0; i < aggregated.cohorts.length; i++) {
     const cohort = aggregated.cohorts[i];
     const isCurrent = cohort.cohort_key === aggregated.current_cohort_key;
     const label = isCurrent ? chalk.bold.green('[CURRENT]') : '';
+    const isCommunity = cohort.source === 'community';
 
     console.log();
-    console.log(chalk.bold(`Generation ${i + 1} (sha:${cohort.cohort_key}) — ${cohort.run_count} run(s) ${label}`));
-    console.log(chalk.gray(`  Period: ${cohort.first_seen?.slice(0, 10)} → ${cohort.last_seen?.slice(0, 10)}`));
+    if (isCommunity) {
+      // cohort_key is already `community:v<version>` — no sha: prefix.
+      console.log(chalk.bold(`Community feedback (${cohort.cohort_key}) — ${cohort.run_count} run(s) ${label}`));
+    } else {
+      console.log(chalk.bold(`Generation ${i + 1} (sha:${cohort.cohort_key}) — ${cohort.run_count} run(s) ${label}`));
+    }
+    if (isIsoDate(cohort.first_seen)) {
+      const end = isIsoDate(cohort.last_seen) ? cohort.last_seen.slice(0, 10) : cohort.first_seen.slice(0, 10);
+      console.log(chalk.gray(`  Period: ${cohort.first_seen.slice(0, 10)} → ${end}`));
+    }
 
     // Step 1 metrics
     if (cohort.avg_confidence != null || cohort.avg_clarification_rounds != null) {
@@ -383,8 +404,10 @@ async function flowViewStatus(): Promise<void> {
       console.log(`    feedback_count:           ${chalk.white(String(cohort.feedback_count))}`);
     }
 
-    // Compare with previous generation
-    if (i > 0) {
+    // Compare with the previous cohort -- but only within the same population.
+    // Community and local cohorts measure different things; a cross-source
+    // delta (e.g. a community cohort vs the last local generation) is noise.
+    if (i > 0 && aggregated.cohorts[i - 1].source === cohort.source) {
       const prev = aggregated.cohorts[i - 1];
       const deltas: string[] = [];
       if (cohort.avg_confidence != null && prev.avg_confidence != null) {
@@ -396,7 +419,7 @@ async function flowViewStatus(): Promise<void> {
         deltas.push(`completion ${d >= 0 ? chalk.green(`▲${(d * 100).toFixed(1)}%`) : chalk.red(`▼${(Math.abs(d) * 100).toFixed(1)}%`)}`);
       }
       if (deltas.length > 0) {
-        console.log(chalk.gray(`  vs Gen ${i}: ${deltas.join('  ')}`));
+        console.log(chalk.gray(`  ${isCommunity ? 'vs prior version' : `vs Gen ${i}`}: ${deltas.join('  ')}`));
       }
     }
   }
@@ -729,6 +752,55 @@ async function flowDelete(): Promise<void> {
   }
 }
 
+// ── Flow: Fetch community submissions ────────────────────────────────────────
+
+const COMMUNITY_REPO = 'jparkerweb/plan2code';
+
+async function flowFetchCommunitySubmissions(): Promise<void> {
+  console.log();
+  console.log(chalk.bold.cyan('── Fetch Community Submissions ──'));
+
+  try {
+    await execa('gh', ['auth', 'status']);
+  } catch {
+    console.log(chalk.red('`gh` CLI not found or not authenticated — install/auth `gh` to use this feature.'));
+    return;
+  }
+
+  let issues;
+  try {
+    issues = await listCommunityIssues(COMMUNITY_REPO);
+  } catch (err) {
+    console.log(chalk.red(`Failed to list community-feedback issues: ${err instanceof Error ? err.message : String(err)}`));
+    return;
+  }
+
+  if (issues.length === 0) {
+    console.log(chalk.yellow('No open community-feedback issues found.'));
+    return;
+  }
+
+  const { runsDir, aggregatedPath } = getMetricsDirs();
+
+  const tally = await ingestCommunityIssues(issues, COMMUNITY_REPO, runsDir, { writeRunFile, closeIssue });
+
+  for (const num of tally.malformedIssues) {
+    console.log(chalk.yellow(`  Skipping issue #${num}: malformed or missing METRICS_JSON payload.`));
+  }
+  for (const num of tally.closeFailedIssues) {
+    console.log(chalk.yellow(`  Imported issue #${num} but failed to close it (still open; will retry next fetch).`));
+  }
+
+  if (tally.imported > 0) {
+    aggregate(runsDir, aggregatedPath);
+  }
+
+  console.log();
+  console.log(chalk.green(
+    `✓ Imported: ${tally.imported}  Skipped (duplicate): ${tally.skippedDuplicate}  Skipped (malformed): ${tally.skippedMalformed}  Closed: ${tally.closed}  Close failed: ${tally.closeFailed}`
+  ));
+}
+
 // ── Main menu ─────────────────────────────────────────────────────────────────
 
 export async function runCLI(): Promise<void> {
@@ -800,6 +872,7 @@ export async function runCLI(): Promise<void> {
         { name: 'Run analysis (diagnose weak steps)', value: 'analyze' },
         { name: 'Generate improvement proposal', value: 'propose' },
         { name: 'Review and apply a proposal', value: 'apply' },
+        { name: 'Fetch community submissions', value: 'fetch-community' },
         { name: chalk.red('Delete metrics data'), value: 'delete' },
         { name: 'Exit', value: 'exit' },
       ],
@@ -824,6 +897,9 @@ export async function runCLI(): Promise<void> {
       case 'apply':
         await flowReviewAndApply();
         break;
+      case 'fetch-community':
+        await flowFetchCommunitySubmissions();
+        break;
       case 'delete':
         await flowDelete();
         break;

plan2code-metrics/src/collector.ts

@@ -43,7 +43,7 @@ function readFileSafe(filePath: string): string | null {
  * If `stepFilter` is provided, returns only the block with matching "step" field.
  * Returns parsed object or null if not found / invalid.
  */
-function extractMetricsJson(content: string, stepFilter?: string): Record<string, unknown> | null {
+export function extractMetricsJson(content: string, stepFilter?: string): Record<string, unknown> | null {
   const re = /<!--\s*METRICS_JSON\s+(\{[\s\S]*?\})\s*-->/g;
   let match: RegExpExecArray | null;
   while ((match = re.exec(content)) !== null) {
@@ -611,6 +611,7 @@ export async function collectRun(opts: CollectorOptions): Promise<RunMetrics> {
     schema_version: '1.0',
     run_id: runId,
     plan2code_version: plan2codeVersion,
+    source: 'local',
     prompt_versions: collectPromptVersions(plan2codeRoot),
     project: {
       name: projectName,

plan2code-metrics/src/community.test.ts

@@ -0,0 +1,243 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { describe, it, expect, afterEach } from 'vitest';
+import { parseSubmissionPayload, ingestCommunityIssues } from './community.js';
+import { writeRunFile } from './aggregator.js';
+import type { RunMetrics } from './types.js';
+
+function issueBody(payload: Record<string, unknown>): string {
+  return `Some issue text.\n\n<!-- METRICS_JSON ${JSON.stringify(payload)} -->\n`;
+}
+
+const VALID_PAYLOAD = {
+  schema_version: '1.0',
+  run_id: 'run-20260715-143000-a1b2',
+  plan2code_version: '1.15.3',
+  prompt_versions_short: {
+    plan: 'abc123def456', revise_plan: 'a', document: 'b', implement: 'c',
+    finalize: 'd', init: 'e', init_update: 'f', quick_task: 'g',
+  },
+  step1: {
+    final_confidence: 95,
+    confidence_breakdown: { requirements: 24, feasibility: 23, integration: 24, risk: 22 },
+    clarification_rounds: 0,
+    tech_stack_revision_rounds: 0,
+    verification_gaps_found: 0,
+    functional_requirements_count: 8,
+    non_functional_requirements_count: 6,
+    risk_count: 7,
+    phase_count: 4,
+  },
+  step2: {
+    total_tasks: 28,
+    phase_count: 4,
+    parallel_groups_identified: 1,
+    requirement_coverage_percent: 100,
+    verification_items_added: 3,
+  },
+  step3: {
+    task_completion_rate: 0.96,
+    tasks_completed: 27,
+    tasks_total: 28,
+    blocker_count: 1,
+  },
+  step4: {
+    completion_rate_at_audit: 0.96,
+    verification_failures_found: 1,
+    documentation_updates_needed: 2,
+  },
+  user_feedback: {
+    overall_rating: 8,
+    rating_reason: 'good stuff',
+    what_went_well: 'well',
+    what_went_poorly: 'poorly',
+  },
+};
+
+// ── parseSubmissionPayload() ─────────────────────────────────────────────────
+
+describe('parseSubmissionPayload', () => {
+  it('parses a fully valid payload into a correctly-shaped RunMetrics', () => {
+    const result = parseSubmissionPayload(issueBody(VALID_PAYLOAD));
+    expect(result).not.toBeNull();
+    expect(result!.run_id).toBe('run-20260715-143000-a1b2');
+    expect(result!.schema_version).toBe('1.0');
+    expect(result!.plan2code_version).toBe('1.15.3');
+    expect(result!.source).toBe('community');
+    expect(result!.step1_plan.present).toBe(true);
+    expect(result!.step1_plan.final_confidence).toBe(95);
+    expect(result!.step2_document.present).toBe(true);
+    expect(result!.step2_document.total_tasks).toBe(28);
+    expect(result!.step3_implement.present).toBe(true);
+    expect(result!.step3_implement.tasks_completed).toBe(27);
+    expect(result!.step4_finalize.present).toBe(true);
+    expect(result!.step4_finalize.completion_rate_at_audit).toBe(0.96);
+    expect(result!.user_feedback).toEqual({
+      overall_rating: 8,
+      rating_reason: 'good stuff',
+      what_went_well: 'well',
+      what_went_poorly: 'poorly',
+    });
+  });
+
+  it('returns null when run_id is missing', () => {
+    const { run_id, ...withoutRunId } = VALID_PAYLOAD;
+    const result = parseSubmissionPayload(issueBody(withoutRunId));
+    expect(result).toBeNull();
+  });
+
+  it('returns null when user_feedback.overall_rating is a string instead of a number', () => {
+    const badPayload = {
+      ...VALID_PAYLOAD,
+      user_feedback: { ...VALID_PAYLOAD.user_feedback, overall_rating: 'eight' },
+    };
+    const result = parseSubmissionPayload(issueBody(badPayload));
+    expect(result).toBeNull();
+  });
+
+  it('returns null when schema_version is not exactly "1.0"', () => {
+    const badPayload = { ...VALID_PAYLOAD, schema_version: '2.0' };
+    const result = parseSubmissionPayload(issueBody(badPayload));
+    expect(result).toBeNull();
+  });
+
+  it('sets all four steps present:false when only user_feedback is included', () => {
+    const minimalPayload = {
+      schema_version: '1.0',
+      run_id: 'run-20260715-150000-c3d4',
+      plan2code_version: '1.15.3',
+      user_feedback: VALID_PAYLOAD.user_feedback,
+    };
+    const result = parseSubmissionPayload(issueBody(minimalPayload));
+    expect(result).not.toBeNull();
+    expect(result!.step1_plan.present).toBe(false);
+    expect(result!.step2_document.present).toBe(false);
+    expect(result!.step3_implement.present).toBe(false);
+    expect(result!.step4_finalize.present).toBe(false);
+  });
+
+  it('backfills missing prompt_versions_short keys with the sha256:missing sentinel', () => {
+    const partialPayload = {
+      ...VALID_PAYLOAD,
+      prompt_versions_short: { plan: 'abc123def456' },
+    };
+    const result = parseSubmissionPayload(issueBody(partialPayload));
+    expect(result).not.toBeNull();
+    expect(result!.prompt_versions.plan).toBe('abc123def456');
+    expect(result!.prompt_versions.revise_plan).toBe('sha256:missing');
+    expect(result!.prompt_versions.document).toBe('sha256:missing');
+    expect(result!.prompt_versions.implement).toBe('sha256:missing');
+    expect(result!.prompt_versions.finalize).toBe('sha256:missing');
+    expect(result!.prompt_versions.init).toBe('sha256:missing');
+    expect(result!.prompt_versions.init_update).toBe('sha256:missing');
+    expect(result!.prompt_versions.quick_task).toBe('sha256:missing');
+  });
+});
+
+// ── writeRunFile() ────────────────────────────────────────────────────────────
+
+describe('writeRunFile', () => {
+  let tmpDir: string;
+
+  afterEach(() => {
+    if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  const RUN: RunMetrics = {
+    schema_version: '1.0',
+    run_id: 'run-20260715-160000-e5f6',
+    plan2code_version: '1.15.3',
+    prompt_versions: {
+      plan: 'sha256:missing', revise_plan: 'sha256:missing', document: 'sha256:missing',
+      implement: 'sha256:missing', finalize: 'sha256:missing', init: 'sha256:missing',
+      init_update: 'sha256:missing', quick_task: 'sha256:missing',
+    },
+    project: { name: '', started_at: null, completed_at: null },
+    step1_plan: { present: false, final_confidence: null, confidence_breakdown: null, clarification_rounds: null, tech_stack_revision_rounds: null, verification_gaps_found: null, functional_requirements_count: null, non_functional_requirements_count: null, risk_count: null, phase_count: null },
+    step2_document: { present: false, total_tasks: null, tasks_per_phase: null, phase_count: null, parallel_groups_identified: null, requirement_coverage_percent: null, verification_items_added: null },
+    step3_implement: { present: false, task_completion_rate: null, tasks_completed: null, tasks_total: null, blocker_count: null },
+    step4_finalize: { present: false, completion_rate_at_audit: null, verification_failures_found: null, documentation_updates_needed: null, archival_succeeded: null },
+    user_feedback: null,
+  };
+
+  it('writes a new run_id to an empty runsDir and returns true', () => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan2code-metrics-test-'));
+    const result = writeRunFile(RUN, tmpDir);
+    expect(result).toBe(true);
+    expect(fs.existsSync(path.join(tmpDir, `${RUN.run_id}.json`))).toBe(true);
+  });
+
+  it('returns false and does not overwrite when the same run_id already exists', () => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan2code-metrics-test-'));
+    writeRunFile(RUN, tmpDir);
+    const modified = { ...RUN, plan2code_version: '9.9.9' };
+    const result = writeRunFile(modified, tmpDir);
+    expect(result).toBe(false);
+    const onDisk = JSON.parse(fs.readFileSync(path.join(tmpDir, `${RUN.run_id}.json`), 'utf8'));
+    expect(onDisk.plan2code_version).toBe('1.15.3');
+  });
+});
+
+// ── ingestCommunityIssues() ───────────────────────────────────────────────────
+
+describe('ingestCommunityIssues', () => {
+  const goodBody = issueBody(VALID_PAYLOAD);
+  const badBody = 'an issue with no METRICS_JSON payload';
+
+  it('imports a new run and closes its issue', async () => {
+    const closed: number[] = [];
+    const tally = await ingestCommunityIssues(
+      [{ number: 1, body: goodBody }],
+      'owner/repo',
+      '/runs',
+      { writeRunFile: () => true, closeIssue: async (_r, n) => { closed.push(n); } },
+    );
+    expect(tally.imported).toBe(1);
+    expect(tally.skippedDuplicate).toBe(0);
+    expect(tally.closed).toBe(1);
+    expect(closed).toEqual([1]);
+  });
+
+  it('closes an already-imported (duplicate) issue instead of skipping the close', async () => {
+    const closed: number[] = [];
+    const tally = await ingestCommunityIssues(
+      [{ number: 7, body: goodBody }],
+      'owner/repo',
+      '/runs',
+      { writeRunFile: () => false, closeIssue: async (_r, n) => { closed.push(n); } },
+    );
+    expect(tally.imported).toBe(0);
+    expect(tally.skippedDuplicate).toBe(1);
+    expect(tally.closed).toBe(1); // the close-retry: duplicates are still closed
+    expect(closed).toEqual([7]);
+  });
+
+  it('records a close failure without throwing and leaves the issue for a later retry', async () => {
+    const tally = await ingestCommunityIssues(
+      [{ number: 9, body: goodBody }],
+      'owner/repo',
+      '/runs',
+      { writeRunFile: () => true, closeIssue: async () => { throw new Error('network'); } },
+    );
+    expect(tally.imported).toBe(1);
+    expect(tally.closed).toBe(0);
+    expect(tally.closeFailed).toBe(1);
+    expect(tally.closeFailedIssues).toEqual([9]);
+  });
+
+  it('skips and reports a malformed issue without writing or closing it', async () => {
+    let wrote = false;
+    let closeCalled = false;
+    const tally = await ingestCommunityIssues(
+      [{ number: 3, body: badBody }],
+      'owner/repo',
+      '/runs',
+      { writeRunFile: () => { wrote = true; return true; }, closeIssue: async () => { closeCalled = true; } },
+    );
+    expect(tally.skippedMalformed).toBe(1);
+    expect(tally.malformedIssues).toEqual([3]);
+    expect(wrote).toBe(false);
+    expect(closeCalled).toBe(false);
+  });
+});

plan2code-metrics/src/community.ts

@@ -0,0 +1,276 @@
+/**
+ * community.ts
+ * Ingestion side of the community feedback flow: list/parse/close
+ * `community-feedback`-labeled GitHub issues on jparkerweb/plan2code.
+ */
+
+import { execa } from 'execa';
+import type {
+  RunMetrics,
+  PromptVersions,
+  UserFeedback,
+  Step1PlanMetrics,
+  Step2DocumentMetrics,
+  Step3ImplementMetrics,
+  Step4FinalizeMetrics,
+} from './types.js';
+import { extractMetricsJson } from './collector.js';
+import { backfillPromptVersions } from './aggregator.js';
+
+// ── GitHub interaction (via gh CLI) ──────────────────────────────────────────
+
+export interface CommunityIssue {
+  number: number;
+  body: string;
+}
+
+interface RawIssue {
+  number: number;
+  title: string;
+  body: string;
+  labels: { name: string }[];
+}
+
+const COMMUNITY_LABEL = 'community-feedback';
+const FEEDBACK_TITLE_PREFIX = '[Feedback]';
+const METRICS_JSON_MARKER = /<!--\s*METRICS_JSON\s+\{/;
+
+export async function listCommunityIssues(repo: string): Promise<CommunityIssue[]> {
+  // Fetch open issues broadly rather than by label alone. Browser/print-tier
+  // submissions from outside contributors can lose the `community-feedback`
+  // label: GitHub only honors the `labels=` query param on issues/new for
+  // users with triage/push access, so the label is silently dropped for
+  // community members without `gh`. We therefore also match by the `[Feedback]`
+  // title prefix and the METRICS_JSON marker. Only OPEN issues are considered
+  // (closed/done submissions are already processed); parseSubmissionPayload is
+  // the final gate that rejects anything without a valid payload.
+  const result = await execa('gh', [
+    'issue', 'list',
+    '--repo', repo,
+    '--state', 'open',
+    '--limit', '1000',
+    '--json', 'number,title,body,labels',
+  ]);
+  const raw = JSON.parse(result.stdout) as RawIssue[];
+  return raw
+    .filter((issue) =>
+      issue.labels.some((l) => l.name === COMMUNITY_LABEL) ||
+      issue.title.startsWith(FEEDBACK_TITLE_PREFIX) ||
+      METRICS_JSON_MARKER.test(issue.body)
+    )
+    .map((issue) => ({ number: issue.number, body: issue.body }));
+}
+
+export async function closeIssue(repo: string, issueNumber: number): Promise<void> {
+  await execa('gh', ['issue', 'close', String(issueNumber), '--repo', repo]);
+}
+
+// ── Ingestion control flow (I/O injected so it is unit-testable) ──────────────
+
+export interface IngestionDeps {
+  writeRunFile: (run: RunMetrics, runsDir: string) => boolean;
+  closeIssue: (repo: string, issueNumber: number) => Promise<void>;
+}
+
+export interface IngestionTally {
+  imported: number;
+  skippedDuplicate: number;
+  skippedMalformed: number;
+  closed: number;
+  closeFailed: number;
+  malformedIssues: number[];
+  closeFailedIssues: number[];
+}
+
+/**
+ * Process a batch of community issues: parse each payload, write new runs
+ * (deduped by run_id), and close every open issue idempotently.
+ *
+ * The close is attempted on the duplicate path too: a submission that imported
+ * on an earlier run but failed to close would otherwise be seen as a duplicate
+ * forever and never closed again, leaving the issue open and reprocessed on
+ * every fetch. Malformed issues are reported (not fixed up) and left open.
+ *
+ * I/O (writeRunFile/closeIssue) is injected so the control flow can be unit
+ * tested without a live `gh`. Returns a tally; the caller owns all logging.
+ */
+export async function ingestCommunityIssues(
+  issues: CommunityIssue[],
+  repo: string,
+  runsDir: string,
+  deps: IngestionDeps,
+): Promise<IngestionTally> {
+  const tally: IngestionTally = {
+    imported: 0, skippedDuplicate: 0, skippedMalformed: 0,
+    closed: 0, closeFailed: 0, malformedIssues: [], closeFailedIssues: [],
+  };
+
+  for (const issue of issues) {
+    const run = parseSubmissionPayload(issue.body);
+    if (!run) {
+      tally.skippedMalformed++;
+      tally.malformedIssues.push(issue.number);
+      continue;
+    }
+
+    if (deps.writeRunFile(run, runsDir)) {
+      tally.imported++;
+    } else {
+      tally.skippedDuplicate++;
+    }
+
+    try {
+      await deps.closeIssue(repo, issue.number);
+      tally.closed++;
+    } catch {
+      tally.closeFailed++;
+      tally.closeFailedIssues.push(issue.number);
+    }
+  }
+
+  return tally;
+}
+
+// ── Payload parsing (type-only validation, per NFR-5) ────────────────────────
+
+function isString(v: unknown): v is string {
+  return typeof v === 'string';
+}
+
+function isNumber(v: unknown): v is number {
+  return typeof v === 'number';
+}
+
+function numOrNull(v: unknown): number | null {
+  return isNumber(v) ? v : null;
+}
+
+/** Type-check `keys` off `raw` (object or not) into a { [key]: number | null } map. */
+function pickNumbers<K extends string>(raw: Record<string, unknown>, keys: readonly K[]): Record<K, number | null> {
+  const result = {} as Record<K, number | null>;
+  for (const key of keys) result[key] = numOrNull(raw[key]);
+  return result;
+}
+
+const STEP1_ABSENT: Step1PlanMetrics = {
+  present: false, final_confidence: null, confidence_breakdown: null,
+  clarification_rounds: null, tech_stack_revision_rounds: null,
+  verification_gaps_found: null, functional_requirements_count: null,
+  non_functional_requirements_count: null, risk_count: null, phase_count: null,
+};
+
+function parseStep1(raw: unknown): Step1PlanMetrics {
+  if (raw == null || typeof raw !== 'object') return STEP1_ABSENT;
+  const step1 = raw as Record<string, unknown>;
+  const bdRaw = step1['confidence_breakdown'];
+  const breakdown = bdRaw != null && typeof bdRaw === 'object'
+    ? pickNumbers(bdRaw as Record<string, unknown>, ['requirements', 'feasibility', 'integration', 'risk'])
+    : null;
+  return {
+    present: true,
+    confidence_breakdown: breakdown,
+    ...pickNumbers(step1, [
+      'final_confidence', 'clarification_rounds', 'tech_stack_revision_rounds',
+      'verification_gaps_found', 'functional_requirements_count',
+      'non_functional_requirements_count', 'risk_count', 'phase_count',
+    ]),
+  };
+}
+
+const STEP2_ABSENT: Step2DocumentMetrics = {
+  present: false, total_tasks: null, tasks_per_phase: null,
+  phase_count: null, parallel_groups_identified: null,
+  requirement_coverage_percent: null, verification_items_added: null,
+};
+
+function parseStep2(raw: unknown): Step2DocumentMetrics {
+  if (raw == null || typeof raw !== 'object') return STEP2_ABSENT;
+  const step2 = raw as Record<string, unknown>;
+  return {
+    present: true,
+    tasks_per_phase: null,
+    ...pickNumbers(step2, [
+      'total_tasks', 'phase_count', 'parallel_groups_identified',
+      'requirement_coverage_percent', 'verification_items_added',
+    ]),
+  };
+}
+
+const STEP3_ABSENT: Step3ImplementMetrics = {
+  present: false, task_completion_rate: null, tasks_completed: null, tasks_total: null, blocker_count: null,
+};
+
+function parseStep3(raw: unknown): Step3ImplementMetrics {
+  if (raw == null || typeof raw !== 'object') return STEP3_ABSENT;
+  const step3 = raw as Record<string, unknown>;
+  return {
+    present: true,
+    ...pickNumbers(step3, ['task_completion_rate', 'tasks_completed', 'tasks_total', 'blocker_count']),
+  };
+}
+
+const STEP4_ABSENT: Step4FinalizeMetrics = {
+  present: false, completion_rate_at_audit: null, verification_failures_found: null,
+  documentation_updates_needed: null, archival_succeeded: null,
+};
+
+function parseStep4(raw: unknown): Step4FinalizeMetrics {
+  if (raw == null || typeof raw !== 'object') return STEP4_ABSENT;
+  const step4 = raw as Record<string, unknown>;
+  const archivalRaw = step4['archival_succeeded'];
+  return {
+    present: true,
+    archival_succeeded: typeof archivalRaw === 'boolean' ? archivalRaw : null,
+    ...pickNumbers(step4, ['completion_rate_at_audit', 'verification_failures_found', 'documentation_updates_needed']),
+  };
+}
+
+function parsePromptVersionsShort(raw: unknown): PromptVersions {
+  const partial: Partial<PromptVersions> = {};
+  if (raw != null && typeof raw === 'object') {
+    const pv = raw as Record<string, unknown>;
+    for (const key of ['plan', 'revise_plan', 'document', 'implement', 'finalize', 'init', 'init_update', 'quick_task'] as const) {
+      const v = pv[key];
+      if (isString(v)) partial[key] = v;
+    }
+  }
+  return backfillPromptVersions(partial as PromptVersions);
+}
+
+export function parseSubmissionPayload(body: string): RunMetrics | null {
+  const parsed = extractMetricsJson(body);
+  if (!parsed) return null;
+
+  if (parsed['schema_version'] !== '1.0') return null;
+  if (!isString(parsed['run_id'])) return null;
+  if (!isString(parsed['plan2code_version'])) return null;
+
+  const feedbackRaw = parsed['user_feedback'];
+  if (feedbackRaw == null || typeof feedbackRaw !== 'object') return null;
+  const feedback = feedbackRaw as Record<string, unknown>;
+  if (!isNumber(feedback['overall_rating'])) return null;
+  if (!isString(feedback['rating_reason'])) return null;
+  if (!isString(feedback['what_went_well'])) return null;
+  if (!isString(feedback['what_went_poorly'])) return null;
+
+  const userFeedback: UserFeedback = {
+    overall_rating: feedback['overall_rating'],
+    rating_reason: feedback['rating_reason'],
+    what_went_well: feedback['what_went_well'],
+    what_went_poorly: feedback['what_went_poorly'],
+  };
+
+  return {
+    schema_version: '1.0',
+    run_id: parsed['run_id'],
+    plan2code_version: parsed['plan2code_version'],
+    source: 'community',
+    prompt_versions: parsePromptVersionsShort(parsed['prompt_versions_short']),
+    project: { name: '', started_at: null, completed_at: null },
+    step1_plan: parseStep1(parsed['step1']),
+    step2_document: parseStep2(parsed['step2']),
+    step3_implement: parseStep3(parsed['step3']),
+    step4_finalize: parseStep4(parsed['step4']),
+    user_feedback: userFeedback,
+  };
+}

plan2code-metrics/src/invoke-llm.ts

@@ -1,7 +1,8 @@
 /**
  * invoke-llm.ts
  * Unified LLM invocation for plan2code-metrics.
- * Supports Claude Code (temp file → stdin) and Copilot CLI (stdin string).
+ * Supports Claude Code (temp file → stdin), Copilot CLI (stdin string), and
+ * Devin CLI (temp prompt file).
  * Mirrors the agent pattern from plan2code-loop.
  */
 
@@ -12,7 +13,7 @@ import { tmpdir } from 'os';
 
 // ── Agent definitions ────────────────────────────────────────────────────────
 
-export type AgentType = 'claude-code' | 'copilot-cli';
+export type AgentType = 'claude-code' | 'copilot-cli' | 'devin-cli';
 
 export interface AgentDef {
   name: AgentType;
@@ -34,6 +35,12 @@ export const AGENTS: Record<AgentType, AgentDef> = {
     command: 'copilot',
     defaultModel: 'claude-sonnet-4',
   },
+  'devin-cli': {
+    name: 'devin-cli',
+    displayName: 'Devin CLI',
+    command: 'devin',
+    defaultModel: 'default',
+  },
 };
 
 // ── Invocation ───────────────────────────────────────────────────────────────
@@ -72,6 +79,23 @@ export async function invokeLLM(opts: InvokeLLMOptions): Promise<string> {
     } finally {
       try { unlinkSync(tempFile); } catch { /* ignore cleanup errors */ }
     }
+  } else if (agent === 'devin-cli') {
+    // Devin CLI: load prompt from a temp file, run single-turn, auto-approve tool calls
+    const tempFile = join(tmpdir(), `plan2code-metrics-prompt-${Date.now()}.txt`);
+    writeFileSync(tempFile, prompt, 'utf-8');
+
+    try {
+      const args: string[] = ['--print', '--prompt-file', tempFile, '--permission-mode', 'dangerous'];
+      // Only add --model if not using default
+      if (model && model !== 'default') {
+        args.push('--model', model);
+      }
+
+      const result = await execa(def.command, args, { timeout });
+      return result.stdout;
+    } finally {
+      try { unlinkSync(tempFile); } catch { /* ignore cleanup errors */ }
+    }
   } else {
     // Copilot CLI: pipe prompt via stdin string
     const args: string[] = [];

plan2code-metrics/src/types.ts

@@ -66,6 +66,11 @@ export interface RunMetrics {
   schema_version: '1.0';
   run_id: string;
   plan2code_version: string;
+  // Origin of the run. Local runs are collected on the maintainer's machine;
+  // community runs are ingested from GitHub feedback issues. Absent on pre-v1.17
+  // run files, which are treated as 'local'. Drives cohort keying (see
+  // cohortKeyForRun in aggregator.ts).
+  source?: 'local' | 'community';
   prompt_versions: PromptVersions;
   project: {
     name: string;
@@ -102,7 +107,8 @@ export interface PromptProposal {
 
 // Aggregated metrics schema
 export interface CohortMetrics {
-  cohort_key: string;          // hash of sorted prompt_versions
+  cohort_key: string;          // local: hash of sorted prompt_versions; community: `community:v<version>`
+  source?: 'local' | 'community';
   prompt_versions: PromptVersions;
   run_count: number;
   run_ids: string[];