mirror of
https://github.com/jparkerweb/plan2code.git
synced 2026-11-17 04:45:33 -08:00
276 lines
9.6 KiB
TypeScript
276 lines
9.6 KiB
TypeScript
/**
|
|||
|
|
* community.ts
|
||
|
|
* Ingestion side of the community feedback flow: list/parse/close
|
||
|
|
* `community-feedback`-labeled GitHub issues on jparkerweb/plan2code.
|
||
|
|
*/
|
||
|
|
|
||
|
|
import { execa } from 'execa';
|
||
|
|
import type {
|
||
|
|
RunMetrics,
|
||
|
|
PromptVersions,
|
||
|
|
UserFeedback,
|
||
|
|
Step1PlanMetrics,
|
||
|
|
Step2DocumentMetrics,
|
||
|
|
Step3ImplementMetrics,
|
||
|
|
Step4FinalizeMetrics,
|
||
|
|
} from './types.js';
|
||
|
|
import { extractMetricsJson } from './collector.js';
|
||
|
|
import { backfillPromptVersions } from './aggregator.js';
|
||
|
|
|
||
|
|
// ── GitHub interaction (via gh CLI) ──────────────────────────────────────────
|
||
|
|
|
||
|
|
export interface CommunityIssue {
|
||
|
|
number: number;
|
||
|
|
body: string;
|
||
|
|
}
|
||
|
|
|
||
|
|
interface RawIssue {
|
||
|
|
number: number;
|
||
|
|
title: string;
|
||
|
|
body: string;
|
||
|
|
labels: { name: string }[];
|
||
|
|
}
|
||
|
|
|
||
|
|
const COMMUNITY_LABEL = 'community-feedback';
|
||
|
|
const FEEDBACK_TITLE_PREFIX = '[Feedback]';
|
||
|
|
const METRICS_JSON_MARKER = /<!--\s*METRICS_JSON\s+\{/;
|
||
|
|
|
||
|
|
export async function listCommunityIssues(repo: string): Promise<CommunityIssue[]> {
|
||
|
|
// Fetch open issues broadly rather than by label alone. Browser/print-tier
|
||
|
|
// submissions from outside contributors can lose the `community-feedback`
|
||
|
|
// label: GitHub only honors the `labels=` query param on issues/new for
|
||
|
|
// users with triage/push access, so the label is silently dropped for
|
||
|
|
// community members without `gh`. We therefore also match by the `[Feedback]`
|
||
|
|
// title prefix and the METRICS_JSON marker. Only OPEN issues are considered
|
||
|
|
// (closed/done submissions are already processed); parseSubmissionPayload is
|
||
|
|
// the final gate that rejects anything without a valid payload.
|
||
|
|
const result = await execa('gh', [
|
||
|
|
'issue', 'list',
|
||
|
|
'--repo', repo,
|
||
|
|
'--state', 'open',
|
||
|
|
'--limit', '1000',
|
||
|
|
'--json', 'number,title,body,labels',
|
||
|
|
]);
|
||
|
|
const raw = JSON.parse(result.stdout) as RawIssue[];
|
||
|
|
return raw
|
||
|
|
.filter((issue) =>
|
||
|
|
issue.labels.some((l) => l.name === COMMUNITY_LABEL) ||
|
||
|
|
issue.title.startsWith(FEEDBACK_TITLE_PREFIX) ||
|
||
|
|
METRICS_JSON_MARKER.test(issue.body)
|
||
|
|
)
|
||
|
|
.map((issue) => ({ number: issue.number, body: issue.body }));
|
||
|
|
}
|
||
|
|
|
||
|
|
export async function closeIssue(repo: string, issueNumber: number): Promise<void> {
|
||
|
|
await execa('gh', ['issue', 'close', String(issueNumber), '--repo', repo]);
|
||
|
|
}
|
||
|
|
|
||
|
|
// ── Ingestion control flow (I/O injected so it is unit-testable) ──────────────
|
||
|
|
|
||
|
|
export interface IngestionDeps {
|
||
|
|
writeRunFile: (run: RunMetrics, runsDir: string) => boolean;
|
||
|
|
closeIssue: (repo: string, issueNumber: number) => Promise<void>;
|
||
|
|
}
|
||
|
|
|
||
|
|
export interface IngestionTally {
|
||
|
|
imported: number;
|
||
|
|
skippedDuplicate: number;
|
||
|
|
skippedMalformed: number;
|
||
|
|
closed: number;
|
||
|
|
closeFailed: number;
|
||
|
|
malformedIssues: number[];
|
||
|
|
closeFailedIssues: number[];
|
||
|
|
}
|
||
|
|
|
||
|
|
/**
|
||
|
|
* Process a batch of community issues: parse each payload, write new runs
|
||
|
|
* (deduped by run_id), and close every open issue idempotently.
|
||
|
|
*
|
||
|
|
* The close is attempted on the duplicate path too: a submission that imported
|
||
|
|
* on an earlier run but failed to close would otherwise be seen as a duplicate
|
||
|
|
* forever and never closed again, leaving the issue open and reprocessed on
|
||
|
|
* every fetch. Malformed issues are reported (not fixed up) and left open.
|
||
|
|
*
|
||
|
|
* I/O (writeRunFile/closeIssue) is injected so the control flow can be unit
|
||
|
|
* tested without a live `gh`. Returns a tally; the caller owns all logging.
|
||
|
|
*/
|
||
|
|
export async function ingestCommunityIssues(
|
||
|
|
issues: CommunityIssue[],
|
||
|
|
repo: string,
|
||
|
|
runsDir: string,
|
||
|
|
deps: IngestionDeps,
|
||
|
|
): Promise<IngestionTally> {
|
||
|
|
const tally: IngestionTally = {
|
||
|
|
imported: 0, skippedDuplicate: 0, skippedMalformed: 0,
|
||
|
|
closed: 0, closeFailed: 0, malformedIssues: [], closeFailedIssues: [],
|
||
|
|
};
|
||
|
|
|
||
|
|
for (const issue of issues) {
|
||
|
|
const run = parseSubmissionPayload(issue.body);
|
||
|
|
if (!run) {
|
||
|
|
tally.skippedMalformed++;
|
||
|
|
tally.malformedIssues.push(issue.number);
|
||
|
|
continue;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (deps.writeRunFile(run, runsDir)) {
|
||
|
|
tally.imported++;
|
||
|
|
} else {
|
||
|
|
tally.skippedDuplicate++;
|
||
|
|
}
|
||
|
|
|
||
|
|
try {
|
||
|
|
await deps.closeIssue(repo, issue.number);
|
||
|
|
tally.closed++;
|
||
|
|
} catch {
|
||
|
|
tally.closeFailed++;
|
||
|
|
tally.closeFailedIssues.push(issue.number);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return tally;
|
||
|
|
}
|
||
|
|
|
||
|
|
// ── Payload parsing (type-only validation, per NFR-5) ────────────────────────
|
||
|
|
|
||
|
|
function isString(v: unknown): v is string {
|
||
|
|
return typeof v === 'string';
|
||
|
|
}
|
||
|
|
|
||
|
|
function isNumber(v: unknown): v is number {
|
||
|
|
return typeof v === 'number';
|
||
|
|
}
|
||
|
|
|
||
|
|
function numOrNull(v: unknown): number | null {
|
||
|
|
return isNumber(v) ? v : null;
|
||
|
|
}
|
||
|
|
|
||
|
|
/** Type-check `keys` off `raw` (object or not) into a { [key]: number | null } map. */
|
||
|
|
function pickNumbers<K extends string>(raw: Record<string, unknown>, keys: readonly K[]): Record<K, number | null> {
|
||
|
|
const result = {} as Record<K, number | null>;
|
||
|
|
for (const key of keys) result[key] = numOrNull(raw[key]);
|
||
|
|
return result;
|
||
|
|
}
|
||
|
|
|
||
|
|
const STEP1_ABSENT: Step1PlanMetrics = {
|
||
|
|
present: false, final_confidence: null, confidence_breakdown: null,
|
||
|
|
clarification_rounds: null, tech_stack_revision_rounds: null,
|
||
|
|
verification_gaps_found: null, functional_requirements_count: null,
|
||
|
|
non_functional_requirements_count: null, risk_count: null, phase_count: null,
|
||
|
|
};
|
||
|
|
|
||
|
|
function parseStep1(raw: unknown): Step1PlanMetrics {
|
||
|
|
if (raw == null || typeof raw !== 'object') return STEP1_ABSENT;
|
||
|
|
const step1 = raw as Record<string, unknown>;
|
||
|
|
const bdRaw = step1['confidence_breakdown'];
|
||
|
|
const breakdown = bdRaw != null && typeof bdRaw === 'object'
|
||
|
|
? pickNumbers(bdRaw as Record<string, unknown>, ['requirements', 'feasibility', 'integration', 'risk'])
|
||
|
|
: null;
|
||
|
|
return {
|
||
|
|
present: true,
|
||
|
|
confidence_breakdown: breakdown,
|
||
|
|
...pickNumbers(step1, [
|
||
|
|
'final_confidence', 'clarification_rounds', 'tech_stack_revision_rounds',
|
||
|
|
'verification_gaps_found', 'functional_requirements_count',
|
||
|
|
'non_functional_requirements_count', 'risk_count', 'phase_count',
|
||
|
|
]),
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
const STEP2_ABSENT: Step2DocumentMetrics = {
|
||
|
|
present: false, total_tasks: null, tasks_per_phase: null,
|
||
|
|
phase_count: null, parallel_groups_identified: null,
|
||
|
|
requirement_coverage_percent: null, verification_items_added: null,
|
||
|
|
};
|
||
|
|
|
||
|
|
function parseStep2(raw: unknown): Step2DocumentMetrics {
|
||
|
|
if (raw == null || typeof raw !== 'object') return STEP2_ABSENT;
|
||
|
|
const step2 = raw as Record<string, unknown>;
|
||
|
|
return {
|
||
|
|
present: true,
|
||
|
|
tasks_per_phase: null,
|
||
|
|
...pickNumbers(step2, [
|
||
|
|
'total_tasks', 'phase_count', 'parallel_groups_identified',
|
||
|
|
'requirement_coverage_percent', 'verification_items_added',
|
||
|
|
]),
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
const STEP3_ABSENT: Step3ImplementMetrics = {
|
||
|
|
present: false, task_completion_rate: null, tasks_completed: null, tasks_total: null, blocker_count: null,
|
||
|
|
};
|
||
|
|
|
||
|
|
function parseStep3(raw: unknown): Step3ImplementMetrics {
|
||
|
|
if (raw == null || typeof raw !== 'object') return STEP3_ABSENT;
|
||
|
|
const step3 = raw as Record<string, unknown>;
|
||
|
|
return {
|
||
|
|
present: true,
|
||
|
|
...pickNumbers(step3, ['task_completion_rate', 'tasks_completed', 'tasks_total', 'blocker_count']),
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
const STEP4_ABSENT: Step4FinalizeMetrics = {
|
||
|
|
present: false, completion_rate_at_audit: null, verification_failures_found: null,
|
||
|
|
documentation_updates_needed: null, archival_succeeded: null,
|
||
|
|
};
|
||
|
|
|
||
|
|
function parseStep4(raw: unknown): Step4FinalizeMetrics {
|
||
|
|
if (raw == null || typeof raw !== 'object') return STEP4_ABSENT;
|
||
|
|
const step4 = raw as Record<string, unknown>;
|
||
|
|
const archivalRaw = step4['archival_succeeded'];
|
||
|
|
return {
|
||
|
|
present: true,
|
||
|
|
archival_succeeded: typeof archivalRaw === 'boolean' ? archivalRaw : null,
|
||
|
|
...pickNumbers(step4, ['completion_rate_at_audit', 'verification_failures_found', 'documentation_updates_needed']),
|
||
|
|
};
|
||
|
|
}
|
||
|
|
|
||
|
|
function parsePromptVersionsShort(raw: unknown): PromptVersions {
|
||
|
|
const partial: Partial<PromptVersions> = {};
|
||
|
|
if (raw != null && typeof raw === 'object') {
|
||
|
|
const pv = raw as Record<string, unknown>;
|
||
|
|
for (const key of ['plan', 'revise_plan', 'document', 'implement', 'finalize', 'init', 'init_update', 'quick_task'] as const) {
|
||
|
|
const v = pv[key];
|
||
|
|
if (isString(v)) partial[key] = v;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return backfillPromptVersions(partial as PromptVersions);
|
||
|
|
}
|
||
|
|
|
||
|
|
export function parseSubmissionPayload(body: string): RunMetrics | null {
|
||
|
|
const parsed = extractMetricsJson(body);
|
||
|
|
if (!parsed) return null;
|
||
|
|
|
||
|
|
if (parsed['schema_version'] !== '1.0') return null;
|
||
|
|
if (!isString(parsed['run_id'])) return null;
|
||
|
|
if (!isString(parsed['plan2code_version'])) return null;
|
||
|
|
|
||
|
|
const feedbackRaw = parsed['user_feedback'];
|
||
|
|
if (feedbackRaw == null || typeof feedbackRaw !== 'object') return null;
|
||
|
|
const feedback = feedbackRaw as Record<string, unknown>;
|
||
|
|
if (!isNumber(feedback['overall_rating'])) return null;
|
||
|
|
if (!isString(feedback['rating_reason'])) return null;
|
||
|
|
if (!isString(feedback['what_went_well'])) return null;
|
||
|
|
if (!isString(feedback['what_went_poorly'])) return null;
|
||
|
|
|
||
|
|
const userFeedback: UserFeedback = {
|
||
|
|
overall_rating: feedback['overall_rating'],
|
||
|
|
rating_reason: feedback['rating_reason'],
|
||
|
|
what_went_well: feedback['what_went_well'],
|
||
|
|
what_went_poorly: feedback['what_went_poorly'],
|
||
|
|
};
|
||
|
|
|
||
|
|
return {
|
||
|
|
schema_version: '1.0',
|
||
|
|
run_id: parsed['run_id'],
|
||
|
|
plan2code_version: parsed['plan2code_version'],
|
||
|
|
source: 'community',
|
||
|
|
prompt_versions: parsePromptVersionsShort(parsed['prompt_versions_short']),
|
||
|
|
project: { name: '', started_at: null, completed_at: null },
|
||
|
|
step1_plan: parseStep1(parsed['step1']),
|
||
|
|
step2_document: parseStep2(parsed['step2']),
|
||
|
|
step3_implement: parseStep3(parsed['step3']),
|
||
|
|
step4_finalize: parseStep4(parsed['step4']),
|
||
|
|
user_feedback: userFeedback,
|
||
|
|
};
|
||
|
|
}
|