Files
plan2code/plan2code-metrics/src/community.test.ts
T

243 lines
9.8 KiB
TypeScript
Raw Normal View History

import fs from 'fs';
import os from 'os';
import path from 'path';
import { describe, it, expect, afterEach } from 'vitest';
import { parseSubmissionPayload, ingestCommunityIssues } from './community.js';
import { writeRunFile } from './aggregator.js';
import type { RunMetrics } from './types.js';
function issueBody(payload: Record<string, unknown>): string {
return `Some issue text.\n\n<!-- METRICS_JSON ${JSON.stringify(payload)} -->\n`;
}
const VALID_PAYLOAD = {
schema_version: '1.0',
run_id: 'run-20260715-143000-a1b2',
plan2code_version: '1.15.3',
prompt_versions_short: {
plan: 'abc123def456', revise_plan: 'a', document: 'b', implement: 'c',
finalize: 'd', init: 'e', init_update: 'f', quick_task: 'g',
},
step1: {
final_confidence: 95,
confidence_breakdown: { requirements: 24, feasibility: 23, integration: 24, risk: 22 },
clarification_rounds: 0,
tech_stack_revision_rounds: 0,
verification_gaps_found: 0,
functional_requirements_count: 8,
non_functional_requirements_count: 6,
risk_count: 7,
phase_count: 4,
},
step2: {
total_tasks: 28,
phase_count: 4,
parallel_groups_identified: 1,
requirement_coverage_percent: 100,
verification_items_added: 3,
},
step3: {
task_completion_rate: 0.96,
tasks_completed: 27,
tasks_total: 28,
blocker_count: 1,
},
step4: {
completion_rate_at_audit: 0.96,
verification_failures_found: 1,
documentation_updates_needed: 2,
},
user_feedback: {
overall_rating: 8,
rating_reason: 'good stuff',
what_went_well: 'well',
what_went_poorly: 'poorly',
},
};
// ── parseSubmissionPayload() ─────────────────────────────────────────────────
describe('parseSubmissionPayload', () => {
it('parses a fully valid payload into a correctly-shaped RunMetrics', () => {
const result = parseSubmissionPayload(issueBody(VALID_PAYLOAD));
expect(result).not.toBeNull();
expect(result!.run_id).toBe('run-20260715-143000-a1b2');
expect(result!.schema_version).toBe('1.0');
expect(result!.plan2code_version).toBe('1.15.3');
expect(result!.source).toBe('community');
expect(result!.step1_plan.present).toBe(true);
expect(result!.step1_plan.final_confidence).toBe(95);
expect(result!.step2_document.present).toBe(true);
expect(result!.step2_document.total_tasks).toBe(28);
expect(result!.step3_implement.present).toBe(true);
expect(result!.step3_implement.tasks_completed).toBe(27);
expect(result!.step4_finalize.present).toBe(true);
expect(result!.step4_finalize.completion_rate_at_audit).toBe(0.96);
expect(result!.user_feedback).toEqual({
overall_rating: 8,
rating_reason: 'good stuff',
what_went_well: 'well',
what_went_poorly: 'poorly',
});
});
it('returns null when run_id is missing', () => {
const { run_id, ...withoutRunId } = VALID_PAYLOAD;
const result = parseSubmissionPayload(issueBody(withoutRunId));
expect(result).toBeNull();
});
it('returns null when user_feedback.overall_rating is a string instead of a number', () => {
const badPayload = {
...VALID_PAYLOAD,
user_feedback: { ...VALID_PAYLOAD.user_feedback, overall_rating: 'eight' },
};
const result = parseSubmissionPayload(issueBody(badPayload));
expect(result).toBeNull();
});
it('returns null when schema_version is not exactly "1.0"', () => {
const badPayload = { ...VALID_PAYLOAD, schema_version: '2.0' };
const result = parseSubmissionPayload(issueBody(badPayload));
expect(result).toBeNull();
});
it('sets all four steps present:false when only user_feedback is included', () => {
const minimalPayload = {
schema_version: '1.0',
run_id: 'run-20260715-150000-c3d4',
plan2code_version: '1.15.3',
user_feedback: VALID_PAYLOAD.user_feedback,
};
const result = parseSubmissionPayload(issueBody(minimalPayload));
expect(result).not.toBeNull();
expect(result!.step1_plan.present).toBe(false);
expect(result!.step2_document.present).toBe(false);
expect(result!.step3_implement.present).toBe(false);
expect(result!.step4_finalize.present).toBe(false);
});
it('backfills missing prompt_versions_short keys with the sha256:missing sentinel', () => {
const partialPayload = {
...VALID_PAYLOAD,
prompt_versions_short: { plan: 'abc123def456' },
};
const result = parseSubmissionPayload(issueBody(partialPayload));
expect(result).not.toBeNull();
expect(result!.prompt_versions.plan).toBe('abc123def456');
expect(result!.prompt_versions.revise_plan).toBe('sha256:missing');
expect(result!.prompt_versions.document).toBe('sha256:missing');
expect(result!.prompt_versions.implement).toBe('sha256:missing');
expect(result!.prompt_versions.finalize).toBe('sha256:missing');
expect(result!.prompt_versions.init).toBe('sha256:missing');
expect(result!.prompt_versions.init_update).toBe('sha256:missing');
expect(result!.prompt_versions.quick_task).toBe('sha256:missing');
});
});
// ── writeRunFile() ────────────────────────────────────────────────────────────
describe('writeRunFile', () => {
let tmpDir: string;
afterEach(() => {
if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true });
});
const RUN: RunMetrics = {
schema_version: '1.0',
run_id: 'run-20260715-160000-e5f6',
plan2code_version: '1.15.3',
prompt_versions: {
plan: 'sha256:missing', revise_plan: 'sha256:missing', document: 'sha256:missing',
implement: 'sha256:missing', finalize: 'sha256:missing', init: 'sha256:missing',
init_update: 'sha256:missing', quick_task: 'sha256:missing',
},
project: { name: '', started_at: null, completed_at: null },
step1_plan: { present: false, final_confidence: null, confidence_breakdown: null, clarification_rounds: null, tech_stack_revision_rounds: null, verification_gaps_found: null, functional_requirements_count: null, non_functional_requirements_count: null, risk_count: null, phase_count: null },
step2_document: { present: false, total_tasks: null, tasks_per_phase: null, phase_count: null, parallel_groups_identified: null, requirement_coverage_percent: null, verification_items_added: null },
step3_implement: { present: false, task_completion_rate: null, tasks_completed: null, tasks_total: null, blocker_count: null },
step4_finalize: { present: false, completion_rate_at_audit: null, verification_failures_found: null, documentation_updates_needed: null, archival_succeeded: null },
user_feedback: null,
};
it('writes a new run_id to an empty runsDir and returns true', () => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan2code-metrics-test-'));
const result = writeRunFile(RUN, tmpDir);
expect(result).toBe(true);
expect(fs.existsSync(path.join(tmpDir, `${RUN.run_id}.json`))).toBe(true);
});
it('returns false and does not overwrite when the same run_id already exists', () => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan2code-metrics-test-'));
writeRunFile(RUN, tmpDir);
const modified = { ...RUN, plan2code_version: '9.9.9' };
const result = writeRunFile(modified, tmpDir);
expect(result).toBe(false);
const onDisk = JSON.parse(fs.readFileSync(path.join(tmpDir, `${RUN.run_id}.json`), 'utf8'));
expect(onDisk.plan2code_version).toBe('1.15.3');
});
});
// ── ingestCommunityIssues() ───────────────────────────────────────────────────
describe('ingestCommunityIssues', () => {
const goodBody = issueBody(VALID_PAYLOAD);
const badBody = 'an issue with no METRICS_JSON payload';
it('imports a new run and closes its issue', async () => {
const closed: number[] = [];
const tally = await ingestCommunityIssues(
[{ number: 1, body: goodBody }],
'owner/repo',
'/runs',
{ writeRunFile: () => true, closeIssue: async (_r, n) => { closed.push(n); } },
);
expect(tally.imported).toBe(1);
expect(tally.skippedDuplicate).toBe(0);
expect(tally.closed).toBe(1);
expect(closed).toEqual([1]);
});
it('closes an already-imported (duplicate) issue instead of skipping the close', async () => {
const closed: number[] = [];
const tally = await ingestCommunityIssues(
[{ number: 7, body: goodBody }],
'owner/repo',
'/runs',
{ writeRunFile: () => false, closeIssue: async (_r, n) => { closed.push(n); } },
);
expect(tally.imported).toBe(0);
expect(tally.skippedDuplicate).toBe(1);
expect(tally.closed).toBe(1); // the close-retry: duplicates are still closed
expect(closed).toEqual([7]);
});
it('records a close failure without throwing and leaves the issue for a later retry', async () => {
const tally = await ingestCommunityIssues(
[{ number: 9, body: goodBody }],
'owner/repo',
'/runs',
{ writeRunFile: () => true, closeIssue: async () => { throw new Error('network'); } },
);
expect(tally.imported).toBe(1);
expect(tally.closed).toBe(0);
expect(tally.closeFailed).toBe(1);
expect(tally.closeFailedIssues).toEqual([9]);
});
it('skips and reports a malformed issue without writing or closing it', async () => {
let wrote = false;
let closeCalled = false;
const tally = await ingestCommunityIssues(
[{ number: 3, body: badBody }],
'owner/repo',
'/runs',
{ writeRunFile: () => { wrote = true; return true; }, closeIssue: async () => { closeCalled = true; } },
);
expect(tally.skippedMalformed).toBe(1);
expect(tally.malformedIssues).toEqual([3]);
expect(wrote).toBe(false);
expect(closeCalled).toBe(false);
});
});