mirror of
https://github.com/jparkerweb/plan2code.git
synced 2026-11-16 20:45:33 -08:00
243 lines
9.8 KiB
TypeScript
243 lines
9.8 KiB
TypeScript
import fs from 'fs';
|
|||
|
|
import os from 'os';
|
||
|
|
import path from 'path';
|
||
|
|
import { describe, it, expect, afterEach } from 'vitest';
|
||
|
|
import { parseSubmissionPayload, ingestCommunityIssues } from './community.js';
|
||
|
|
import { writeRunFile } from './aggregator.js';
|
||
|
|
import type { RunMetrics } from './types.js';
|
||
|
|
|
||
|
|
function issueBody(payload: Record<string, unknown>): string {
|
||
|
|
return `Some issue text.\n\n<!-- METRICS_JSON ${JSON.stringify(payload)} -->\n`;
|
||
|
|
}
|
||
|
|
|
||
|
|
const VALID_PAYLOAD = {
|
||
|
|
schema_version: '1.0',
|
||
|
|
run_id: 'run-20260715-143000-a1b2',
|
||
|
|
plan2code_version: '1.15.3',
|
||
|
|
prompt_versions_short: {
|
||
|
|
plan: 'abc123def456', revise_plan: 'a', document: 'b', implement: 'c',
|
||
|
|
finalize: 'd', init: 'e', init_update: 'f', quick_task: 'g',
|
||
|
|
},
|
||
|
|
step1: {
|
||
|
|
final_confidence: 95,
|
||
|
|
confidence_breakdown: { requirements: 24, feasibility: 23, integration: 24, risk: 22 },
|
||
|
|
clarification_rounds: 0,
|
||
|
|
tech_stack_revision_rounds: 0,
|
||
|
|
verification_gaps_found: 0,
|
||
|
|
functional_requirements_count: 8,
|
||
|
|
non_functional_requirements_count: 6,
|
||
|
|
risk_count: 7,
|
||
|
|
phase_count: 4,
|
||
|
|
},
|
||
|
|
step2: {
|
||
|
|
total_tasks: 28,
|
||
|
|
phase_count: 4,
|
||
|
|
parallel_groups_identified: 1,
|
||
|
|
requirement_coverage_percent: 100,
|
||
|
|
verification_items_added: 3,
|
||
|
|
},
|
||
|
|
step3: {
|
||
|
|
task_completion_rate: 0.96,
|
||
|
|
tasks_completed: 27,
|
||
|
|
tasks_total: 28,
|
||
|
|
blocker_count: 1,
|
||
|
|
},
|
||
|
|
step4: {
|
||
|
|
completion_rate_at_audit: 0.96,
|
||
|
|
verification_failures_found: 1,
|
||
|
|
documentation_updates_needed: 2,
|
||
|
|
},
|
||
|
|
user_feedback: {
|
||
|
|
overall_rating: 8,
|
||
|
|
rating_reason: 'good stuff',
|
||
|
|
what_went_well: 'well',
|
||
|
|
what_went_poorly: 'poorly',
|
||
|
|
},
|
||
|
|
};
|
||
|
|
|
||
|
|
// ── parseSubmissionPayload() ─────────────────────────────────────────────────
|
||
|
|
|
||
|
|
describe('parseSubmissionPayload', () => {
|
||
|
|
it('parses a fully valid payload into a correctly-shaped RunMetrics', () => {
|
||
|
|
const result = parseSubmissionPayload(issueBody(VALID_PAYLOAD));
|
||
|
|
expect(result).not.toBeNull();
|
||
|
|
expect(result!.run_id).toBe('run-20260715-143000-a1b2');
|
||
|
|
expect(result!.schema_version).toBe('1.0');
|
||
|
|
expect(result!.plan2code_version).toBe('1.15.3');
|
||
|
|
expect(result!.source).toBe('community');
|
||
|
|
expect(result!.step1_plan.present).toBe(true);
|
||
|
|
expect(result!.step1_plan.final_confidence).toBe(95);
|
||
|
|
expect(result!.step2_document.present).toBe(true);
|
||
|
|
expect(result!.step2_document.total_tasks).toBe(28);
|
||
|
|
expect(result!.step3_implement.present).toBe(true);
|
||
|
|
expect(result!.step3_implement.tasks_completed).toBe(27);
|
||
|
|
expect(result!.step4_finalize.present).toBe(true);
|
||
|
|
expect(result!.step4_finalize.completion_rate_at_audit).toBe(0.96);
|
||
|
|
expect(result!.user_feedback).toEqual({
|
||
|
|
overall_rating: 8,
|
||
|
|
rating_reason: 'good stuff',
|
||
|
|
what_went_well: 'well',
|
||
|
|
what_went_poorly: 'poorly',
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
it('returns null when run_id is missing', () => {
|
||
|
|
const { run_id, ...withoutRunId } = VALID_PAYLOAD;
|
||
|
|
const result = parseSubmissionPayload(issueBody(withoutRunId));
|
||
|
|
expect(result).toBeNull();
|
||
|
|
});
|
||
|
|
|
||
|
|
it('returns null when user_feedback.overall_rating is a string instead of a number', () => {
|
||
|
|
const badPayload = {
|
||
|
|
...VALID_PAYLOAD,
|
||
|
|
user_feedback: { ...VALID_PAYLOAD.user_feedback, overall_rating: 'eight' },
|
||
|
|
};
|
||
|
|
const result = parseSubmissionPayload(issueBody(badPayload));
|
||
|
|
expect(result).toBeNull();
|
||
|
|
});
|
||
|
|
|
||
|
|
it('returns null when schema_version is not exactly "1.0"', () => {
|
||
|
|
const badPayload = { ...VALID_PAYLOAD, schema_version: '2.0' };
|
||
|
|
const result = parseSubmissionPayload(issueBody(badPayload));
|
||
|
|
expect(result).toBeNull();
|
||
|
|
});
|
||
|
|
|
||
|
|
it('sets all four steps present:false when only user_feedback is included', () => {
|
||
|
|
const minimalPayload = {
|
||
|
|
schema_version: '1.0',
|
||
|
|
run_id: 'run-20260715-150000-c3d4',
|
||
|
|
plan2code_version: '1.15.3',
|
||
|
|
user_feedback: VALID_PAYLOAD.user_feedback,
|
||
|
|
};
|
||
|
|
const result = parseSubmissionPayload(issueBody(minimalPayload));
|
||
|
|
expect(result).not.toBeNull();
|
||
|
|
expect(result!.step1_plan.present).toBe(false);
|
||
|
|
expect(result!.step2_document.present).toBe(false);
|
||
|
|
expect(result!.step3_implement.present).toBe(false);
|
||
|
|
expect(result!.step4_finalize.present).toBe(false);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('backfills missing prompt_versions_short keys with the sha256:missing sentinel', () => {
|
||
|
|
const partialPayload = {
|
||
|
|
...VALID_PAYLOAD,
|
||
|
|
prompt_versions_short: { plan: 'abc123def456' },
|
||
|
|
};
|
||
|
|
const result = parseSubmissionPayload(issueBody(partialPayload));
|
||
|
|
expect(result).not.toBeNull();
|
||
|
|
expect(result!.prompt_versions.plan).toBe('abc123def456');
|
||
|
|
expect(result!.prompt_versions.revise_plan).toBe('sha256:missing');
|
||
|
|
expect(result!.prompt_versions.document).toBe('sha256:missing');
|
||
|
|
expect(result!.prompt_versions.implement).toBe('sha256:missing');
|
||
|
|
expect(result!.prompt_versions.finalize).toBe('sha256:missing');
|
||
|
|
expect(result!.prompt_versions.init).toBe('sha256:missing');
|
||
|
|
expect(result!.prompt_versions.init_update).toBe('sha256:missing');
|
||
|
|
expect(result!.prompt_versions.quick_task).toBe('sha256:missing');
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
// ── writeRunFile() ────────────────────────────────────────────────────────────
|
||
|
|
|
||
|
|
describe('writeRunFile', () => {
|
||
|
|
let tmpDir: string;
|
||
|
|
|
||
|
|
afterEach(() => {
|
||
|
|
if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true });
|
||
|
|
});
|
||
|
|
|
||
|
|
const RUN: RunMetrics = {
|
||
|
|
schema_version: '1.0',
|
||
|
|
run_id: 'run-20260715-160000-e5f6',
|
||
|
|
plan2code_version: '1.15.3',
|
||
|
|
prompt_versions: {
|
||
|
|
plan: 'sha256:missing', revise_plan: 'sha256:missing', document: 'sha256:missing',
|
||
|
|
implement: 'sha256:missing', finalize: 'sha256:missing', init: 'sha256:missing',
|
||
|
|
init_update: 'sha256:missing', quick_task: 'sha256:missing',
|
||
|
|
},
|
||
|
|
project: { name: '', started_at: null, completed_at: null },
|
||
|
|
step1_plan: { present: false, final_confidence: null, confidence_breakdown: null, clarification_rounds: null, tech_stack_revision_rounds: null, verification_gaps_found: null, functional_requirements_count: null, non_functional_requirements_count: null, risk_count: null, phase_count: null },
|
||
|
|
step2_document: { present: false, total_tasks: null, tasks_per_phase: null, phase_count: null, parallel_groups_identified: null, requirement_coverage_percent: null, verification_items_added: null },
|
||
|
|
step3_implement: { present: false, task_completion_rate: null, tasks_completed: null, tasks_total: null, blocker_count: null },
|
||
|
|
step4_finalize: { present: false, completion_rate_at_audit: null, verification_failures_found: null, documentation_updates_needed: null, archival_succeeded: null },
|
||
|
|
user_feedback: null,
|
||
|
|
};
|
||
|
|
|
||
|
|
it('writes a new run_id to an empty runsDir and returns true', () => {
|
||
|
|
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan2code-metrics-test-'));
|
||
|
|
const result = writeRunFile(RUN, tmpDir);
|
||
|
|
expect(result).toBe(true);
|
||
|
|
expect(fs.existsSync(path.join(tmpDir, `${RUN.run_id}.json`))).toBe(true);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('returns false and does not overwrite when the same run_id already exists', () => {
|
||
|
|
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'plan2code-metrics-test-'));
|
||
|
|
writeRunFile(RUN, tmpDir);
|
||
|
|
const modified = { ...RUN, plan2code_version: '9.9.9' };
|
||
|
|
const result = writeRunFile(modified, tmpDir);
|
||
|
|
expect(result).toBe(false);
|
||
|
|
const onDisk = JSON.parse(fs.readFileSync(path.join(tmpDir, `${RUN.run_id}.json`), 'utf8'));
|
||
|
|
expect(onDisk.plan2code_version).toBe('1.15.3');
|
||
|
|
});
|
||
|
|
});
|
||
|
|
|
||
|
|
// ── ingestCommunityIssues() ───────────────────────────────────────────────────
|
||
|
|
|
||
|
|
describe('ingestCommunityIssues', () => {
|
||
|
|
const goodBody = issueBody(VALID_PAYLOAD);
|
||
|
|
const badBody = 'an issue with no METRICS_JSON payload';
|
||
|
|
|
||
|
|
it('imports a new run and closes its issue', async () => {
|
||
|
|
const closed: number[] = [];
|
||
|
|
const tally = await ingestCommunityIssues(
|
||
|
|
[{ number: 1, body: goodBody }],
|
||
|
|
'owner/repo',
|
||
|
|
'/runs',
|
||
|
|
{ writeRunFile: () => true, closeIssue: async (_r, n) => { closed.push(n); } },
|
||
|
|
);
|
||
|
|
expect(tally.imported).toBe(1);
|
||
|
|
expect(tally.skippedDuplicate).toBe(0);
|
||
|
|
expect(tally.closed).toBe(1);
|
||
|
|
expect(closed).toEqual([1]);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('closes an already-imported (duplicate) issue instead of skipping the close', async () => {
|
||
|
|
const closed: number[] = [];
|
||
|
|
const tally = await ingestCommunityIssues(
|
||
|
|
[{ number: 7, body: goodBody }],
|
||
|
|
'owner/repo',
|
||
|
|
'/runs',
|
||
|
|
{ writeRunFile: () => false, closeIssue: async (_r, n) => { closed.push(n); } },
|
||
|
|
);
|
||
|
|
expect(tally.imported).toBe(0);
|
||
|
|
expect(tally.skippedDuplicate).toBe(1);
|
||
|
|
expect(tally.closed).toBe(1); // the close-retry: duplicates are still closed
|
||
|
|
expect(closed).toEqual([7]);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('records a close failure without throwing and leaves the issue for a later retry', async () => {
|
||
|
|
const tally = await ingestCommunityIssues(
|
||
|
|
[{ number: 9, body: goodBody }],
|
||
|
|
'owner/repo',
|
||
|
|
'/runs',
|
||
|
|
{ writeRunFile: () => true, closeIssue: async () => { throw new Error('network'); } },
|
||
|
|
);
|
||
|
|
expect(tally.imported).toBe(1);
|
||
|
|
expect(tally.closed).toBe(0);
|
||
|
|
expect(tally.closeFailed).toBe(1);
|
||
|
|
expect(tally.closeFailedIssues).toEqual([9]);
|
||
|
|
});
|
||
|
|
|
||
|
|
it('skips and reports a malformed issue without writing or closing it', async () => {
|
||
|
|
let wrote = false;
|
||
|
|
let closeCalled = false;
|
||
|
|
const tally = await ingestCommunityIssues(
|
||
|
|
[{ number: 3, body: badBody }],
|
||
|
|
'owner/repo',
|
||
|
|
'/runs',
|
||
|
|
{ writeRunFile: () => { wrote = true; return true; }, closeIssue: async () => { closeCalled = true; } },
|
||
|
|
);
|
||
|
|
expect(tally.skippedMalformed).toBe(1);
|
||
|
|
expect(tally.malformedIssues).toEqual([3]);
|
||
|
|
expect(wrote).toBe(false);
|
||
|
|
expect(closeCalled).toBe(false);
|
||
|
|
});
|
||
|
|
});
|