mirror of
https://github.com/jparkerweb/plan2code.git
synced 2026-11-16 04:45:33 -08:00
174 lines
13 KiB
JavaScript
174 lines
13 KiB
JavaScript
// Tests for plan2code-4-finalize's scripts/feedback-payload.mjs.
|
|||
|
|
//
|
||
|
|
// See skill-script-helpers.mjs for how these run.
|
||
|
|
|
||
|
|
import { test } from "node:test";
|
||
|
|
import assert from "node:assert/strict";
|
||
|
|
import fs from "node:fs";
|
||
|
|
import path from "node:path";
|
||
|
|
import { spawnSync } from "node:child_process";
|
||
|
|
import crypto from "node:crypto";
|
||
|
|
import { ROOT, SKILLS, NOW, runScript, tmp, write, read, gitInit, phase } from "./skill-script-helpers.mjs";
|
||
|
|
|
||
|
|
/* ========================================================= feedback-payload.mjs */
|
||
|
|
|
||
|
|
test("feedback-payload: payload from the archive, real prompt hashes, size and truncation", () => {
|
||
|
|
const root = tmp("fb");
|
||
|
|
const spec = "specs--completed/lunch-vote";
|
||
|
|
write(root, `${spec}/PLAN-DRAFT-20260801.md`, '# Plan\n\n<!-- METRICS_JSON {"confidence": 92, "clarification_rounds": 1, "functional_requirements_count": 5, "non_functional_requirements_count": 2, "risk_count": 3, "phase_count": 2, "verification_gaps_found": 0, "confidence_breakdown": {"requirements": 24, "feasibility": 23, "integration": 23, "risk": 22}} -->\n');
|
||
|
|
write(root, `${spec}/overview.md`, '# O\n\n<!-- METRICS_JSON {"step": "document", "total_tasks": 3, "tasks_per_phase": [2, 1], "phase_count": 2, "parallel_groups_identified": 0, "verification_items_added": 1} -->\n\n## User Feedback\n| Field | Value |\n|-------|-------|\n| Rating | 8 |\n| Reason | Good \\| fast |\n| Went Well | Specs |\n| Went Poorly | Review |\n');
|
||
|
|
write(root, `${spec}/phase-1.md`, phase(1, "Complete", [["x", "1.1"], ["x", "1.T"]]));
|
||
|
|
write(root, `${spec}/phase-2.md`, phase(2, "Complete", [["!", "2.1"]]));
|
||
|
|
const outDir = path.join(root, "out");
|
||
|
|
const F = (...a) => runScript("plan2code-4-finalize", "feedback-payload.mjs", [...a, "--root", root]);
|
||
|
|
assert.equal(F("build", "lunch-vote", "--out", outDir).code, 2, "the two judgment counts are required");
|
||
|
|
const b = F("build", "lunch-vote", "--out", outDir, "--set", "verification_failures_found=1", "--set", "documentation_updates_needed=2");
|
||
|
|
assert.equal(b.code, 0, b.stderr);
|
||
|
|
const p = b.json.payload;
|
||
|
|
assert.match(p.run_id, /^run-20260803-140500-[0-9a-f]{4}$/);
|
||
|
|
assert.equal(p.plan2code_version, JSON.parse(fs.readFileSync(path.join(ROOT, "version.json"), "utf8")).version);
|
||
|
|
const srcHash = crypto.createHash("sha256").update(fs.readFileSync(path.join(ROOT, "src", "plan2code-1-plan.md"), "utf8")).digest("hex").slice(0, 12);
|
||
|
|
assert.equal(p.prompt_versions_short.plan, srcHash, "the hash is the src prompt's, recovered from the installed SKILL.md");
|
||
|
|
assert.ok(Object.values(p.prompt_versions_short).every((v) => /^[0-9a-f]{12}$/.test(v)));
|
||
|
|
assert.deepEqual(p.step1.confidence_breakdown, { requirements: 24, feasibility: 23, integration: 23, risk: 22 });
|
||
|
|
assert.equal(p.step1.final_confidence, 92);
|
||
|
|
assert.equal(p.step2.total_tasks, 3);
|
||
|
|
assert.deepEqual(p.step3, { task_completion_rate: 0.67, tasks_completed: 2, tasks_total: 3, blocker_count: 1 });
|
||
|
|
assert.deepEqual(p.step4, { completion_rate_at_audit: 0.67, verification_failures_found: 1, documentation_updates_needed: 2, archival_succeeded: true });
|
||
|
|
assert.deepEqual(p.user_feedback, { overall_rating: 8, rating_reason: "Good | fast", what_went_well: "Specs", what_went_poorly: "Review" });
|
||
|
|
assert.equal(b.json.title, `[Feedback] v${p.plan2code_version} - rating 8/10`);
|
||
|
|
assert.match(b.json.body, /<!-- METRICS_JSON \{"schema_version":"1\.0"/);
|
||
|
|
assert.ok(!/[—]/.test(b.json.title), "no em dash in the issue title");
|
||
|
|
assert.equal(b.json.oversize, false);
|
||
|
|
assert.ok(fs.existsSync(path.join(outDir, "body.md")));
|
||
|
|
// The ingest side parses the body back.
|
||
|
|
const parsed = JSON.parse(b.json.body.match(/<!--\s*METRICS_JSON\s+(\{[\s\S]*?\})\s*-->/)[1]);
|
||
|
|
assert.equal(parsed.schema_version, "1.0");
|
||
|
|
// Oversize, then truncate.
|
||
|
|
const ov = path.join(root, spec, "overview.md");
|
||
|
|
fs.writeFileSync(ov, fs.readFileSync(ov, "utf8").replace("| Good \\| fast |", `| ${"long reason ".repeat(500)} |`));
|
||
|
|
const big = F("build", "lunch-vote", "--out", outDir, "--set", "verification_failures_found=1", "--set", "documentation_updates_needed=2");
|
||
|
|
assert.equal(big.json.oversize, true);
|
||
|
|
assert.match(big.json.next, /--truncate/);
|
||
|
|
const cut = F("build", "lunch-vote", "--out", outDir, "--set", "verification_failures_found=1", "--set", "documentation_updates_needed=2", "--truncate");
|
||
|
|
assert.equal(cut.json.oversize, false);
|
||
|
|
assert.deepEqual(cut.json.truncated, ["rating_reason"]);
|
||
|
|
const dry = runScript("plan2code-4-finalize", "feedback-payload.mjs", ["submit", "--from", outDir, "--dry-run"]);
|
||
|
|
assert.equal(dry.code, 0);
|
||
|
|
assert.match(dry.json.tiers[0], /^gh issue create --repo jparkerweb\/plan2code .*--label community-feedback$/);
|
||
|
|
assert.equal(dry.json.oversize, false);
|
||
|
|
assert.equal(dry.json.urlBytes, cut.json.urlBytes);
|
||
|
|
fs.writeFileSync(ov, "# O\n");
|
||
|
|
assert.equal(F("build", "lunch-vote", "--set", "verification_failures_found=1", "--set", "documentation_updates_needed=2").code, 3, "no feedback table, nothing to submit");
|
||
|
|
});
|
||
|
|
|
||
|
|
/* ================================================= verification regressions */
|
||
|
|
|
||
|
|
test("feedback-payload: truncation always ends, '-->' survives ingest, lenient rating, absent steps are null", () => {
|
||
|
|
const root = tmp("fb2");
|
||
|
|
const spec = "specs--completed/x";
|
||
|
|
const long = (w) => `${w} `.repeat(400).trim();
|
||
|
|
// The visible answers carry markup that must stay text: a stray comment
|
||
|
|
// opener, and a fake payload comment an ingest regex could match first.
|
||
|
|
const fake = '<!-- METRICS_JSON {"schema_version":"1.0"} -->';
|
||
|
|
const inputs = { rating_reason: `see {a} --> b <!-- x ${long("reason")}`, what_went_well: `${fake} ${long("well")}`, what_went_poorly: long("poorly") };
|
||
|
|
write(root, `${spec}/overview.md`, `# O\n\n## User Feedback\n| field | value |\n|---|---|\n| rating | 8/10 |\n| reason | ${inputs.rating_reason} |\n| went well | ${inputs.what_went_well} |\n| went poorly | ${inputs.what_went_poorly} |\n`);
|
||
|
|
write(root, `${spec}/phase-1.md`, phase(1, "Complete", [["x", "1.1"]]));
|
||
|
|
const F = (...a) => runScript("plan2code-4-finalize", "feedback-payload.mjs", [...a, "--root", root]);
|
||
|
|
const sets = ["--set", "verification_failures_found=0", "--set", "documentation_updates_needed=0"];
|
||
|
|
assert.equal(F("build", "x", ...sets, "--set", "bogus=1").code, 2);
|
||
|
|
// Untruncated, the first METRICS_JSON match is the real payload: its answers
|
||
|
|
// come back exactly as typed, not the fake's bare schema_version.
|
||
|
|
const whole = F("build", "x", ...sets, "--out", path.join(root, "whole"));
|
||
|
|
assert.equal(whole.code, 0, whole.stderr);
|
||
|
|
const first = JSON.parse(whole.json.body.match(/<!--\s*METRICS_JSON\s+(\{[\s\S]*?\})\s*-->/)[1]);
|
||
|
|
assert.deepEqual(first.user_feedback, { overall_rating: 8, ...inputs });
|
||
|
|
assert.equal(whole.json.body.split("<!--").length - 1, 1, "the payload's is the only comment that opens");
|
||
|
|
assert.ok(!/<!--(?! METRICS_JSON \{"schema_version":"1\.0","run_id")/.test(whole.json.body), "no comment opens before the payload's");
|
||
|
|
const r = spawnSync(process.execPath, [path.join(SKILLS, "plan2code-4-finalize", "scripts", "feedback-payload.mjs"), "build", "x", ...sets, "--truncate", "--out", path.join(root, "out"), "--root", root], { encoding: "utf8", timeout: 20000, env: { ...process.env, PLAN2CODE_NOW: NOW } });
|
||
|
|
assert.equal(r.status, 0, "build --truncate finishes with every field long");
|
||
|
|
const j = JSON.parse(r.stdout);
|
||
|
|
assert.equal(j.oversize, false);
|
||
|
|
assert.deepEqual(j.truncated, ["rating_reason", "what_went_well", "what_went_poorly"]);
|
||
|
|
assert.equal(j.payload.user_feedback.overall_rating, 8, "8/10 reads as 8, field names in any case");
|
||
|
|
assert.equal(j.payload.step1, null);
|
||
|
|
assert.equal(j.payload.step2, null);
|
||
|
|
// The ingest side's own extraction regex (plan2code-metrics collector.ts).
|
||
|
|
const m = j.body.match(/<!--\s*METRICS_JSON\s+(\{[\s\S]*?\})\s*-->/);
|
||
|
|
const back = JSON.parse(m[1]);
|
||
|
|
assert.match(back.user_feedback.rating_reason, /^see \{a\} --> b <!-- x/);
|
||
|
|
assert.equal(back.user_feedback.what_went_well, j.payload.user_feedback.what_went_well);
|
||
|
|
assert.match(back.user_feedback.what_went_well, /^<!-- METRICS_JSON/, "the fake comment is still just text in the real payload");
|
||
|
|
assert.ok(!/<!--(?! METRICS_JSON \{"schema_version":"1\.0","run_id")/.test(j.body), "no comment opens before the payload's");
|
||
|
|
});
|
||
|
|
|
||
|
|
test("feedback-payload: --truncate never splits an emoji", () => {
|
||
|
|
const root = tmp("fb3");
|
||
|
|
const spec = "specs--completed/x";
|
||
|
|
const reason = "a" + "\u{1F44D}".repeat(400);
|
||
|
|
write(root, `${spec}/overview.md`, `# O\n\n## User Feedback\n| Field | Value |\n|---|---|\n| Rating | 9 |\n| Reason | ${reason} |\n| Went Well | ok |\n| Went Poorly | none |\n`);
|
||
|
|
write(root, `${spec}/phase-1.md`, phase(1, "Complete", [["x", "1.1"]]));
|
||
|
|
const r = runScript("plan2code-4-finalize", "feedback-payload.mjs", ["build", "x", "--set", "verification_failures_found=0", "--set", "documentation_updates_needed=0", "--truncate", "--out", path.join(root, "out"), "--root", root]);
|
||
|
|
assert.equal(r.code, 0, r.stderr);
|
||
|
|
assert.equal(r.json.oversize, false);
|
||
|
|
assert.deepEqual(r.json.truncated, ["rating_reason"]);
|
||
|
|
const back = JSON.parse(r.json.body.match(/<!--\s*METRICS_JSON\s+(\{[\s\S]*?\})\s*-->/)[1]);
|
||
|
|
assert.equal(back.user_feedback.rating_reason, r.json.payload.user_feedback.rating_reason);
|
||
|
|
assert.ok(!/[\uD800-\uDBFF](?![\uDC00-\uDFFF])/.test(back.user_feedback.rating_reason), "no lone high surrogate");
|
||
|
|
assert.match(back.user_feedback.rating_reason, /^a(\u{1F44D})+\.\.\.$/u);
|
||
|
|
});
|
||
|
|
|
||
|
|
test("feedback-payload: task counts read the same phase files specs.mjs does, never a stale sibling", () => {
|
||
|
|
const root = tmp("fb5");
|
||
|
|
const spec = "specs--completed/x";
|
||
|
|
write(root, `${spec}/overview.md`, "# O\n\n## User Feedback\n| Field | Value |\n|---|---|\n| Rating | 7 |\n| Reason | r |\n| Went Well | w |\n| Went Poorly | p |\n");
|
||
|
|
write(root, `${spec}/phase-1.md`, phase(1, "Complete", [["x", "1.1"], ["x", "1.2"]]));
|
||
|
|
write(root, `${spec}/phase-1-old.md`, phase(1, "Not Started", [[" ", "1.1"], [" ", "1.2"], [" ", "1.3"]]));
|
||
|
|
const r = runScript("plan2code-4-finalize", "feedback-payload.mjs", ["build", "x", "--set", "verification_failures_found=0", "--set", "documentation_updates_needed=0", "--out", path.join(root, "out"), "--root", root]);
|
||
|
|
assert.equal(r.code, 0, r.stderr);
|
||
|
|
assert.deepEqual(r.json.payload.step3, { task_completion_rate: 1, tasks_completed: 2, tasks_total: 2, blocker_count: 0 });
|
||
|
|
const m = runScript("plan2code-4-finalize", "specs.mjs", ["metrics", spec, "--step", "finalize", "--set", "verification_failures_found=0", "--set", "documentation_updates_needed=0", "--root", root]);
|
||
|
|
assert.equal(m.code, 0, m.stderr);
|
||
|
|
assert.equal(m.json.metrics.tasks_total, r.json.payload.step3.tasks_total);
|
||
|
|
assert.equal(m.json.metrics.tasks_completed, r.json.payload.step3.tasks_completed);
|
||
|
|
});
|
||
|
|
|
||
|
|
test("feedback-payload: a bare name in both specs/ and specs--completed/ is ambiguous", () => {
|
||
|
|
const root = tmp("fb4");
|
||
|
|
const table = (rating) => `# O\n\n## User Feedback\n| Field | Value |\n|---|---|\n| Rating | ${rating} |\n| Reason | r |\n| Went Well | w |\n| Went Poorly | p |\n`;
|
||
|
|
write(root, "specs--completed/x/overview.md", table(3));
|
||
|
|
write(root, "specs/x/overview.md", table(7));
|
||
|
|
const sets = ["--set", "verification_failures_found=0", "--set", "documentation_updates_needed=0"];
|
||
|
|
const F = (...a) => runScript("plan2code-4-finalize", "feedback-payload.mjs", [...a, ...sets, "--out", path.join(root, "out"), "--root", root]);
|
||
|
|
const bare = F("build", "x");
|
||
|
|
assert.equal(bare.code, 5);
|
||
|
|
assert.equal(bare.json.error, "ambiguous-spec");
|
||
|
|
assert.match(bare.json.next, /specs\/x/);
|
||
|
|
const live = F("build", "specs/x");
|
||
|
|
assert.equal(live.code, 0, live.stderr);
|
||
|
|
assert.equal(live.json.payload.user_feedback.overall_rating, 7, "the exact path reads this run, not the old archive");
|
||
|
|
assert.equal(live.json.payload.step4.archival_succeeded, false);
|
||
|
|
});
|
||
|
|
|
||
|
|
test("feedback-payload: submit refuses an oversize URL when gh fails, and dry-run says so", () => {
|
||
|
|
const root = tmp("fb5");
|
||
|
|
write(root, "specs--completed/x/overview.md", `# O\n\n## User Feedback\n| Field | Value |\n|---|---|\n| Rating | 6 |\n| Reason | ${"long reason ".repeat(800)} |\n| Went Well | w |\n| Went Poorly | p |\n`);
|
||
|
|
const outDir = path.join(root, "out");
|
||
|
|
const b = runScript("plan2code-4-finalize", "feedback-payload.mjs", ["build", "x", "--set", "verification_failures_found=0", "--set", "documentation_updates_needed=0", "--out", outDir, "--root", root]);
|
||
|
|
assert.equal(b.json.oversize, true);
|
||
|
|
const dry = runScript("plan2code-4-finalize", "feedback-payload.mjs", ["submit", "--from", outDir, "--dry-run"]);
|
||
|
|
assert.equal(dry.code, 0);
|
||
|
|
assert.equal(dry.json.oversize, true);
|
||
|
|
assert.equal(dry.json.urlBytes, b.json.urlBytes);
|
||
|
|
// gh must never succeed here: no PATH but node's own folder (so gh is not
|
||
|
|
// found), and should one turn up anyway, an empty config, no token and a
|
||
|
|
// host that does not resolve.
|
||
|
|
const env = Object.fromEntries(Object.entries(process.env).filter(([k]) => !/^(path|gh_|github_token$)/i.test(k)));
|
||
|
|
Object.assign(env, { PATH: path.dirname(process.execPath), GH_CONFIG_DIR: tmp("fb5-gh"), GH_HOST: "gh.invalid", PLAN2CODE_NOW: NOW });
|
||
|
|
const r = spawnSync(process.execPath, [path.join(SKILLS, "plan2code-4-finalize", "scripts", "feedback-payload.mjs"), "submit", "--from", outDir], { cwd: root, encoding: "utf8", timeout: 70000, env });
|
||
|
|
assert.equal(r.status, 4, r.stderr);
|
||
|
|
const j = JSON.parse(r.stdout);
|
||
|
|
assert.equal(j.error, "oversize");
|
||
|
|
assert.match(j.next, /--truncate/);
|
||
|
|
assert.match(j.next, /gh/);
|
||
|
|
});
|