Add test command discovery and a check runner, and offer a test map in Init

AI Assisted
This commit is contained in:
jparkerweb committed 2026-11-03 21:04:32 -08:00
1 parent aecc3afb4c
commit 82298d256c
10 files changed
+706 -13

No files matched your search

+112
View File
@@ -0,0 +1,112 @@
// run-check.mjs: the four outcomes, the process-tree kill on timeout, the log
// and the tail cap. Runs src/skill-scripts/run-check.mjs directly until a
// skill ships it. See skill-script-helpers.mjs for the fixtures.
import { test } from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import path from "node:path";
import { spawnSync } from "node:child_process";
import { ROOT, NOW, scrubbedEnv, tmp, write } from "./skill-script-helpers.mjs";
const SCRIPT = path.join(ROOT, "src", "skill-scripts", "run-check.mjs");
const NODE = `"${process.execPath}"`;
function runShared(args, { cwd } = {}) {
const r = spawnSync(process.execPath, [SCRIPT, ...args], { cwd: cwd || ROOT, encoding: "utf8", env: scrubbedEnv({ PLAN2CODE_NOW: NOW }) });
let json = null;
try {
json = JSON.parse(r.stdout);
} catch {}
return { code: r.status, json, stdout: r.stdout, stderr: r.stderr };
}
const alive = (pid) => {
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
};
test("run-check: passed on exit 0", () => {
const r = runShared(["--cmd", `${NODE} -e "process.exit(0)"`]);
assert.equal(r.code, 0, r.stderr);
assert.equal(r.json.outcome, "passed");
assert.equal(r.json.exit, 0);
assert.equal(r.json.cwd, ".");
assert.equal(typeof r.json.seconds, "number");
assert.equal(r.json.log, null);
});
test("run-check: failed on exit 1, with the printed lines in the tail", () => {
const dir = tmp("check-fail");
write(dir, "fail.js", "console.log('first line');\nconsole.error('second line');\nprocess.exit(1);\n");
const r = runShared(["--cmd", `${NODE} fail.js`, "--cwd", dir]);
assert.equal(r.code, 0, r.stderr);
assert.equal(r.json.outcome, "failed");
assert.equal(r.json.exit, 1);
assert.ok(r.json.tail.includes("first line"), JSON.stringify(r.json.tail));
assert.ok(r.json.tail.includes("second line"), JSON.stringify(r.json.tail));
});
test("run-check: not-started for a made-up command and for a missing --cwd", () => {
const r = runShared(["--cmd", "p2c-no-such-command-xyz --flag"]);
assert.equal(r.code, 0, r.stderr);
assert.equal(r.json.outcome, "not-started");
assert.equal(r.json.reason, "command-not-found");
const c = runShared(["--cmd", `${NODE} -e "process.exit(0)"`, "--cwd", path.join(tmp("check-cwd"), "missing")]);
assert.equal(c.code, 0, c.stderr);
assert.equal(c.json.outcome, "not-started");
assert.equal(c.json.reason, "cwd-missing");
});
test("run-check: timeout kills the whole process tree", async () => {
const dir = tmp("check-timeout");
write(dir, "grandchild.js", "require('fs').writeFileSync(require('path').join(__dirname, 'pid.txt'), String(process.pid));\nsetInterval(() => {}, 1000);\n");
write(dir, "parent.js", "require('child_process').spawn(process.execPath, [require('path').join(__dirname, 'grandchild.js')], { stdio: 'inherit' });\nsetInterval(() => {}, 1000);\n");
const r = runShared(["--cmd", `${NODE} parent.js`, "--cwd", dir, "--timeout", "2"]);
assert.equal(r.code, 0, r.stderr);
assert.equal(r.json.outcome, "timeout");
const pidFile = path.join(dir, "pid.txt");
assert.ok(fs.existsSync(pidFile), "the grandchild started");
const pid = Number(fs.readFileSync(pidFile, "utf8"));
const until = Date.now() + 5000;
while (alive(pid) && Date.now() < until) await new Promise((res) => setTimeout(res, 100));
assert.equal(alive(pid), false, `grandchild ${pid} is gone`);
});
test("run-check: --log holds the full output and the tail keeps 60 lines of at most 400 characters", () => {
const dir = tmp("check-log");
write(dir, "many.js", "for (let i = 1; i <= 100; i++) console.log('line ' + i);\nconsole.log('x'.repeat(500));\n");
const log = path.join(dir, "logs", "deep", "run.log");
const r = runShared(["--cmd", `${NODE} many.js`, "--cwd", dir, "--log", log]);
assert.equal(r.code, 0, r.stderr);
assert.equal(r.json.outcome, "passed");
assert.equal(r.json.log, log);
const full = fs.readFileSync(log, "utf8").split(/\r?\n/).filter(Boolean);
assert.equal(full.length, 101);
assert.equal(full[0], "line 1");
assert.equal(r.json.tail.length, 60);
assert.equal(r.json.tail[0], "line 42");
assert.equal(r.json.tail[59].length, 400);
assert.deepEqual(fs.readdirSync(dir).sort(), ["logs", "many.js"], "nothing written outside --log");
});
test("run-check: bad arguments exit 2", () => {
assert.equal(runShared([]).code, 2);
assert.equal(runShared(["--cmd", "x", "--timeout", "soon"]).code, 2);
assert.equal(runShared(["--cmd", "x", "--timeout", "0"]).code, 2);
assert.equal(runShared(["--cmd", "x", "--timeout", "86401"]).code, 2, "past one day the timer would overflow");
assert.equal(runShared(["--cmd", "x", "extra"]).code, 2);
});
test("run-check and test-commands do nothing when imported", () => {
for (const file of ["run-check.mjs", "test-commands.mjs"]) {
const url = new URL(`file:///${path.join(ROOT, "src", "skill-scripts", file).replace(/\\/g, "/")}`).href;
const r = spawnSync(process.execPath, ["--input-type=module", "-e", `await import(${JSON.stringify(url)});`], { encoding: "utf8", env: scrubbedEnv() });
assert.equal(r.status, 0, `${file}: ${r.stderr}`);
assert.equal(r.stdout, "", `${file} printed nothing on import`);
}
});
+154
View File
@@ -0,0 +1,154 @@
// test-commands.mjs discover: one fixture per ecosystem, run from the built
// plan2code-init skill. See skill-script-helpers.mjs for how these run.
import { test } from "node:test";
import assert from "node:assert/strict";
import { runScript, tmp, write } from "./skill-script-helpers.mjs";
function discover(files) {
const dir = tmp("cmds");
for (const [rel, content] of Object.entries(files)) write(dir, rel, typeof content === "string" ? content : JSON.stringify(content));
const r = runScript("plan2code-init", "test-commands.mjs", ["discover"], { cwd: dir });
assert.equal(r.code, 0, r.stderr);
assert.equal(r.json.ok, true);
return r.json;
}
test("npm project: suite, lint, types and build checks", () => {
const r = discover({ "package.json": { scripts: { test: "node --test", lint: "eslint .", typecheck: "tsc --noEmit", build: "tsc" } } });
assert.deepEqual(r.suite, { cmd: "npm test", why: "package.json scripts.test" });
assert.deepEqual(r.runners, [{ cmd: "npm test", cwd: ".", covers: ["**"] }]);
assert.deepEqual(r.checks, [
{ kind: "lint", cmd: "npm run lint" },
{ kind: "types", cmd: "npm run typecheck" },
{ kind: "build", cmd: "npm run build" },
]);
assert.deepEqual(r.coverage, [{ runner: "node", cmd: "node --test --experimental-test-coverage --test-reporter=lcov --test-reporter-destination=<out>", cwd: ".", reads: "lcov" }]);
assert.equal(r.builtin, null);
});
test("the lockfile picks the package manager; tsconfig gives npx tsc", () => {
const r = discover({ "package.json": { scripts: { test: "jest", lint: "eslint ." } }, "pnpm-lock.yaml": "", "tsconfig.json": "{}" });
assert.equal(r.suite.cmd, "pnpm test");
assert.deepEqual(r.checks, [
{ kind: "lint", cmd: "pnpm run lint" },
{ kind: "types", cmd: "npx tsc --noEmit" },
]);
assert.equal(discover({ "package.json": { scripts: { test: "x" } }, "yarn.lock": "" }).suite.cmd, "yarn test");
assert.equal(discover({ "package.json": { scripts: { test: "x" } }, "bun.lock": "" }).suite.cmd, "bun test");
});
test("npm's placeholder test script is no suite", () => {
const r = discover({ "package.json": { scripts: { test: 'echo "Error: no test specified" && exit 1' } } });
assert.equal(r.suite, null);
assert.deepEqual(r.runners, []);
assert.deepEqual(r.builtin, { language: "node", cmd: "node --test" });
});
test("npm workspaces array: one runner per package with a test script", () => {
const r = discover({
"package.json": { workspaces: ["packages/*"], scripts: { test: "npm test --workspaces" } },
"packages/a/package.json": { scripts: { test: "node --test" } },
"packages/b/package.json": { scripts: { test: "vitest" }, devDependencies: { vitest: "1", "@vitest/coverage-v8": "1" } },
"packages/c/package.json": { scripts: {} },
});
assert.deepEqual(r.runners, [
{ cmd: "npm test", cwd: ".", covers: ["**"] },
{ cmd: "npm test", cwd: "packages/a", covers: ["packages/a/**"] },
{ cmd: "npm test", cwd: "packages/b", covers: ["packages/b/**"] },
]);
assert.deepEqual(r.coverage.map((c) => [c.runner, c.cwd]), [["node", "packages/a"], ["vitest", "packages/b"]]);
});
test("pnpm workspace: packages lines and the { packages } form", () => {
const r = discover({
"package.json": { name: "root" },
"pnpm-lock.yaml": "",
"pnpm-workspace.yaml": "packages:\n - 'apps/*'\n - \"libs/core\"\n - '!apps/skip'\n",
"apps/web/package.json": { scripts: { test: "vitest" } },
"apps/skip/package.json": { scripts: { test: "vitest" } },
"libs/core/package.json": { scripts: { test: "jest" } },
});
assert.equal(r.suite, null);
assert.deepEqual(r.runners, [
{ cmd: "pnpm test", cwd: "apps/web", covers: ["apps/web/**"] },
{ cmd: "pnpm test", cwd: "libs/core", covers: ["libs/core/**"] },
], "a ! line excludes its folders");
assert.deepEqual(r.coverage, [{ runner: "jest", cmd: "npx jest --coverage --coverageReporters=lcov", cwd: "libs/core", reads: "lcov" }]);
const obj = discover({ "package.json": { workspaces: { packages: ["pkgs/*"] } }, "pkgs/x/package.json": { scripts: { test: "t" } } });
assert.deepEqual(obj.runners, [{ cmd: "npm test", cwd: "pkgs/x", covers: ["pkgs/x/**"] }]);
});
test("Go module with a nested second module", () => {
const r = discover({ "go.mod": "module example.com/a\n", "main.go": "package main\n", "tools/go.mod": "module example.com/tools\n" });
assert.deepEqual(r.runners, [
{ cmd: "go test ./...", cwd: ".", covers: ["**"] },
{ cmd: "go test ./...", cwd: "tools", covers: ["tools/**"] },
]);
assert.deepEqual(r.coverage[0], { runner: "go", cmd: "go test -coverprofile=<out> ./... && go tool cover -func=<out>", cwd: ".", reads: "go-func" });
assert.equal(r.builtin, null);
});
test("Cargo workspace: one runner at the root, members run from it", () => {
const r = discover({
"Cargo.toml": '[workspace]\nmembers = ["crates/a"]\n',
"crates/a/Cargo.toml": '[package]\nname = "a"\n',
"crates/a/src/lib.rs": "",
});
assert.deepEqual(r.runners, [{ cmd: "cargo test", cwd: ".", covers: ["**"] }]);
});
test("pytest project and plain Python project", () => {
const py = discover({ "pyproject.toml": "[tool.pytest.ini_options]\naddopts = \"-q\"\n", "pkg/a.py": "" });
assert.deepEqual(py.runners, [{ cmd: "python -m pytest", cwd: ".", covers: ["**"] }]);
assert.deepEqual(py.coverage, [], "no pytest-cov, no coverage");
const plain = discover({ "setup.cfg": "[metadata]\nname = x\n", "x/a.py": "" });
assert.deepEqual(plain.runners, [{ cmd: "python -m unittest", cwd: ".", covers: ["**"] }]);
const req = discover({ "tox.ini": "[tox]\n", "requirements-dev.txt": "pytest==8\n" });
assert.equal(req.runners[0].cmd, "python -m pytest");
});
test(".NET project", () => {
const r = discover({ "App.sln": "", "src/App/App.csproj": "<Project />", "tests/App.Tests/App.Tests.csproj": "<Project />" });
assert.deepEqual(r.runners, [{ cmd: "dotnet test", cwd: ".", covers: ["**"] }]);
assert.deepEqual(r.coverage, []);
});
test("Makefile-only project: make test is the suite and the root runner", () => {
const r = discover({ Makefile: "build:\n\tcc a.c\n\ntest: build\n\t./run-tests\n" });
assert.deepEqual(r.suite, { cmd: "make test", why: "Makefile test target" });
assert.deepEqual(r.runners, [{ cmd: "make test", cwd: ".", covers: ["**"] }]);
assert.equal(discover({ Makefile: "build:\n\tcc a.c\n" }).suite, null);
});
test("coverage is listed only with its add-on present", () => {
const vitest = (extra) => discover({ "package.json": { scripts: { test: "vitest" }, devDependencies: { vitest: "1", ...extra } } });
assert.deepEqual(vitest({ "@vitest/coverage-v8": "1" }).coverage, [{ runner: "vitest", cmd: "npx vitest run --coverage --coverage.reporter=lcov", cwd: ".", reads: "lcov" }]);
assert.deepEqual(vitest({}).coverage, []);
const pytest = (deps) => discover({ "pyproject.toml": `[project]\ndependencies = [${deps}]\n` });
assert.deepEqual(pytest('"pytest", "pytest-cov"').coverage, [{ runner: "pytest", cmd: "python -m pytest --cov --cov-report=lcov:<out>", cwd: ".", reads: "lcov" }]);
assert.deepEqual(pytest('"pytest"').coverage, []);
const nested = discover({ "requirements-dev.txt": "pytest-cov\n", "svc/setup.cfg": "[tool:pytest]\n" });
assert.deepEqual(nested.coverage, [{ runner: "pytest", cmd: "python -m pytest --cov --cov-report=lcov:<out>", cwd: "svc", reads: "lcov" }], "pytest-cov listed in a folder above the runner");
const dotnet = (ref) => discover({ "App.Tests/App.Tests.csproj": `<Project><ItemGroup>${ref}</ItemGroup></Project>` });
assert.deepEqual(dotnet('<PackageReference Include="coverlet.collector" Version="6" />').coverage, [{ runner: "dotnet", cmd: 'dotnet test --collect:"XPlat Code Coverage"', cwd: "App.Tests", reads: "lcov" }]);
assert.deepEqual(dotnet("").coverage, []);
});
test("builtin: an empty Node project, a Python-only folder, and others in a mixed folder", () => {
assert.deepEqual(discover({ "package.json": { name: "x" } }).builtin, { language: "node", cmd: "node --test" });
assert.deepEqual(discover({ "a.py": "", "lib/b.py": "" }).builtin, { language: "python", cmd: "python -m unittest" });
const mixed = discover({ "package.json": { name: "x" }, "index.js": "", "a.py": "", "b.py": "", "c.py": "" });
assert.deepEqual(mixed.builtin, { language: "python", cmd: "python -m unittest", others: [{ language: "node", cmd: "node --test" }] });
assert.equal(discover({ "README.md": "hi" }).builtin, null);
});
test("skipped folders are never walked; bad input is a usage error", () => {
const r = discover({ "node_modules/x/go.mod": "module x\n", "vendor/y/Cargo.toml": "[package]\n", "a.txt": "" });
assert.deepEqual(r.runners, []);
const dir = tmp("cmds-bad");
assert.equal(runScript("plan2code-init", "test-commands.mjs", ["list"], { cwd: dir }).code, 2);
assert.equal(runScript("plan2code-init", "test-commands.mjs", ["discover", "--root", "nope"], { cwd: dir }).code, 3);
});
+14 -4
View File
@@ -123,9 +123,19 @@ test("test-runs.md says to ask when a check or its tests are unclear", () => {
test("Init and Init-update keep a test map beside the single-test command", () => {
const init = src("plan2code-init.md");
assert.match(init, /run single test \(and beside it a short \*\*Test map\*\*: `area -> tests or command`/);
assert.match(init, /which checks are quick/);
assert.match(init, /run single test \(and beside it a note of which checks are quick\)/);
assert.ok(init.includes("## Test map"), "Init names the ## Test map section");
assert.match(init, /test-commands\.mjs" discover/);
assert.match(init, /write it only on a yes with `test-map\.mjs write --rows <file>`/);
assert.doesNotMatch(init, /area -> tests or command/, "the old line format is gone");
const upd = src("plan2code-init-update.md");
assert.match(upd, /\*\*4\. Testing\*\* - .*the test map and quick checks/);
assert.match(upd, /\| Testing \| Commands\? Test map and quick checks stale\?/);
assert.match(upd, /\*\*4\. Testing\*\* - .*the `## Test map` table and quick checks/);
assert.match(upd, /\| Testing \| Commands\? Test map `Paths` still match real files and `Run` commands still exist \(`test-map\.mjs read` problems, `test-commands\.mjs discover`\)\?/);
assert.match(upd, /written only on a yes through `test-map\.mjs write`/);
});
test("test-runs.md reads the ## Test map table and treats Shared rows as shared ground", () => {
const rule = src("shared-references", "test-runs.md");
assert.ok(rule.includes("`## Test map`"), "names the ## Test map table");
assert.ok(rule.includes("`Shared: yes`"), "a Shared: yes row is shared ground");
});