From d5d8278cfeeae14f50ac748a820a84d8d0a6e458 Mon Sep 17 00:00:00 2001 From: Justin Parker Date: Mon, 26 Oct 2026 22:46:16 -0700 Subject: [PATCH] Fail npm test when a page test reads a style or scrolls through a raw locator.evaluate, and document the CI lessons from PR 16: cargo fmt and clippy before pushing app changes, the test concurrency cap, and the redraw-safe page-test helpers AI Assisted --- .agents-docs/AGENTS-architecture.md | 1 + .agents-docs/AGENTS-development-commands.md | 10 ++++- AGENTS.md | 4 +- plan2code-app/README.md | 5 +++ plan2code-app/RELEASE-CHECKLIST.md | 3 +- scripts/test-web-console-page-git.mjs | 4 +- scripts/test-web-console-shape.mjs | 46 +++++++++++++++++++++ 7 files changed, 67 insertions(+), 6 deletions(-) diff --git a/.agents-docs/AGENTS-architecture.md b/.agents-docs/AGENTS-architecture.md index 6aebb5e..3e48ee0 100644 --- a/.agents-docs/AGENTS-architecture.md +++ b/.agents-docs/AGENTS-architecture.md @@ -58,6 +58,7 @@ plan2code/ │ ├── parse-check.mjs # Imports every shipped browser module as real ESM │ ├── screenshots/ # JSON payloads posted per shot (`ask` reuses `question.json`; `finish` follows `signoff.json`) │ ├── test-web-console-*.mjs, web-console-test-helpers.mjs # Web console suites, split by area (the page tests are `-page-*`, the source shape `-shape`), and their shared setup +│ ├── node-test.mjs, page-test-helpers.mjs # `npm test`'s `node --test`, files at once capped at one per spare core; the page tests' browser sessions and their redraw-safe `styleOf()`, `stylesOf()`, `scrollCard()` │ ├── test-launcher.mjs, test-getting-started.mjs # Launcher suite; newcomer docs kept in step │ ├── test-skill-scripts.mjs, test-script-.mjs, test-git-repo.mjs, skill-script-helpers.mjs, fixture-helpers.mjs # Skill script suites and fixtures │ └── validate-char-count.js # Pre-commit character count validator diff --git a/.agents-docs/AGENTS-development-commands.md b/.agents-docs/AGENTS-development-commands.md index 27d6113..9a56486 100644 --- a/.agents-docs/AGENTS-development-commands.md +++ b/.agents-docs/AGENTS-development-commands.md @@ -47,12 +47,20 @@ cd plan2code-metrics && npm install # First time setup cd plan2code-metrics && npm run build # Build the CLI cd plan2code-metrics && npm test # vitest (aggregator, community, improver) +# Desktop app (plan2code-app/, Rust + Tauri). CI's `app` workflow runs all three on +# Windows, macOS and Linux and fails on any rustfmt difference or clippy warning, so run +# them before pushing, not only cargo test (see plan2code-app/README.md "Tests"). +# First time: rustup component add rustfmt clippy (for the toolchain in rust-toolchain.toml) +cd plan2code-app && node scripts/vendor.mjs && cargo fmt --check +cd plan2code-app && cargo clippy --workspace --all-targets -- -D warnings +cd plan2code-app && cargo test --workspace + # Plan2Code Bot (end-to-end workflow runner, Claude Agent SDK) cd plan2code-bot && npm install && npm run build cd plan2code-bot && npm test # vitest ``` -The root `npm test` does not run the metrics or bot vitest suites; run them in their package. `npm run dev` in loop, metrics and bot is `tsup --watch`. The web console and launcher suites isolate themselves with `PLAN2CODE_CONSOLE_HOME`, `PLAN2CODE_MODELS_DIR` and `PLAN2CODE_NO_BROWSER=1` (`scripts/web-console-test-helpers.mjs`), so they never open a browser or touch your real console home. The web console suites are split by area so `node --test` runs them in parallel. +The root `npm test` does not run the metrics or bot vitest suites; run them in their package. `npm run dev` in loop, metrics and bot is `tsup --watch`. The web console and launcher suites isolate themselves with `PLAN2CODE_CONSOLE_HOME`, `PLAN2CODE_MODELS_DIR` and `PLAN2CODE_NO_BROWSER=1` (`scripts/web-console-test-helpers.mjs`), so they never open a browser or touch your real console home. The web console suites are split by area so `node --test` runs them in parallel. `npm test` runs them through `scripts/node-test.mjs`, which caps the files run at once at one per spare core (8 at most): each page test file drives a browser, and more at once than the cores can carry fails page tests at random (a flat 8 did on 3 and 4 core CI runners). Page tests (`test-web-console-page-*.mjs`) read styles and set scrolls only through `styleOf()`, `stylesOf()` and `scrollCard()` from `page-test-helpers.mjs`, which retry until the element is still on the page; `test-web-console-shape.mjs` fails a raw `locator.evaluate` that does either. ## Test/Dev Env Vars diff --git a/AGENTS.md b/AGENTS.md index 5efc2d6..6cf98e9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -103,8 +103,8 @@ When you make a mistake, get corrected, or discover something about this codebas - Before deleting a test file that upstream split or removed, diff plan2code's copy against the rebranded upstream base and port the plan2code-only tests: v2.7.0's split of the web console suite would otherwise have dropped seven hardening tests (now in `scripts/test-web-console-hardening.mjs`). - Register a new skill in `install.js` `SOURCE_PROMPTS` before running its `scripts/test-script-.mjs`: importing `scripts/skill-script-helpers.mjs` rebuilds `skills/` and prunes any skill folder `install.js` does not list, so the script under test vanishes. - When a `CHANGELOG.md` heading gets a version number, set the same number in `version.json` (with its `releaseDate`), the root `package.json` and README's `Version X.Y.Z` line in the same change, then run `npm run build:skills`. The changelog skill only touches `package.json`: v2.9.0 nearly shipped with `version.json` at 2.8.1, and README's line sat at 2.6.0 for three releases. -- When `npm test` fails only timing-bound suites (the chat stress test, the page-cards redraw test "a redraw keeps the place of...", then a cascade in the session suite starting at "no submitted frame within 5s") while the machine is busy, rerun those files alone, and the `node --test` list with `--test-concurrency=4`, before suspecting the change: they failed that way on untouched code under load during the web console file split, and the redraw test did again in three phases of web-console-health. In web-console-notepad, four more failed only in full runs and passed alone: the Workspace page's @-mention test, the dashboard page's handoff chip test, the session suite's document route (`connect ETIMEDOUT`) and the notepad page's 200 KB typing test (why its budget there is 200 ms, not the 92 ms it takes alone). In helper-strength, the page-cards test "a card cut by the window fades at its bottom edge..." did the same. On CI the flat `--test-concurrency=8` was itself the load: 3 and 4 core runners ran 8 browser suites at once and a different handful of page tests failed each run, so `scripts/node-test.mjs` now caps it at one file per spare core (still 8 at most). -- Run `npm test` and `cargo test` from a terminal outside the desktop app, or with `PLAN2CODE_IN_APP`, `PLAN2CODE_APP_TOKEN`, `PLAN2CODE_APP_VERSION`, `PLAN2CODE_APP_ANNOUNCE`, `PLAN2CODE_EMBED_ORIGIN` and `PLAN2CODE_CONSOLE_SID` unset (`env -u ... npm test`): inherited from the app they fail about 34 tests that check behavior outside the app (the update banner, the terminal line, cookies, open's browser launch), and a test console announces itself to the live window. A `cargo test` run mid-build once took the window over that way, then left an agent-less dashboard in it, and the build's question was never seen. The app now ignores sessions outside its console store (`take_announced` in `main.rs`), and the Rust console tests scrub the variables, but the JS suites do not. +- When a page test (`scripts/test-web-console-page-*.mjs`) fails only on CI or only in a full run, look for a race with a redraw before suspecting the change, and fix the test, not the timeout: read styles and set scrolls through `styleOf()`, `stylesOf()` and `scrollCard()` in `page-test-helpers.mjs` (a card a redraw replaced reads "" for every style and scrolls nothing; `test-web-console-shape.mjs` now fails a raw `locator.evaluate` that does either), and wait for the page to hold what the next step needs before acting (the workspace frame before typing `@`, the notepad's drawn marks, a newly selected tab). Keep `npm test`'s file concurrency under `scripts/node-test.mjs`'s cap of one file per spare core: a flat `--test-concurrency=8` ran 8 browsers on 3 and 4 core runners and failed a different page test on every CI run. On a busy local machine, rerun the failing files alone first; the chat stress test and the session suite's "no submitted frame within 5s" cascade still fail that way under load. +- Run `npm test` and `cargo test` from a terminal outside the desktop app, or with `PLAN2CODE_IN_APP`, `PLAN2CODE_APP_TOKEN`, `PLAN2CODE_APP_VERSION`, `PLAN2CODE_APP_ANNOUNCE`, `PLAN2CODE_EMBED_ORIGIN` and `PLAN2CODE_CONSOLE_SID` unset (`env -u ... npm test`): inherited from the app they fail about 34 tests that check behavior outside the app (the update banner, the terminal line, cookies, open's browser launch), and a test console announces itself to the live window. A `cargo test` run mid-build once took the window over that way, then left an agent-less dashboard in it, and the build's question was never seen. The app now ignores sessions outside its console store (`take_announced` in `main.rs`), and the Rust console tests scrub the variables, but the JS suites do not. Before pushing a change under `plan2code-app/`, also run `cargo fmt --check` and `cargo clippy --workspace --all-targets -- -D warnings` there, not only `cargo test`: CI's `app` workflow fails on either, and the Rust code once reached a PR never formatted (20 files) with three clippy errors, one only macOS and Linux could see (see plan2code-app/README.md "Tests"). - Post a helper as `done` or `failed` in the same turn its completion notice arrives: its row stays `running` on the Subagents tab until a post says otherwise (a build once reached its sign-off with one row spinning). A `finish` is now refused while any helper is `running`. - Bring the Windows folder picker (`picker.mjs`, copied into `src/launcher/plan2code.js`) to the front with an Alt tap before `SetForegroundWindow`, never with `windowsHide`, the owner form's `TopMost`, `HWND_TOPMOST` or `AttachThreadInput`: measured on Windows 11, `SetWindowPos(HWND_TOPMOST)` returns true while the dialog stays non-topmost behind the browser, and only the Alt tap made it the foreground window. Check with `GetForegroundWindow` from a second process, since no test can see it. - Add any new file or folder Plan2Code writes at the top of `~/.plan2code/` to `KEPT_PLAN2CODE_ENTRIES` in `src/web-console/sessions.mjs` and to the Cleanup test in `scripts/test-web-console-server.mjs`: User Preferences → Cleanup offers every other entry there for deletion once it is an hour old, and the desktop app's `app/` and `app.json` were about to be listed. diff --git a/plan2code-app/README.md b/plan2code-app/README.md index 168489c..aeed2df 100644 --- a/plan2code-app/README.md +++ b/plan2code-app/README.md @@ -44,7 +44,12 @@ npm run build # build the installable bundle ``` cd plan2code-app +node scripts/vendor.mjs # once: the xterm files cargo builds against +cargo fmt --check +cargo clippy --workspace --all-targets -- -D warnings cargo test --workspace ``` +The `app` workflow runs all three on Windows, macOS and Linux, and fails on any rustfmt difference or clippy warning, so run them before pushing, not only `cargo test`. Clippy on one system does not see code for another: an import used only inside a `#[cfg(windows)]` test is clean on Windows and an unused-import error on macOS and Linux, so put such imports inside the `cfg`'d item. + Real Claude, Devin and Codex are never started by a test. The real window and terminal panel are checked by hand before each release. diff --git a/plan2code-app/RELEASE-CHECKLIST.md b/plan2code-app/RELEASE-CHECKLIST.md index d79ced7..051f752 100644 --- a/plan2code-app/RELEASE-CHECKLIST.md +++ b/plan2code-app/RELEASE-CHECKLIST.md @@ -19,7 +19,8 @@ file directly after checking it against the release's checksum list. - [ ] `node scripts/check-versions.mjs --tag v` passes (version.json, package.json, CHANGELOG.md and the app's tauri.conf.json, package.json and Cargo.toml all agree) - [ ] `npm test` passes at the repo root, with the app absent -- [ ] `cargo test --workspace` passes in `plan2code-app/`, and the `app` workflow is green on all +- [ ] `cargo fmt --check`, `cargo clippy --workspace --all-targets -- -D warnings` and + `cargo test --workspace` pass in `plan2code-app/`, and the `app` workflow is green on all three systems - [ ] The pre-release has one `plan2code-app---` file per system and a `plan2code-app--SHA256SUMS.txt` diff --git a/scripts/test-web-console-page-git.mjs b/scripts/test-web-console-page-git.mjs index 3129946..17807cd 100644 --- a/scripts/test-web-console-page-git.mjs +++ b/scripts/test-web-console-page-git.mjs @@ -14,7 +14,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import fs from "node:fs"; import path from "node:path"; -import { pageSuite, openSession, tempProject, textOf, itemCard } from "./page-test-helpers.mjs"; +import { pageSuite, openSession, tempProject, textOf, itemCard, scrollCard } from "./page-test-helpers.mjs"; import { until } from "./web-console-test-helpers.mjs"; const suite = pageSuite(test); @@ -74,7 +74,7 @@ test("each git tab keeps its own scroll place, across a tab switch and an agent const column = () => page.locator("#main").evaluate((el) => el.scrollTop); await tab("Changes").click(); await page.getByRole("tabpanel").getByText("src/app.js", { exact: true }).waitFor(); - await page.locator("#main").evaluate((el) => (el.scrollTop = 120)); + await scrollCard(page.locator("#main"), 120); const changesAt = await column(); assert.ok(changesAt > 0, "the board's column scrolls"); // Another tab opens at its own place, not where Changes was left. diff --git a/scripts/test-web-console-shape.mjs b/scripts/test-web-console-shape.mjs index 33f2775..0ae92c3 100644 --- a/scripts/test-web-console-shape.mjs +++ b/scripts/test-web-console-shape.mjs @@ -335,3 +335,49 @@ test("shape: the docs-table check names a file with no row, and passes one that fs.rmSync(dir, { recursive: true, force: true }); } }); + +// A page test that reads a style or sets a scroll through locator.evaluate +// acts on whatever element the locator found, and a redraw can replace that +// element before the call runs: one off the page reads "" for every style and +// scrolls nothing anyone sees. On 3 and 4 core CI runners that failed a +// different page test at random on each run until the reads went through +// page-test-helpers.mjs's styleOf(), stylesOf() and scrollCard(), which retry +// until the element is still on the page. page.evaluate and waitForFunction +// look the element up afresh, so they are not flagged. +const RACY_STYLE = /(?[^\n]*getComputedStyle\(\1\)/; +const RACY_SCROLL = /(?\s*\(?\s*\1\.scrollTop\s*=[^=]/; + +function racyPageReads(name, text) { + const problems = []; + text.split("\n").forEach((line, i) => { + if (RACY_STYLE.test(line)) problems.push(`${name}:${i + 1}: read the style with styleOf() or stylesOf() from page-test-helpers.mjs`); + if (RACY_SCROLL.test(line)) problems.push(`${name}:${i + 1}: scroll with scrollCard() from page-test-helpers.mjs`); + }); + return problems; +} + +test("shape: page tests read styles and set scrolls through the redraw-safe helpers", () => { + const dir = path.join(ROOT, "scripts"); + const pageTests = fs.readdirSync(dir).filter((f) => /^test-web-console-page-.*\.mjs$/.test(f)); + assert.ok(pageTests.length > 0, "the page tests were found"); + const problems = pageTests.flatMap((f) => racyPageReads(f, fs.readFileSync(path.join(dir, f), "utf8"))); + assert.deepEqual(problems, []); +}); + +test("shape: the redraw-safe check names a locator's style read and scroll, and passes page-wide ones", () => { + const racy = [ + 'await card.evaluate((el) => (el.scrollTop = 400));', + 'const c = await chip.evaluate((b) => getComputedStyle(b).borderTopStyle);', + ].join("\n"); + assert.deepEqual(racyPageReads("x.mjs", racy), [ + "x.mjs:1: scroll with scrollCard() from page-test-helpers.mjs", + "x.mjs:2: read the style with styleOf() or stylesOf() from page-test-helpers.mjs", + ]); + const fine = [ + 'await s.page.evaluate(() => { document.getElementById("main").scrollTop = 4000; });', + 'const top = await card.evaluate((el) => el.scrollTop === 0);', + 'await s.page.waitForFunction(() => getComputedStyle(document.body).color);', + 'const color = await s.page.evaluate((el) => getComputedStyle(el).color);', + ].join("\n"); + assert.deepEqual(racyPageReads("x.mjs", fine), []); +});