Refuse run ids that could leave the runs folder and stop replacements being read as patterns

AI Assisted
This commit is contained in:
jparkerweb committed 2026-11-05 11:38:58 -08:00
1 parent b17213322c
commit 0b6a376915
8 files changed
+37 -13

No files matched your search

+14 -1
View File
@@ -2,7 +2,7 @@ import fs from 'fs';
import os from 'os';
import path from 'path';
import { describe, it, expect, afterEach } from 'vitest';
import { avg, rate, buildCohortKey, backfillPromptVersions, cohortKeyForRun, aggregate } from './aggregator.js';
import { avg, rate, buildCohortKey, backfillPromptVersions, cohortKeyForRun, aggregate, writeRunFile } from './aggregator.js';
import type { PromptVersions, RunMetrics } from './types.js';
// ── avg() ─────────────────────────────────────────────────────────────────────
@@ -251,3 +251,16 @@ describe('aggregate', () => {
expect(result.current_cohort_key).toBe(buildCohortKey(local.prompt_versions));
});
});
describe('writeRunFile', () => {
it('refuses a run_id that is not a run-<id> name, so it cannot leave the runs folder', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'p2c-runs-'));
try {
const run = { ...makeRun({}), run_id: '../../escape' };
expect(() => writeRunFile(run, path.join(dir, 'runs'))).toThrow(/run-<id>/);
expect(fs.existsSync(path.join(dir, 'escape.json'))).toBe(false);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
});
+4
View File
@@ -230,6 +230,10 @@ export function loadAggregated(outputPath: string): AggregatedMetrics | null {
* Returns true if written, false if a file for that run_id already existed.
*/
export function writeRunFile(run: RunMetrics, runsDir: string): boolean {
// The id becomes a file name: only the collector's own shape may reach a path.
if (!/^run-[A-Za-z0-9-]{1,64}$/.test(run.run_id)) {
throw new Error(`Refusing run_id "${String(run.run_id).slice(0, 80)}": it is not a run-<id> name.`);
}
const destPath = path.join(runsDir, `${run.run_id}.json`);
if (fs.existsSync(destPath)) {
+3 -3
View File
@@ -5,11 +5,11 @@
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import type { AggregatedMetrics } from './types.js';
import { invokeLLM, type AgentType } from './invoke-llm.js';
const ANALYZE_PROMPT_PATH = new URL('../src/prompts/analyze.md', import.meta.url).pathname
.replace(/^\/([A-Za-z]:)/, '$1'); // Fix Windows path
const ANALYZE_PROMPT_PATH = fileURLToPath(new URL('../src/prompts/analyze.md', import.meta.url));
// ── Helpers ───────────────────────────────────────────────────────────────────
@@ -40,7 +40,7 @@ function readPromptFiles(plan2codeRoot: string): Record<string, string> {
function interpolate(template: string, vars: Record<string, string>): string {
let result = template;
for (const [key, value] of Object.entries(vars)) {
result = result.replaceAll(`{{${key}}}`, value);
result = result.replaceAll(`{{${key}}}`, () => value);
}
return result;
}
+2 -2
View File
@@ -67,7 +67,7 @@ function applyEdit(edit: PromptEdit, plan2codeRoot: string): boolean {
console.error(chalk.red(`✗ Cannot apply: old_text not found in ${edit.file} (may have been modified by a previous edit)`));
return false;
}
content = content.replace(edit.old_text, edit.new_text);
content = content.replace(edit.old_text, () => edit.new_text);
// Post-apply char count check
if (content.length > CHAR_LIMIT) {
@@ -173,7 +173,7 @@ export async function reviewAndApply(opts: ApplierOptions): Promise<ApplierResul
if (success) {
applied++;
// Update in-memory content to reflect the edit
promptContents[edit.file] = promptContents[edit.file].replace(edit.old_text, edit.new_text);
promptContents[edit.file] = promptContents[edit.file].replace(edit.old_text, () => edit.new_text);
} else {
failed++;
}
+6
View File
@@ -87,6 +87,12 @@ describe('parseSubmissionPayload', () => {
expect(result).toBeNull();
});
it('returns null when run_id is not a run-<id> name (it becomes a file name)', () => {
for (const run_id of ['../../x', 'run-../x', 'x', 'run-' + 'a'.repeat(65), 'run-a/b']) {
expect(parseSubmissionPayload(issueBody({ ...VALID_PAYLOAD, run_id }))).toBeNull();
}
});
it('returns null when user_feedback.overall_rating is a string instead of a number', () => {
const badPayload = {
...VALID_PAYLOAD,
+2 -1
View File
@@ -242,7 +242,8 @@ export function parseSubmissionPayload(body: string): RunMetrics | null {
if (!parsed) return null;
if (parsed['schema_version'] !== '1.0') return null;
if (!isString(parsed['run_id'])) return null;
// The id becomes a file name in the runs folder: only the collector's own shape.
if (!isString(parsed['run_id']) || !/^run-[A-Za-z0-9-]{1,64}$/.test(parsed['run_id'])) return null;
if (!isString(parsed['plan2code_version'])) return null;
const feedbackRaw = parsed['user_feedback'];
+1 -1
View File
@@ -128,7 +128,7 @@ describe('parseProposalFromResponse', () => {
});
it('returns null for malformed JSON', () => {
const result = parseProposalFromResponse('```json\n{broken json]\n```');
const result = parseProposalFromResponse('```json\n[{"old_text": broken]\n```');
expect(result).toBeNull();
});
+5 -5
View File
@@ -6,19 +6,19 @@
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import type { PromptEdit, PromptProposal } from './types.js';
import { invokeLLM, type AgentType } from './invoke-llm.js';
const CHAR_LIMIT = 20_000;
const IMPROVE_PROMPT_PATH = new URL('../src/prompts/improve.md', import.meta.url).pathname
.replace(/^\/([A-Za-z]:)/, '$1'); // Fix Windows path
const IMPROVE_PROMPT_PATH = fileURLToPath(new URL('../src/prompts/improve.md', import.meta.url));
// ── Helpers ───────────────────────────────────────────────────────────────────
function interpolate(template: string, vars: Record<string, string>): string {
let result = template;
for (const [key, value] of Object.entries(vars)) {
result = result.replaceAll(`{{${key}}}`, value);
result = result.replaceAll(`{{${key}}}`, () => value);
}
return result;
}
@@ -93,7 +93,7 @@ export function validateEdit(
}
// Check char count after edit
const afterContent = fileContent.replace(edit.old_text, edit.new_text);
const afterContent = fileContent.replace(edit.old_text, () => edit.new_text);
if (afterContent.length > CHAR_LIMIT) {
errors.push(`Edit would cause "${edit.file}" to exceed ${CHAR_LIMIT} char limit (would be ${afterContent.length} chars).`);
}
@@ -232,7 +232,7 @@ export async function generateImprovement(opts: ImproverOptions): Promise<Improv
if (result.valid) {
// Compute accurate char counts
const fileContent = promptContents[edit.file] ?? '';
const afterContent = fileContent.replace(edit.old_text, edit.new_text);
const afterContent = fileContent.replace(edit.old_text, () => edit.new_text);
edit.char_count_before = fileContent.length;
edit.char_count_after = afterContent.length;
edit.char_count_delta = afterContent.length - fileContent.length;