Ingest community feedback submissions in plan2code-metrics

Adds a CLI flow that lists open community-feedback issues via gh, validates
and parses each METRICS_JSON payload, imports them deduped by run_id,
re-aggregates and closes the issue. Community runs cohort by Plan2Code
version rather than prompt fingerprint, and local cohorts stay preferred so
ingested feedback never displaces the maintainer's current generation. Adds
Devin CLI as an AI backend.

AI Assisted
This commit is contained in:
jparkerweb committed 2026-08-08 12:39:28 -07:00
1 parent 987f7b8b04
commit 1d88d787a3
9 files changed
+788 -26

No files matched your search

+26 -2
View File
@@ -1,7 +1,8 @@
/**
* invoke-llm.ts
* Unified LLM invocation for plan2code-metrics.
* Supports Claude Code (temp file → stdin) and Copilot CLI (stdin string).
* Supports Claude Code (temp file → stdin), Copilot CLI (stdin string), and
* Devin CLI (temp prompt file).
* Mirrors the agent pattern from plan2code-loop.
*/
@@ -12,7 +13,7 @@ import { tmpdir } from 'os';
// ── Agent definitions ────────────────────────────────────────────────────────
export type AgentType = 'claude-code' | 'copilot-cli';
export type AgentType = 'claude-code' | 'copilot-cli' | 'devin-cli';
export interface AgentDef {
name: AgentType;
@@ -34,6 +35,12 @@ export const AGENTS: Record<AgentType, AgentDef> = {
command: 'copilot',
defaultModel: 'claude-sonnet-4',
},
'devin-cli': {
name: 'devin-cli',
displayName: 'Devin CLI',
command: 'devin',
defaultModel: 'default',
},
};
// ── Invocation ───────────────────────────────────────────────────────────────
@@ -72,6 +79,23 @@ export async function invokeLLM(opts: InvokeLLMOptions): Promise<string> {
} finally {
try { unlinkSync(tempFile); } catch { /* ignore cleanup errors */ }
}
} else if (agent === 'devin-cli') {
// Devin CLI: load prompt from a temp file, run single-turn, auto-approve tool calls
const tempFile = join(tmpdir(), `plan2code-metrics-prompt-${Date.now()}.txt`);
writeFileSync(tempFile, prompt, 'utf-8');
try {
const args: string[] = ['--print', '--prompt-file', tempFile, '--permission-mode', 'dangerous'];
// Only add --model if not using default
if (model && model !== 'default') {
args.push('--model', model);
}
const result = await execa(def.command, args, { timeout });
return result.stdout;
} finally {
try { unlinkSync(tempFile); } catch { /* ignore cleanup errors */ }
}
} else {
// Copilot CLI: pipe prompt via stdin string
const args: string[] = [];