mirror of
https://github.com/jparkerweb/plan2code.git
synced 2026-11-16 20:45:33 -08:00
167 lines
5.1 KiB
TypeScript
167 lines
5.1 KiB
TypeScript
// All TypeScript interfaces for plan2code-metrics
|
|
|
|
/**
|
|
* SHA-256 fingerprint of every src/plan2code-*.md prompt, keyed by name
|
|
* (promptKey in collector.ts). The eight prompts recorded before the
|
|
* collector read the folder keep their old keys: plan, revise_plan, document,
|
|
* implement, finalize, init, init_update, quick_task.
|
|
*/
|
|
export type PromptVersions = Record<string, string>;
|
|
|
|
export interface Step1PlanMetrics {
|
|
present: boolean;
|
|
final_confidence: number | null;
|
|
confidence_breakdown: {
|
|
requirements: number | null;
|
|
feasibility: number | null;
|
|
integration: number | null;
|
|
risk: number | null;
|
|
} | null;
|
|
clarification_rounds: number | null;
|
|
tech_stack_revision_rounds: number | null;
|
|
verification_gaps_found: number | null;
|
|
functional_requirements_count: number | null;
|
|
non_functional_requirements_count: number | null;
|
|
risk_count: number | null;
|
|
phase_count: number | null;
|
|
}
|
|
|
|
export interface Step2DocumentMetrics {
|
|
present: boolean;
|
|
total_tasks: number | null;
|
|
tasks_per_phase: number[] | null;
|
|
phase_count: number | null;
|
|
parallel_groups_identified: number | null;
|
|
requirement_coverage_percent: number | null;
|
|
verification_items_added: number | null;
|
|
}
|
|
|
|
export interface Step3ImplementMetrics {
|
|
present: boolean;
|
|
task_completion_rate: number | null;
|
|
tasks_completed: number | null;
|
|
tasks_total: number | null;
|
|
blocker_count: number | null;
|
|
}
|
|
|
|
export interface Step4FinalizeMetrics {
|
|
present: boolean;
|
|
completion_rate_at_audit: number | null;
|
|
verification_failures_found: number | null;
|
|
documentation_updates_needed: number | null;
|
|
archival_succeeded: boolean | null;
|
|
}
|
|
|
|
export interface UserFeedback {
|
|
overall_rating: number; // 1-10
|
|
rating_reason: string;
|
|
what_went_well: string;
|
|
what_went_poorly: string;
|
|
}
|
|
|
|
export interface RunMetrics {
|
|
schema_version: '1.0';
|
|
run_id: string;
|
|
plan2code_version: string;
|
|
// Origin of the run. Local runs are collected on the maintainer's machine;
|
|
// community runs are ingested from GitHub feedback issues. Absent on pre-v1.17
|
|
// run files, which are treated as 'local'. Drives cohort keying (see
|
|
// cohortKeyForRun in aggregator.ts).
|
|
source?: 'local' | 'community';
|
|
prompt_versions: PromptVersions;
|
|
project: {
|
|
name: string;
|
|
started_at: string | null;
|
|
completed_at: string | null;
|
|
};
|
|
step1_plan: Step1PlanMetrics;
|
|
step2_document: Step2DocumentMetrics;
|
|
step3_implement: Step3ImplementMetrics;
|
|
step4_finalize: Step4FinalizeMetrics;
|
|
user_feedback: UserFeedback | null;
|
|
}
|
|
|
|
export interface PromptEdit {
|
|
file: string;
|
|
rationale: string;
|
|
expected_metric_impact: string;
|
|
char_count_before: number;
|
|
char_count_after: number;
|
|
char_count_delta: number;
|
|
old_text: string;
|
|
new_text: string;
|
|
}
|
|
|
|
export interface PromptProposal {
|
|
proposal_id: string;
|
|
created_at: string;
|
|
based_on_runs: string[];
|
|
analyst_model: string;
|
|
proposals: PromptEdit[];
|
|
status: 'pending' | 'applied' | 'rejected';
|
|
diagnosis_file: string | null;
|
|
}
|
|
|
|
// Aggregated metrics schema
|
|
export interface CohortMetrics {
|
|
cohort_key: string; // local: hash of sorted prompt_versions; community: `community:v<version>`
|
|
source?: 'local' | 'community';
|
|
prompt_versions: PromptVersions;
|
|
run_count: number;
|
|
run_ids: string[];
|
|
first_seen: string;
|
|
last_seen: string;
|
|
|
|
// Step 1 averages
|
|
avg_confidence: number | null;
|
|
avg_clarification_rounds: number | null;
|
|
avg_verification_gaps_found: number | null;
|
|
avg_functional_requirements_count: number | null;
|
|
avg_non_functional_requirements_count: number | null;
|
|
avg_risk_count: number | null;
|
|
avg_phase_count_step1: number | null;
|
|
|
|
// Step 2 averages
|
|
avg_total_tasks: number | null;
|
|
avg_phase_count_step2: number | null;
|
|
avg_parallel_groups: number | null;
|
|
avg_requirement_coverage_percent: number | null;
|
|
avg_verification_items_added: number | null;
|
|
|
|
// Step 3 averages
|
|
avg_task_completion_rate: number | null;
|
|
avg_blocker_count: number | null;
|
|
|
|
// Step 4 averages
|
|
avg_completion_rate_at_audit: number | null;
|
|
avg_verification_failures_found: number | null;
|
|
avg_documentation_updates_needed: number | null;
|
|
archival_success_rate: number | null;
|
|
|
|
// User feedback
|
|
avg_user_rating: number | null;
|
|
feedback_count: number;
|
|
}
|
|
|
|
export interface AggregatedMetrics {
|
|
schema_version: '1.0';
|
|
last_updated: string;
|
|
total_runs: number;
|
|
cohorts: CohortMetrics[];
|
|
current_cohort_key: string | null;
|
|
}
|
|
|
|
// Metric targets for health assessment
|
|
export const METRIC_TARGETS = {
|
|
avg_confidence: { target: 90, direction: 'gte' as const },
|
|
avg_clarification_rounds: { target: 2.0, direction: 'lte' as const },
|
|
avg_verification_gaps_found: { target: 2.0, direction: 'lte' as const },
|
|
avg_parallel_groups: { target: 0.5, direction: 'gte' as const },
|
|
avg_verification_items_added: { target: 1.5, direction: 'lte' as const },
|
|
avg_task_completion_rate: { target: 0.95, direction: 'gte' as const },
|
|
avg_blocker_count: { target: 1.5, direction: 'lte' as const },
|
|
avg_verification_failures_found: { target: 1.0, direction: 'lte' as const },
|
|
archival_success_rate: { target: 0.99, direction: 'gte' as const },
|
|
avg_user_rating: { target: 7.0, direction: 'gte' as const },
|
|
} as const;
|