From 23a40d4c2ff74d993b654f558c74a1276244bc51 Mon Sep 17 00:00:00 2001 From: Jack Felke Date: Thu, 5 Mar 2026 00:45:22 -0700 Subject: [PATCH 1/2] feat: add export_report tool for timeline markdown reports Implements #5. Adds a new export_report MCP tool that generates: - Weekly summaries with commit lists, correction rates, and daily breakdowns - Activity reports with event breakdowns, tool usage stats, and heatmaps Supports saving reports to ~/.preflight/reports/ and all existing search scopes (current/related/all). Includes test suite with 4 tests covering registration, empty state, weekly report generation, and activity report heatmap output. --- src/index.ts | 2 + src/tools/export-report.ts | 348 ++++++++++++++++++++++++++++++++++++ tests/export-report.test.ts | 99 ++++++++++ 3 files changed, 449 insertions(+) create mode 100644 src/tools/export-report.ts create mode 100644 tests/export-report.test.ts diff --git a/src/index.ts b/src/index.ts index e7e9d00..81a9b49 100644 --- a/src/index.ts +++ b/src/index.ts @@ -49,6 +49,7 @@ import { registerScanSessions } from "./tools/scan-sessions.js"; import { registerGenerateScorecard } from "./tools/generate-scorecard.js"; import { registerSearchContracts } from "./tools/search-contracts.js"; import { registerEstimateCost } from "./tools/estimate-cost.js"; +import { registerExportReport } from "./tools/export-report.js"; // Validate related projects from config function validateRelatedProjects(): void { @@ -109,6 +110,7 @@ const toolRegistry: Array<[string, RegisterFn]> = [ ["scan_sessions", registerScanSessions], ["generate_scorecard", registerGenerateScorecard], ["estimate_cost", registerEstimateCost], + ["export_report", registerExportReport], ["search_contracts", registerSearchContracts], ]; diff --git a/src/tools/export-report.ts b/src/tools/export-report.ts new file mode 100644 index 0000000..bd0c1f7 --- /dev/null +++ b/src/tools/export-report.ts @@ -0,0 +1,348 @@ +// ============================================================================= +// export_report — Generate markdown reports from timeline data +// Implements: https://github.com/TerminalGravity/preflight/issues/5 +// ============================================================================= + +import { z } from "zod"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { getTimeline, listIndexedProjects } from "../lib/timeline-db.js"; +import { getRelatedProjects } from "../lib/config.js"; +import { writeFileSync, mkdirSync } from "fs"; +import { join } from "path"; +import { homedir } from "os"; +import type { SearchScope } from "../types.js"; + +// --- Helpers --- + +function getWeekStart(date: Date): string { + const d = new Date(date); + d.setDate(d.getDate() - d.getDay()); + return d.toISOString().slice(0, 10); +} + +function daysAgo(n: number): string { + const d = new Date(); + d.setDate(d.getDate() - n); + return d.toISOString(); +} + +async function getSearchProjects(scope: SearchScope): Promise { + const currentProject = process.env.CLAUDE_PROJECT_DIR; + switch (scope) { + case "current": + return currentProject ? [currentProject] : []; + case "related": { + const related = getRelatedProjects(); + return currentProject ? [currentProject, ...related] : related; + } + case "all": { + const projects = await listIndexedProjects(); + return projects.map((p) => p.project); + } + default: + return currentProject ? [currentProject] : []; + } +} + +interface EventRecord { + timestamp?: string; + type: string; + content?: string; + summary?: string; + commit_hash?: string; + tool_name?: string; + metadata?: string; + project_name?: string; +} + +// --- Report generators --- + +function generateWeeklySummary( + events: EventRecord[], + projectName: string, + weekOf: string, +): string { + const counts: Record = {}; + for (const e of events) { + counts[e.type] = (counts[e.type] || 0) + 1; + } + + const commits = events.filter((e) => e.type === "commit"); + const corrections = events.filter((e) => e.type === "correction"); + const prompts = events.filter((e) => e.type === "prompt"); + const errors = events.filter((e) => e.type === "error"); + + // Group by day + const days = new Map(); + for (const e of events) { + const day = e.timestamp + ? new Date(e.timestamp).toISOString().slice(0, 10) + : "unknown"; + if (!days.has(day)) days.set(day, []); + days.get(day)!.push(e); + } + + const lines: string[] = [ + `# Weekly Report: ${projectName}`, + `**Week of ${weekOf}**`, + `_Generated ${new Date().toISOString().slice(0, 10)}_`, + "", + "## Summary", + "", + `| Metric | Count |`, + `|--------|-------|`, + `| Total events | ${events.length} |`, + `| Prompts | ${prompts.length} |`, + `| Commits | ${commits.length} |`, + `| Corrections | ${corrections.length} |`, + `| Errors | ${errors.length} |`, + `| Active days | ${days.size} |`, + "", + ]; + + // Prompt quality signal + if (prompts.length > 0 && corrections.length > 0) { + const correctionRate = ((corrections.length / prompts.length) * 100).toFixed( + 1, + ); + lines.push( + `## Prompt Quality`, + "", + `- Correction rate: **${correctionRate}%** (${corrections.length} corrections / ${prompts.length} prompts)`, + "", + ); + } + + // Commits + if (commits.length > 0) { + lines.push("## Commits", ""); + for (const c of commits) { + const hash = c.commit_hash ? c.commit_hash.slice(0, 7) : "???????"; + const msg = (c.content || c.summary || "").slice(0, 100).replace(/\n/g, " "); + lines.push(`- \`${hash}\` ${msg}`); + } + lines.push(""); + } + + // Corrections (lessons learned) + if (corrections.length > 0) { + lines.push("## Corrections", ""); + for (const c of corrections.slice(0, 10)) { + const msg = (c.content || c.summary || "").slice(0, 120).replace(/\n/g, " "); + lines.push(`- ${msg}`); + } + if (corrections.length > 10) { + lines.push(`- _...and ${corrections.length - 10} more_`); + } + lines.push(""); + } + + // Errors + if (errors.length > 0) { + lines.push("## Errors", ""); + for (const e of errors.slice(0, 5)) { + const msg = (e.content || e.summary || "").slice(0, 120).replace(/\n/g, " "); + lines.push(`- ⚠️ ${msg}`); + } + if (errors.length > 5) { + lines.push(`- _...and ${errors.length - 5} more_`); + } + lines.push(""); + } + + // Daily breakdown + lines.push("## Daily Activity", ""); + const sortedDays = [...days.keys()].sort(); + for (const day of sortedDays) { + const dayEvents = days.get(day)!; + const dayCounts: Record = {}; + for (const e of dayEvents) { + dayCounts[e.type] = (dayCounts[e.type] || 0) + 1; + } + const parts = Object.entries(dayCounts) + .map(([t, c]) => `${c} ${t}${c > 1 ? "s" : ""}`) + .join(", "); + lines.push(`- **${day}**: ${parts}`); + } + lines.push(""); + + return lines.join("\n"); +} + +function generateActivityReport( + events: EventRecord[], + projectName: string, + since: string, + until: string, +): string { + const lines: string[] = [ + `# Activity Report: ${projectName}`, + `**${since.slice(0, 10)} to ${until.slice(0, 10)}**`, + `_Generated ${new Date().toISOString().slice(0, 10)}_`, + "", + ]; + + // Type breakdown + const counts: Record = {}; + for (const e of events) { + counts[e.type] = (counts[e.type] || 0) + 1; + } + + lines.push("## Event Breakdown", "", "| Type | Count |", "|------|-------|"); + for (const [type, count] of Object.entries(counts).sort( + (a, b) => b[1] - a[1], + )) { + lines.push(`| ${type} | ${count} |`); + } + lines.push(""); + + // Tool usage + const toolCalls = events.filter((e) => e.type === "tool_call"); + if (toolCalls.length > 0) { + const toolCounts: Record = {}; + for (const e of toolCalls) { + const name = e.tool_name || "unknown"; + toolCounts[name] = (toolCounts[name] || 0) + 1; + } + lines.push( + "## Tool Usage", + "", + "| Tool | Calls |", + "|------|-------|", + ); + for (const [tool, count] of Object.entries(toolCounts).sort( + (a, b) => b[1] - a[1], + )) { + lines.push(`| ${tool} | ${count} |`); + } + lines.push(""); + } + + // Timeline (condensed) + const days = new Map(); + for (const e of events) { + const day = e.timestamp + ? new Date(e.timestamp).toISOString().slice(0, 10) + : "unknown"; + days.set(day, (days.get(day) || 0) + 1); + } + + lines.push("## Activity Heatmap", ""); + for (const [day, count] of [...days.entries()].sort()) { + const bar = "█".repeat(Math.min(count, 40)); + lines.push(`${day} ${bar} ${count}`); + } + lines.push(""); + + return lines.join("\n"); +} + +// --- Registration --- + +export function registerExportReport(server: McpServer) { + server.tool( + "export_report", + "Generate markdown reports from timeline data. Weekly summaries, activity reports, and prompt quality trends.", + { + scope: z + .enum(["current", "related", "all"]) + .default("current") + .describe("Search scope"), + project: z + .string() + .optional() + .describe("Filter to specific project (overrides scope)"), + format: z + .enum(["weekly", "activity"]) + .default("weekly") + .describe( + "Report format: weekly (7-day summary) or activity (custom range)", + ), + since: z + .string() + .optional() + .describe("Start date (ISO or relative like '7days'). Default: 7 days ago"), + until: z + .string() + .optional() + .describe("End date (ISO or relative). Default: now"), + save: z + .boolean() + .default(false) + .describe("Save to ~/.preflight/reports/"), + }, + async (params) => { + const sinceDate = params.since || daysAgo(7); + const untilDate = params.until || new Date().toISOString(); + + // Resolve projects + let projectDirs: string[]; + if (params.project) { + projectDirs = [params.project]; + } else { + projectDirs = await getSearchProjects(params.scope); + } + + if (projectDirs.length === 0) { + return { + content: [ + { + type: "text" as const, + text: `No projects found for scope "${params.scope}". Onboard a project first.`, + }, + ], + }; + } + + // Fetch events + const events = (await getTimeline({ + project_dirs: projectDirs, + since: sinceDate, + until: untilDate, + limit: 1000, + offset: 0, + })) as EventRecord[]; + + if (events.length === 0) { + return { + content: [ + { + type: "text" as const, + text: "No events found for the given time range.", + }, + ], + }; + } + + const projectName = + params.project || events[0]?.project_name || "Project"; + + let report: string; + if (params.format === "weekly") { + const weekOf = getWeekStart(new Date(sinceDate)); + report = generateWeeklySummary(events, projectName, weekOf); + } else { + report = generateActivityReport( + events, + projectName, + sinceDate, + untilDate, + ); + } + + // Optionally save + if (params.save) { + const reportsDir = join(homedir(), ".preflight", "reports"); + mkdirSync(reportsDir, { recursive: true }); + const filename = `${params.format}-${new Date().toISOString().slice(0, 10)}.md`; + const filepath = join(reportsDir, filename); + writeFileSync(filepath, report, "utf-8"); + report += `\n---\n_Saved to ${filepath}_\n`; + } + + return { + content: [{ type: "text" as const, text: report }], + }; + }, + ); +} diff --git a/tests/export-report.test.ts b/tests/export-report.test.ts new file mode 100644 index 0000000..e4a9079 --- /dev/null +++ b/tests/export-report.test.ts @@ -0,0 +1,99 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; + +// Mock timeline-db before importing the module +vi.mock("../src/lib/timeline-db.js", () => ({ + getTimeline: vi.fn(), + listIndexedProjects: vi.fn().mockResolvedValue([]), +})); + +vi.mock("../src/lib/config.js", () => ({ + getRelatedProjects: vi.fn().mockReturnValue([]), +})); + +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { getTimeline } from "../src/lib/timeline-db.js"; +import { registerExportReport } from "../src/tools/export-report.js"; + +describe("export_report", () => { + let server: McpServer; + let registeredHandler: any; + + beforeEach(() => { + vi.clearAllMocks(); + // Capture the handler when tool is registered + server = { + tool: vi.fn((name, desc, schema, handler) => { + registeredHandler = handler; + }), + } as any; + registerExportReport(server); + }); + + it("registers the tool", () => { + expect(server.tool).toHaveBeenCalledWith( + "export_report", + expect.any(String), + expect.any(Object), + expect.any(Function), + ); + }); + + it("returns no-events message for empty timeline", async () => { + vi.mocked(getTimeline).mockResolvedValue([]); + process.env.CLAUDE_PROJECT_DIR = "/test/project"; + + const result = await registeredHandler({ + scope: "current", + format: "weekly", + save: false, + }); + + expect(result.content[0].text).toContain("No events found"); + }); + + it("generates weekly report with commits and corrections", async () => { + const now = new Date().toISOString(); + vi.mocked(getTimeline).mockResolvedValue([ + { timestamp: now, type: "prompt", content: "fix the bug", project_name: "test-proj" }, + { timestamp: now, type: "commit", content: "fix: resolved bug", commit_hash: "abc1234def" }, + { timestamp: now, type: "correction", content: "wrong approach first time" }, + { timestamp: now, type: "error", content: "type error in foo.ts" }, + ]); + process.env.CLAUDE_PROJECT_DIR = "/test/project"; + + const result = await registeredHandler({ + scope: "current", + format: "weekly", + save: false, + }); + + const text = result.content[0].text; + expect(text).toContain("Weekly Report"); + expect(text).toContain("abc1234"); + expect(text).toContain("wrong approach"); + expect(text).toContain("Correction rate"); + expect(text).toContain("type error in foo.ts"); + }); + + it("generates activity report with heatmap", async () => { + const events = Array.from({ length: 10 }, (_, i) => ({ + timestamp: new Date(Date.now() - i * 3600000).toISOString(), + type: "prompt", + content: `prompt ${i}`, + project_name: "test-proj", + })); + vi.mocked(getTimeline).mockResolvedValue(events); + process.env.CLAUDE_PROJECT_DIR = "/test/project"; + + const result = await registeredHandler({ + scope: "current", + format: "activity", + save: false, + }); + + const text = result.content[0].text; + expect(text).toContain("Activity Report"); + expect(text).toContain("Activity Heatmap"); + expect(text).toContain("█"); + }); +}); From 511c2272002e7e36a8a048da682d01cca1842bca Mon Sep 17 00:00:00 2001 From: Jack Felke Date: Thu, 5 Mar 2026 01:15:10 -0700 Subject: [PATCH 2/2] fix(prompt-score): remove false scope credit for long prompts Previously, any prompt over 100 chars got full scope score (25/25) regardless of actual scope clarity. Now scope scoring only rewards explicit bounding words (only, just, single, specific, this) and penalizes broad words (all, every, entire, whole). Also exports scorePrompt for testability and adds 10 tests covering all scoring dimensions, edge cases, and the regression. --- src/tools/prompt-score.ts | 12 +++++-- tests/prompt-score.test.ts | 71 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) create mode 100644 tests/prompt-score.test.ts diff --git a/src/tools/prompt-score.ts b/src/tools/prompt-score.ts index 1cecf01..272bc52 100644 --- a/src/tools/prompt-score.ts +++ b/src/tools/prompt-score.ts @@ -40,7 +40,7 @@ interface ScoreResult { feedback: string[]; } -function scorePrompt(text: string): ScoreResult { +export function scorePrompt(text: string): ScoreResult { const feedback: string[] = []; let specificity: number; let scope: number; @@ -59,9 +59,15 @@ function scorePrompt(text: string): ScoreResult { } // Scope: bounded task - if (/\b(only|just|single|one|specific|this)\b/i.test(text) || text.length > 100) { + const hasBoundingWords = /\b(only|just|single|one|specific|this)\b/i.test(text); + const hasBroadWords = /\b(all|every|entire|whole)\b/i.test(text); + if (hasBoundingWords && !hasBroadWords) { scope = 25; - } else if (/\b(all|every|entire|whole)\b/i.test(text)) { + } else if (hasBoundingWords && hasBroadWords) { + // Mixed signals — e.g. "only update all tests" — give partial credit + scope = 18; + feedback.push("🎯 Mixed scope signals — try to narrow what 'all' applies to"); + } else if (hasBroadWords) { scope = 10; feedback.push("🎯 'All/every' is broad — can you narrow the scope?"); } else { diff --git a/tests/prompt-score.test.ts b/tests/prompt-score.test.ts new file mode 100644 index 0000000..1e8fec2 --- /dev/null +++ b/tests/prompt-score.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect } from "vitest"; +import { scorePrompt } from "../src/tools/prompt-score.js"; + +describe("scorePrompt", () => { + it("gives high score to a specific, scoped, actionable prompt with done condition", () => { + const result = scorePrompt( + "Rename the `handleClick` function in `src/components/Button.tsx` to `onButtonPress`. It should pass the existing tests." + ); + expect(result.grade).toMatch(/^[AB]/); + expect(result.specificity).toBe(25); + expect(result.actionability).toBe(25); + expect(result.doneCondition).toBe(25); + }); + + it("gives low score to a vague prompt", () => { + const result = scorePrompt("make it work"); + expect(result.total).toBeLessThan(50); + expect(result.grade).toMatch(/^[DF]/); + expect(result.feedback.length).toBeGreaterThan(0); + }); + + it("penalizes broad scope words like 'all' and 'every'", () => { + const broad = scorePrompt("fix all the bugs"); + const narrow = scorePrompt("fix only this bug"); + expect(narrow.scope).toBeGreaterThan(broad.scope); + }); + + it("does NOT give full scope score just because prompt is long", () => { + // This was a bug: text.length > 100 used to grant scope = 25 + const longVague = scorePrompt( + "I need you to look at the code and figure out what might be going wrong with it because something seems off and I am not sure what the problem is exactly" + ); + expect(longVague.scope).toBeLessThan(25); + }); + + it("handles mixed scope signals", () => { + const result = scorePrompt("only update all the test files"); + expect(result.scope).toBe(18); + expect(result.feedback.some((f) => f.includes("Mixed scope"))).toBe(true); + }); + + it("recognizes file paths as specific", () => { + const result = scorePrompt("check src/lib/utils.ts"); + expect(result.specificity).toBe(25); + }); + + it("recognizes backtick identifiers as specific", () => { + const result = scorePrompt("refactor `parseConfig` to use zod"); + expect(result.specificity).toBe(25); + expect(result.actionability).toBe(25); + }); + + it("gives done-condition credit for questions", () => { + const result = scorePrompt("Why is `fetchData` broken?"); + expect(result.doneCondition).toBe(20); + }); + + it("gives perfect feedback for an excellent prompt", () => { + const result = scorePrompt( + "Add just one test in `tests/auth.test.ts` for the `validateToken` function. It should return false when the token is expired." + ); + expect(result.feedback).toEqual(["🏆 Excellent prompt! Clear target, scope, action, and done condition."]); + }); + + it("returns numeric total as sum of dimensions", () => { + const result = scorePrompt("do stuff"); + expect(result.total).toBe( + result.specificity + result.scope + result.actionability + result.doneCondition + ); + }); +});