Skip to content

Commit f6fa433

Browse files
devakoneclaude
andcommitted
feat: add AI coding tool detection with per-tool metrics and VCP display
Detect which AI coding tools (Claude, Copilot, Cursor, Aider, etc.) are used in commits via Co-Authored-By trailer parsing. Compute per-tool breakdowns, collaboration rates, and confidence levels. Display in repo VCPs, unified dashboard, and public profiles with a settings toggle. - Add AI_TOOL_REGISTRY (11 tools) and identifyAITool() to core engine - Add AIToolMetrics type, extractAIToolMetrics(), and cross-repo aggregation - Add ai_tools_json column to vibe_insights via migration - Store AI tool metrics in Inngest worker pipeline - Add VCPAIToolsSection component for repo, unified, and public VCPs - Add show_ai_tools toggle to public profile settings (default: on) - Add architecture doc, update PRD, methodology page, and pipeline docs Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 6276317 commit f6fa433

22 files changed

Lines changed: 1164 additions & 18 deletions

File tree

apps/web/src/app/analysis/[jobId]/AnalysisClient.tsx

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
import Link from "next/link";
44
import { useEffect, useMemo, useState } from "react";
55
import { computeAnalysisInsights } from "@vibed/core";
6-
import type { AnalysisInsights, AnalysisMetrics, CommitEvent } from "@vibed/core";
6+
import type { AnalysisInsights, AnalysisMetrics, CommitEvent, AIToolMetrics } from "@vibed/core";
77
import { formatMetricLabel, formatMetricValue } from "@/lib/format-labels";
88
import { computeShareCardMetrics } from "@/lib/vcp/metrics";
99
import { isVibeAxes } from "@/lib/vcp/validators";
@@ -15,6 +15,7 @@ import {
1515
RepoMetricsGrid,
1616
ProfileContributionCard,
1717
} from "@/components/vcp/repo";
18+
import { VCPAIToolsSection } from "@/components/vcp/blocks";
1819

1920
type Job = {
2021
id: string;
@@ -103,6 +104,7 @@ type VibeInsightsRow = {
103104
persona_tagline: string | null;
104105
persona_confidence: string;
105106
persona_score: number | null;
107+
ai_tools_json?: unknown;
106108
};
107109

108110
type InsightHistoryEntry = {
@@ -453,6 +455,33 @@ export default function AnalysisClient({ jobId }: { jobId: string }) {
453455
const persona = wrapped.persona;
454456
const shareTemplate = wrapped.share_template;
455457

458+
// AI tool metrics — prefer vibe_insights data, fall back to analysis_insights signals
459+
const aiToolMetrics = useMemo<AIToolMetrics | null>(() => {
460+
const vibeAiTools = parsedVibeInsights?.ai_tools_json;
461+
if (vibeAiTools && typeof vibeAiTools === "object" && "detected" in (vibeAiTools as Record<string, unknown>)) {
462+
return vibeAiTools as unknown as AIToolMetrics;
463+
}
464+
// Fall back to tool_co_authors from analysis insights
465+
const toolCoAuthors = wrapped.multi_agent_signals?.tool_co_authors;
466+
if (!toolCoAuthors || toolCoAuthors.length === 0) return null;
467+
const totalAiCommits = toolCoAuthors.reduce((s, t) => s + t.commit_count, 0);
468+
const totalCommits = wrapped.totals?.commits ?? 0;
469+
return {
470+
detected: true,
471+
ai_assisted_commits: totalAiCommits,
472+
ai_collaboration_rate: totalCommits > 0 ? totalAiCommits / totalCommits : 0,
473+
primary_tool: { id: toolCoAuthors[0].tool_id, name: toolCoAuthors[0].tool_name },
474+
tool_diversity: toolCoAuthors.length,
475+
tools: toolCoAuthors.map((t) => ({
476+
tool_id: t.tool_id,
477+
tool_name: t.tool_name,
478+
commit_count: t.commit_count,
479+
percentage: totalAiCommits > 0 ? Math.round((t.commit_count / totalAiCommits) * 100) : 0,
480+
})),
481+
confidence: totalAiCommits >= 10 ? "high" : totalAiCommits >= 3 ? "medium" : "low",
482+
} satisfies AIToolMetrics;
483+
}, [parsedVibeInsights, wrapped]);
484+
456485
const shareText = useMemo(() => {
457486
if (!shareTemplate) return "";
458487
const metricsLine = shareTemplate.metrics
@@ -1063,6 +1092,13 @@ export default function AnalysisClient({ jobId }: { jobId: string }) {
10631092
)}
10641093
</div>
10651094

1095+
{/* AI Coding Tools Section */}
1096+
{aiToolMetrics?.detected ? (
1097+
<div className="mt-6 rounded-2xl border border-black/5 bg-white/60 p-5 backdrop-blur">
1098+
<VCPAIToolsSection aiTools={aiToolMetrics} />
1099+
</div>
1100+
) : null}
1101+
10661102
{profileContribution ? (
10671103
<ProfileContributionCard
10681104
contribution={profileContribution}

apps/web/src/app/api/analysis/[id]/route.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -324,7 +324,7 @@ export async function GET(
324324
const { data: v } = await (supabase as unknown as SupabaseQueryLike)
325325
.from("vibe_insights")
326326
.select(
327-
"job_id, axes_json, persona_id, persona_name, persona_tagline, persona_confidence, persona_score, cards_json"
327+
"job_id, axes_json, persona_id, persona_name, persona_tagline, persona_confidence, persona_score, cards_json, ai_tools_json"
328328
)
329329
.eq("job_id", id)
330330
.single();
@@ -402,6 +402,7 @@ export async function GET(
402402
persona_confidence: computed.persona.confidence,
403403
persona_score: computed.persona.score,
404404
cards_json: computed.cards,
405+
ai_tools_json: computed.ai_tools,
405406
};
406407
}
407408
return null;

apps/web/src/app/api/public/[username]/route.ts

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { NextResponse } from "next/server";
22
import { createSupabaseServiceClient } from "@/lib/supabase/service";
33
import type { PublicProfileSettings } from "@/types/public-profile";
44
import { DEFAULT_PUBLIC_PROFILE_SETTINGS } from "@/types/public-profile";
5+
import type { AIToolMetrics } from "@vibed/core";
56

67
export const runtime = "nodejs";
78

@@ -41,7 +42,7 @@ export async function GET(
4142
const { data: profile } = await service
4243
.from("user_profiles")
4344
.select(
44-
"persona_id, persona_name, persona_tagline, persona_confidence, persona_score, total_repos, total_commits, axes_json, cards_json, repo_personas_json, narrative_json"
45+
"persona_id, persona_name, persona_tagline, persona_confidence, persona_score, total_repos, total_commits, axes_json, cards_json, repo_personas_json, narrative_json, job_ids"
4546
)
4647
.eq("user_id", user.id)
4748
.maybeSingle();
@@ -99,6 +100,62 @@ export async function GET(
99100

100101
const narrative = profile.narrative_json as { insight?: string; summary?: string } | null;
101102

103+
// Aggregate AI tool metrics from vibe_insights
104+
let aiTools: AIToolMetrics | null = null;
105+
if (settings.show_ai_tools) {
106+
const jobIds = Array.isArray(profile.job_ids)
107+
? (profile.job_ids as string[]).filter((id): id is string => typeof id === "string")
108+
: [];
109+
110+
if (jobIds.length > 0) {
111+
const { data: vibeRows } = await service
112+
.from("vibe_insights")
113+
.select("ai_tools_json")
114+
.in("job_id", jobIds);
115+
116+
if (vibeRows && vibeRows.length > 0) {
117+
const toolCounts = new Map<string, { name: string; count: number }>();
118+
let totalAiCommits = 0;
119+
120+
for (const row of vibeRows) {
121+
const tools = row.ai_tools_json as AIToolMetrics | null;
122+
if (!tools || !tools.detected) continue;
123+
totalAiCommits += tools.ai_assisted_commits;
124+
for (const tool of tools.tools) {
125+
const existing = toolCounts.get(tool.tool_id);
126+
if (existing) {
127+
existing.count += tool.commit_count;
128+
} else {
129+
toolCounts.set(tool.tool_id, { name: tool.tool_name, count: tool.commit_count });
130+
}
131+
}
132+
}
133+
134+
if (toolCounts.size > 0) {
135+
const totalCommits = profile.total_commits ?? 0;
136+
const tools = Array.from(toolCounts.entries())
137+
.map(([id, data]) => ({
138+
tool_id: id,
139+
tool_name: data.name,
140+
commit_count: data.count,
141+
percentage: totalAiCommits > 0 ? Math.round((data.count / totalAiCommits) * 100) : 0,
142+
}))
143+
.sort((a, b) => b.commit_count - a.commit_count);
144+
145+
aiTools = {
146+
detected: true,
147+
ai_assisted_commits: totalAiCommits,
148+
ai_collaboration_rate: totalCommits > 0 ? totalAiCommits / totalCommits : 0,
149+
primary_tool: { id: tools[0].tool_id, name: tools[0].tool_name },
150+
tool_diversity: tools.length,
151+
tools,
152+
confidence: totalAiCommits >= 10 ? "high" : totalAiCommits >= 3 ? "medium" : "low",
153+
};
154+
}
155+
}
156+
}
157+
}
158+
102159
const response = {
103160
username: user.username,
104161
avatar_url: settings.show_avatar ? user.avatar_url : null,
@@ -114,6 +171,7 @@ export async function GET(
114171
narrative: settings.show_narrative ? (narrative?.insight ?? narrative?.summary ?? null) : null,
115172
insight_cards: settings.show_insight_cards ? (profile.cards_json as unknown[] | null) : null,
116173
repo_breakdown: repoBreakdown,
174+
ai_tools: aiTools,
117175
settings,
118176
};
119177

apps/web/src/app/methodology/page.tsx

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ export default async function MethodologyPage() {
2929
<li>Commit metadata: timestamps, files changed, additions/deletions.</li>
3030
<li>Commit message subjects: lightweight patterns like feat/fix/test/docs.</li>
3131
<li>Changed file paths when available (to infer which subsystems changed together).</li>
32+
<li>Commit trailers: Co-Authored-By attribution (used to detect AI tool usage).</li>
3233
<li>PR metadata when available: changed-files counts, issue linking, checklists.</li>
3334
</ul>
3435
<p className="text-sm text-zinc-700">
@@ -140,7 +141,39 @@ export default async function MethodologyPage() {
140141
</section>
141142

142143
<section className={`${wrappedTheme.card} space-y-4 p-6`}>
143-
<h2 className="text-lg font-semibold text-zinc-950">5) Why it can be wrong</h2>
144+
<h2 className="text-lg font-semibold text-zinc-950">5) AI tool detection</h2>
145+
<p className="text-sm text-zinc-700">
146+
We detect which AI coding tools you use by parsing{" "}
147+
<code className="rounded bg-zinc-100 px-1 py-0.5 text-xs">Co-Authored-By</code>{" "}
148+
trailers in your commit messages. Many AI tools (Claude Code, GitHub Copilot, Cursor,
149+
Aider, and others) automatically add these trailers when they help write code.
150+
</p>
151+
<p className="text-sm text-zinc-700">
152+
From these trailers we compute:
153+
</p>
154+
<ul className="list-disc space-y-2 pl-5 text-sm text-zinc-700">
155+
<li>
156+
<span className="font-semibold text-zinc-950">AI collaboration rate</span> — the
157+
fraction of your commits that have AI co-authorship.
158+
</li>
159+
<li>
160+
<span className="font-semibold text-zinc-950">Primary tool</span> — the AI tool that
161+
appears most frequently in your history.
162+
</li>
163+
<li>
164+
<span className="font-semibold text-zinc-950">Tool breakdown</span> — per-tool usage
165+
percentages across all detected tools.
166+
</li>
167+
</ul>
168+
<p className="text-sm text-zinc-700">
169+
We currently recognize 11 tools: Claude, GitHub Copilot, Cursor, Aider, Cline, Roo Code,
170+
Windsurf, Devin, Codegen, SWE-Agent, and Gemini. If a tool doesn&apos;t add
171+
Co-Authored-By trailers, we can&apos;t detect it — this is a known limitation.
172+
</p>
173+
</section>
174+
175+
<section className={`${wrappedTheme.card} space-y-4 p-6`}>
176+
<h2 className="text-lg font-semibold text-zinc-950">6) Why it can be wrong</h2>
144177
<ul className="list-disc space-y-2 pl-5 text-sm text-zinc-700">
145178
<li>GitHub only shows what’s pushed; local work and private repos may be missing.</li>
146179
<li>Some repos have incomplete metadata (e.g., missing file paths).</li>

apps/web/src/app/page.tsx

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
aggregateUserProfile,
1010
computeVibeFromCommits,
1111
detectVibePersona,
12+
type AIToolMetrics,
1213
type RepoInsightSummary,
1314
type VibeAxes,
1415
type VibeCommitEvent,
@@ -21,6 +22,7 @@ import {
2122
RepoBreakdownSection,
2223
UnifiedMethodologySection,
2324
} from "@/components/vcp/unified";
25+
import { VCPAIToolsSection } from "@/components/vcp/blocks";
2426

2527
const heroFeatures = [
2628
"A Vibe Coding Profile (VCP) built from AI-assisted engineering signals in your commit history",
@@ -110,6 +112,7 @@ type AuthStats = {
110112
} | null;
111113
llmModel?: string | null;
112114
llmKeySource?: string | null;
115+
aiTools?: AIToolMetrics | null;
113116
};
114117
};
115118

@@ -675,6 +678,50 @@ export default async function Home({
675678
const latestInsightRepoName = (latestInsightRepoNameResult?.data ??
676679
null) as unknown as RepoNameRow | null;
677680

681+
// Fetch aggregated AI tool metrics from vibe_insights
682+
let profileAiTools: AIToolMetrics | null = null;
683+
if (userProfileData) {
684+
const effectiveJobIds = profileJobIds ?? [];
685+
if (effectiveJobIds.length > 0) {
686+
const { data: vibeRows } = await supabase
687+
.from("vibe_insights")
688+
.select("ai_tools_json")
689+
.in("job_id", effectiveJobIds);
690+
691+
const vibeToolRows = (vibeRows ?? []) as Array<{ ai_tools_json: unknown }>;
692+
if (vibeToolRows.length > 0) {
693+
const toolCounts = new Map<string, { name: string; count: number }>();
694+
let totalAiCommits = 0;
695+
for (const row of vibeToolRows) {
696+
const tools = row.ai_tools_json as AIToolMetrics | null;
697+
if (!tools || !tools.detected) continue;
698+
totalAiCommits += tools.ai_assisted_commits;
699+
for (const tool of tools.tools) {
700+
const existing = toolCounts.get(tool.tool_id);
701+
if (existing) existing.count += tool.commit_count;
702+
else toolCounts.set(tool.tool_id, { name: tool.tool_name, count: tool.commit_count });
703+
}
704+
}
705+
if (toolCounts.size > 0) {
706+
const total = userProfileData.total_commits ?? 0;
707+
const tools = Array.from(toolCounts.entries())
708+
.map(([id, d]) => ({
709+
tool_id: id, tool_name: d.name, commit_count: d.count,
710+
percentage: totalAiCommits > 0 ? Math.round((d.count / totalAiCommits) * 100) : 0,
711+
}))
712+
.sort((a, b) => b.commit_count - a.commit_count);
713+
profileAiTools = {
714+
detected: true, ai_assisted_commits: totalAiCommits,
715+
ai_collaboration_rate: total > 0 ? totalAiCommits / total : 0,
716+
primary_tool: { id: tools[0].tool_id, name: tools[0].tool_name },
717+
tool_diversity: tools.length, tools,
718+
confidence: totalAiCommits >= 10 ? "high" : totalAiCommits >= 3 ? "medium" : "low",
719+
};
720+
}
721+
}
722+
}
723+
}
724+
678725
const debugParam = resolvedSearchParams.debug;
679726
const debugEnabled =
680727
debugParam === "1" || (Array.isArray(debugParam) && debugParam.includes("1"));
@@ -719,6 +766,7 @@ export default async function Home({
719766
narrative: userProfileData.narrative_json ?? null,
720767
llmModel: userProfileData.llm_model ?? null,
721768
llmKeySource: userProfileData.llm_key_source ?? null,
769+
aiTools: profileAiTools,
722770
}
723771
: undefined,
724772
};
@@ -1211,6 +1259,13 @@ function AuthenticatedDashboard({
12111259
<UnifiedAxesSection axes={stats.userProfile.axes} />
12121260
) : null}
12131261

1262+
{/* Section 3b: AI Coding Tools */}
1263+
{stats.userProfile?.aiTools?.detected ? (
1264+
<div className="border-t border-black/5 px-8 py-6 sm:px-10">
1265+
<VCPAIToolsSection aiTools={stats.userProfile.aiTools} />
1266+
</div>
1267+
) : null}
1268+
12141269
{/* Section 4: Evolution */}
12151270
<EvolutionSection
12161271
repoVcpCount={stats.completedJobs}

0 commit comments

Comments
 (0)