diff --git a/backend/src/controllers/setup.controller.ts b/backend/src/controllers/setup.controller.ts index e8abb58..1280b8b 100644 --- a/backend/src/controllers/setup.controller.ts +++ b/backend/src/controllers/setup.controller.ts @@ -112,6 +112,140 @@ class SetupController { } } + async validateInstallations(req: Request, res: Response) { + try { + const diagnostics = { + timestamp: new Date().toISOString(), + appConnected: !!app.github.app, + totalInstallations: app.github.installations.length, + installations: [] as any[], + errors: [] as string[], + appInfo: null as any, + summary: { + validInstallations: 0, + invalidInstallations: 0, + organizationNames: [] as string[], + accountTypes: {} as Record + } + }; + + // Basic app validation + if (!app.github.app) { + diagnostics.errors.push('GitHub App is not initialized'); + return res.json(diagnostics); + } + + // Validate each installation + for (let i = 0; i < app.github.installations.length; i++) { + const { installation, octokit } = app.github.installations[i]; + + const installationDiag = { + index: i, + installationId: installation.id, + accountLogin: installation.account?.login || 'MISSING', + accountId: installation.account?.id || 'MISSING', + accountType: installation.account?.type || 'MISSING', + accountAvatarUrl: installation.account?.avatar_url || 'MISSING', + appId: installation.app_id, + appSlug: installation.app_slug, + targetType: installation.target_type, + permissions: installation.permissions, + events: installation.events, + createdAt: installation.created_at, + updatedAt: installation.updated_at, + suspendedAt: installation.suspended_at, + suspendedBy: installation.suspended_by, + hasOctokit: !!octokit, + octokitTest: null as any, + isValid: true, + validationErrors: [] as string[] + }; + + // Validate required fields + if (!installation.account?.login) { + installationDiag.isValid = false; + installationDiag.validationErrors.push('Missing account.login (organization name)'); + } + + if (!installation.account?.id) { + installationDiag.isValid = false; + installationDiag.validationErrors.push('Missing account.id'); + } + + if (!installation.account?.type) { + installationDiag.isValid = false; + installationDiag.validationErrors.push('Missing account.type'); + } + + // Test Octokit functionality + if (octokit) { + try { + // Test basic API call with the installation's octokit + const authTest = await octokit.rest.apps.getAuthenticated(); + installationDiag.octokitTest = { + success: true, + appName: authTest.data?.name || 'Unknown', + appOwner: (authTest.data?.owner as any)?.login || 'Unknown', + permissions: authTest.data?.permissions || {} + }; + } catch (error) { + installationDiag.octokitTest = { + success: false, + error: error instanceof Error ? error.message : 'Unknown error' + }; + installationDiag.isValid = false; + installationDiag.validationErrors.push(`Octokit API test failed: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + } else { + installationDiag.isValid = false; + installationDiag.validationErrors.push('Octokit instance is missing'); + } + + // Update summary + if (installationDiag.isValid) { + diagnostics.summary.validInstallations++; + if (installation.account?.login) { + diagnostics.summary.organizationNames.push(installation.account.login); + } + } else { + diagnostics.summary.invalidInstallations++; + } + + // Track account types + const accountType = installation.account?.type || 'Unknown'; + diagnostics.summary.accountTypes[accountType] = (diagnostics.summary.accountTypes[accountType] || 0) + 1; + + diagnostics.installations.push(installationDiag); + } + + // Additional app-level diagnostics + try { + const appInfo = await app.github.app.octokit.rest.apps.getAuthenticated(); + diagnostics.appInfo = { + name: appInfo.data?.name || 'Unknown', + description: appInfo.data?.description || 'No description', + owner: (appInfo.data?.owner as any)?.login || 'Unknown', + htmlUrl: appInfo.data?.html_url || 'Unknown', + permissions: appInfo.data?.permissions || {}, + events: appInfo.data?.events || [] + }; + } catch (error) { + diagnostics.errors.push(`Failed to get app info: ${error instanceof Error ? error.message : 'Unknown error'}`); + } + + // Sort organization names for easier reading + diagnostics.summary.organizationNames.sort(); + + res.json(diagnostics); + } catch (error) { + logger.error('Installation validation failed', error); + res.status(500).json({ + error: 'Installation validation failed', + details: error instanceof Error ? error.message : 'Unknown error' + }); + } + } + } diff --git a/backend/src/routes/index.ts b/backend/src/routes/index.ts index a31a9d1..e977ce0 100644 --- a/backend/src/routes/index.ts +++ b/backend/src/routes/index.ts @@ -52,6 +52,7 @@ router.get('/setup/manifest', setupController.getManifest); router.post('/setup/existing-app', setupController.addExistingApp); router.post('/setup/db', setupController.setupDB); router.get('/setup/status', setupController.setupStatus); +router.get('/setup/validate-installations', setupController.validateInstallations); router.get('/status', setupController.getStatus); diff --git a/backend/src/services/value_modeling_doc.md b/backend/src/services/value_modeling_doc.md new file mode 100644 index 0000000..8c7d62c --- /dev/null +++ b/backend/src/services/value_modeling_doc.md @@ -0,0 +1,118 @@ +**Value Modeling & Targeting Documentation** + +This document outlines the rationale and logic behind the calculated metrics and targets used in the "Value Modeling & Targeting" dashboard. Each section corresponds to a category of metrics displayed in the dashboard. + +--- + +### Org Metrics + +**Seats** + +- **Logic**: Based on the average total active seats(licenses) across top 10 recent days for the organization. +- **Max**: Set to known total developer headcount. + +**Adopted Devs** + +- **Logic**: Average of total active developers using AI tooling (e.g. Copilot) from top 10 recent days for the organization. +- **Max**: Total known developer count. + +**Monthly Devs Reporting Time Savings** + +- **Logic**: Count of distinct users who responded to time-savings surveys in past 30 days. +- **Target**: Double the current, indicating intent to increase reporting. + +**% of Seats Reporting Time Savings** + +- **Logic**: (Monthly reporting users / total seats) \* 100. +- **Purpose**: Shows how broadly time savings are captured. + +**% of Seats Adopted** + +- **Logic**: (Adopted Devs / Total Seats) \* 100. +- **Use**: Adoption penetration relative to seat assignments. + +**% of Max Adopted** + +- **Logic**: (Adopted Devs / Total Developer Count) \* 100. +- **Use**: Indicates potential ceiling for adoption. + +--- + +### Daily User Metrics + +**Daily IDE Suggestions** + +- **Logic**: Averaged from last 5 valid daily records. +- **Target/Max**: Calibrated based on observed high-performing usage. + +**Daily IDE Acceptances** + +- **Logic**: Suggestions \* 30% (default assumed acceptance rate). +- **Target/Max**: Reflects healthy usage from productive orgs. + +**Daily IDE Chat Turns** + +- **Logic**: Average of chat turns per day per user from recent week. +- **Target/Max**: Reflects healthy usage from productive orgs. + +**Daily Dot-Com Chats** + +- **Logic**: Chat Turns \* 33% (estimated portion on dot-com). +- **Target**: Not yet set pending more data. + +**Weekly PR Summaries** + +- **Logic**: Total PR summaries / daily active users from last week. + +**Weekly Time Saved** + +- **Logic**: Weekly average from time savings reports per developer.\ + // Calculate weekly hours saved based on settings and average percent + + const weeklyHours = hoursPerYear / 50; // Assuming 50 working weeks + + const weeklyDevHours = weeklyHours \* (percentCoding / 100); + + const avgWeeklyTimeSaved = weeklyDevHours \* (avgPercentTimeSaved / 100); + +--- + +### Calculated Impacts + +**Monthly Time Savings (hrs)** + +- **Formula**: Adopted Devs \* Weekly Time Saved \* 4. +- **Max**: 80 hours/month \* total seats (full work month). + +**Annual Time Savings (Dollars)** + +- **Formula**: Weekly Time Saved \* 50 weeks \* \$100/hr \* Adopted Devs. +- **Note**: \$100/hr is assumed average developer cost. + +**Productivity / Throughput Boost** + +- **Formula**: ((40 + Weekly Time Saved) / 40 - 1) \* 100. +- **Purpose**: Estimates effective increase in output per dev. + +--- + +### Source of Calculations + +All calculations were derived from one or more of: + +- Recent metric exports (5 most recent days) +- Monthly time-savings surveys +- Developer seat and activity data +- Assumed baselines (e.g., 40-hr weeks, \$100/hr, 70% acceptance) + +Targets are either: + +- Reflective of past top 10 org benchmarks +- Strategically aspirational (2x current, known limits) + +--- + +This model provides a structured framework for tracking usage, estimating impact, and guiding adoption investments. + +> Edits can include notes on thresholds, cohort segmentation, or more nuanced modeling (e.g., p50/p90 range breakdowns). + diff --git a/frontend/src/app/app.routes.ts b/frontend/src/app/app.routes.ts index 7f97481..a600cf6 100644 --- a/frontend/src/app/app.routes.ts +++ b/frontend/src/app/app.routes.ts @@ -15,6 +15,7 @@ import { CopilotSeatComponent } from './main/copilot/copilot-seats/copilot-seat/ import { DatabaseComponent } from './database/database.component'; import { ErrorComponent } from './error/error.component'; import { CopilotValueModelingComponent } from './main/copilot/copilot-value-modeling/copilot-value-modeling.component'; +import { MainDiagnosticsComponent } from './main/diagnostics/main-diagnostics.component'; export const routes: Routes = [ { path: 'setup', component: InstallComponent }, @@ -38,6 +39,7 @@ export const routes: Routes = [ { path: 'copilot/surveys/:id', component: CopilotSurveyComponent, title: 'Survey' }, { path: 'copilot/value-modeling', component: CopilotValueModelingComponent, title: 'Value Modeling' }, { path: 'settings', component: SettingsComponent, title: 'Settings' }, + { path: 'diagnostics', component: MainDiagnosticsComponent, title: 'Diagnostics' }, { path: '', redirectTo: 'copilot', pathMatch: 'full' } ] }, diff --git a/frontend/src/app/main/diagnostics/main-diagnostics.component.ts b/frontend/src/app/main/diagnostics/main-diagnostics.component.ts new file mode 100644 index 0000000..dc1804c --- /dev/null +++ b/frontend/src/app/main/diagnostics/main-diagnostics.component.ts @@ -0,0 +1,481 @@ +import { Component, Inject } from '@angular/core'; +import { CommonModule, DatePipe } from '@angular/common'; +import { MatButtonModule } from '@angular/material/button'; +import { MatCardModule } from '@angular/material/card'; +import { MatIconModule } from '@angular/material/icon'; +import { MatDialogModule, MatDialog, MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { MatExpansionModule } from '@angular/material/expansion'; +import { MatChipsModule } from '@angular/material/chips'; +import { MatSnackBar } from '@angular/material/snack-bar'; +import { MatProgressSpinnerModule } from '@angular/material/progress-spinner'; +import { MatToolbarModule } from '@angular/material/toolbar'; +import { SetupService } from '../../services/api/setup.service'; + +@Component({ + selector: 'app-main-diagnostics', + standalone: true, + imports: [ + CommonModule, + MatButtonModule, + MatCardModule, + MatIconModule, + MatProgressSpinnerModule, + MatToolbarModule + ], + template: ` + + bug_report + Installation Diagnostics + + +
+ + + GitHub App Installation Validator + Validate your GitHub App installations and Octokit connections + + +

This tool helps you diagnose issues with your GitHub App installations by:

+
    +
  • Validating that all installation accounts have proper data
    • +
  • Testing Octokit authentication for each installation
    • +
  • Listing all organization names (account.login) available
    • +
  • Verifying account types and permissions
    • +
  • Providing detailed error information for troubleshooting
    • +
+ + + + + + + + + + Quick Summary + Last run: {{ lastResult.timestamp | date:'medium' }} + + +
+
+
{{ lastResult.totalInstallations }}
+
Total Installations
+
+
+
+ {{ lastResult.summary.validInstallations }} +
+
Valid
+
+
+
+ {{ lastResult.summary.invalidInstallations }} +
+
Invalid
+
+
+
{{ getSuccessRate() }}%
+
Success Rate
+
+
+ +
+

Organizations Found:

+
+ + {{ org }} + +
+
+ + + + + + +
+ `, + styles: [` + .diagnostics-container { + padding: 24px; + max-width: 1200px; + margin: 0 auto; + } + + .welcome-card, .results-card { + margin-bottom: 24px; + } + + .welcome-card ul { + margin: 16px 0; + padding-left: 24px; + } + + .welcome-card li { + margin: 8px 0; + } + + .summary-stats { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(120px, 1fr)); + gap: 16px; + margin: 16px 0; + } + + .stat { + text-align: center; + padding: 16px; + border: 1px solid #e0e0e0; + border-radius: 8px; + } + + .stat-value { + font-size: 2em; + font-weight: bold; + color: #666; + } + + .stat-value.success { + color: #4caf50; + } + + .stat-value.error { + color: #f44336; + } + + .stat-label { + font-size: 0.875em; + color: #888; + margin-top: 4px; + } + + .organizations { + margin-top: 16px; + } + + .org-list { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-top: 8px; + } + + .org-chip { + background: #e3f2fd; + color: #1976d2; + padding: 4px 12px; + border-radius: 16px; + font-size: 0.875em; + border: 1px solid #bbdefb; + } + + mat-card-actions { + padding: 16px; + } + + mat-card-actions button { + margin-right: 8px; + } + `] +}) +export class MainDiagnosticsComponent { + isLoading = false; + lastResult: any = null; + + constructor( + private setupService: SetupService, + private dialog: MatDialog, + private snackBar: MatSnackBar + ) {} + + runDiagnostics(): void { + this.isLoading = true; + this.setupService.validateInstallations().subscribe({ + next: (result) => { + this.isLoading = false; + this.lastResult = result; + + if (result.summary.invalidInstallations > 0 || result.errors.length > 0) { + this.snackBar.open('Diagnostics completed with some issues. Check the details.', 'Close', { + duration: 5000 + }); + } else { + this.snackBar.open('All installations validated successfully!', 'Close', { + duration: 3000 + }); + } + }, + error: (error) => { + this.isLoading = false; + this.snackBar.open('Failed to run diagnostics: ' + error.message, 'Close', { + duration: 5000 + }); + } + }); + } + + getSuccessRate(): number { + if (!this.lastResult || this.lastResult.totalInstallations === 0) return 0; + return Math.round((this.lastResult.summary.validInstallations / this.lastResult.totalInstallations) * 100); + } + + showFullDetails(): void { + this.dialog.open(InstallationDiagnosticsDialogComponent, { + width: '90vw', + maxWidth: '1200px', + height: '80vh', + data: this.lastResult + }); + } + + downloadResults(): void { + if (!this.lastResult) return; + + const dataStr = JSON.stringify(this.lastResult, null, 2); + const dataBlob = new Blob([dataStr], { type: 'application/json' }); + const url = URL.createObjectURL(dataBlob); + const link = document.createElement('a'); + link.href = url; + link.download = `installation-diagnostics-${new Date().toISOString().split('T')[0]}.json`; + link.click(); + URL.revokeObjectURL(url); + } +} + +@Component({ + selector: 'app-installation-diagnostics-dialog', + template: ` +

Installation Diagnostics Details

+ +
+

Summary

+
+ + + App Status + + +

Connected: {{ data.appConnected ? 'Yes' : 'No' }}

+

Total Installations: {{ data.totalInstallations }}

+

Timestamp: {{ data.timestamp | date:'medium' }}

+ + + + + + Validation Results + + +

Valid: {{ data.summary.validInstallations }}

+

Invalid: {{ data.summary.invalidInstallations }}

+

Success Rate: {{ getSuccessRate() }}%

+ + +
+ + + + App Information + + +

Name: {{ data.appInfo.name }}

+

Description: {{ data.appInfo.description }}

+

Owner: {{ data.appInfo.owner }}

+

HTML URL: {{ data.appInfo.htmlUrl }}

+ + + + + + Organizations ({{ data.summary.organizationNames.length }}) + + + + {{ org }} + + + +
+ +
+

Errors

+ + +
    +
  • {{ error }}
    • +
+ + +
+ +
+

Installation Details

+ + + + + + {{ installation.isValid ? 'check_circle' : 'error' }} + + {{ installation.accountLogin }} (ID: {{ installation.installationId }}) + + + {{ installation.accountType }} - {{ installation.isValid ? 'Valid' : 'Invalid' }} + + + +
+
+
+ Installation ID: {{ installation.installationId }} +
+
+ Account Login: {{ installation.accountLogin }} +
+
+ Account ID: {{ installation.accountId }} +
+
+ Account Type: {{ installation.accountType }} +
+
+ App ID: {{ installation.appId }} +
+
+ Target Type: {{ installation.targetType }} +
+
+ Has Octokit: {{ installation.hasOctokit ? 'Yes' : 'No' }} +
+
+ Created: {{ installation.createdAt | date:'medium' }} +
+
+ +
+

Octokit Test

+

Success: {{ installation.octokitTest.success ? 'Yes' : 'No' }}

+
+

App Name: {{ installation.octokitTest.appName }}

+

App Owner: {{ installation.octokitTest.appOwner }}

+
+
+

Error: {{ installation.octokitTest.error }}

+
+
+ +
+

Validation Errors

+
    +
  • {{ error }}
    • +
+
+
+ + +
+ + + + + + `, + styles: [` + .diagnostics-content { + max-height: 70vh; + overflow-y: auto; + } + .summary-section { + margin-bottom: 20px; + } + .summary-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 16px; + margin-bottom: 16px; + } + .organizations-card { + margin-bottom: 16px; + } + .detail-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: 8px; + margin-bottom: 16px; + } + .detail-item { + padding: 4px 0; + } + .error-item { + color: #f44336; + margin: 4px 0; + } + .invalid { + border-left: 4px solid #f44336; + } + .octokit-test, .validation-errors { + margin-top: 16px; + padding: 12px; + border: 1px solid #e0e0e0; + border-radius: 4px; + } + .error { + color: #f44336; + } + .installations-section { + margin-top: 20px; + } + .errors-section { + margin-bottom: 20px; + } + mat-chip-set { + margin: 8px 0; + } + `], + standalone: true, + imports: [ + MatDialogModule, + MatButtonModule, + MatCardModule, + MatIconModule, + MatExpansionModule, + MatChipsModule, + CommonModule, + DatePipe + ] +}) +export class InstallationDiagnosticsDialogComponent { + constructor( + public dialogRef: MatDialogRef, + @Inject(MAT_DIALOG_DATA) public data: any + ) {} + + getSuccessRate(): number { + if (this.data.totalInstallations === 0) return 0; + return Math.round((this.data.summary.validInstallations / this.data.totalInstallations) * 100); + } + + downloadDiagnostics(): void { + const dataStr = JSON.stringify(this.data, null, 2); + const dataBlob = new Blob([dataStr], { type: 'application/json' }); + const url = URL.createObjectURL(dataBlob); + const link = document.createElement('a'); + link.href = url; + link.download = `installation-diagnostics-${new Date().toISOString().split('T')[0]}.json`; + link.click(); + URL.revokeObjectURL(url); + } +} diff --git a/frontend/src/app/main/main.component.html b/frontend/src/app/main/main.component.html index e57f33d..9e73562 100644 --- a/frontend/src/app/main/main.component.html +++ b/frontend/src/app/main/main.component.html @@ -43,6 +43,10 @@ settings Settings + + bug_report + Diagnostics + diff --git a/frontend/src/app/services/api/setup.service.ts b/frontend/src/app/services/api/setup.service.ts index 2a8dfc0..1b88651 100644 --- a/frontend/src/app/services/api/setup.service.ts +++ b/frontend/src/app/services/api/setup.service.ts @@ -60,4 +60,8 @@ export class SetupService { return this.http.post(`${this.apiUrl}/db`, request); } + validateInstallations() { + return this.http.get(`${this.apiUrl}/validate-installations`); + } + }