Diagnostics-for-account-and-installations

backend/src/controllers/setup.controller.ts

@@ -2,6 +2,62 @@
 import app from '../index.js';
 import StatusService from '../services/status.service.js';
 import logger from '../services/logger.js';
+import { Endpoints } from '@octokit/types';
+
+// Type definitions for the diagnostic response
+interface OctokitTestResult {
+  success: boolean;
+  appName?: string;
+  appOwner?: string;
+  permissions?: Record<string, string | undefined>;
+  error?: string;
+}
+
+interface InstallationDiagnostic {
+  index: number;
+  installationId: number;
+  accountLogin: string;
+  accountId: string | number;
+  accountType: string;
+  accountAvatarUrl: string;
+  appId: number;
+  appSlug: string;
+  targetType: string;
+  permissions: Record<string, string | undefined>;
+  events: string[];
+  createdAt: string;
+  updatedAt: string;
+  suspendedAt: string | null;
+  suspendedBy: { login: string; id: number } | null;
+  hasOctokit: boolean;
+  octokitTest: OctokitTestResult | null;
+  isValid: boolean;
+  validationErrors: string[];
+}
+
+interface AppInfo {
+  name: string;
+  description: string;
+  owner: string;
+  htmlUrl: string;
+  permissions: Record<string, string | undefined>;
+  events: string[];
+}
+
+interface DiagnosticsResponse {
+  timestamp: string;
+  appConnected: boolean;
+  totalInstallations: number;
+  installations: InstallationDiagnostic[];
+  errors: string[];
+  appInfo: AppInfo | null;
+  summary: {
+    validInstallations: number;
+    invalidInstallations: number;
+    organizationNames: string[];
+    accountTypes: Record<string, number>;
+  };
+}
 
 class SetupController {
   async registrationComplete(req: Request, res: Response) {
@@ -112,6 +168,140 @@
     }
   }
 
+  async validateInstallations(req: Request, res: Response) {
+    try {
+      const diagnostics: DiagnosticsResponse = {
+        timestamp: new Date().toISOString(),
+        appConnected: !!app.github.app,
+        totalInstallations: app.github.installations.length,
+        installations: [],
+        errors: [],
+        appInfo: null,
+        summary: {
+          validInstallations: 0,
+          invalidInstallations: 0,
+          organizationNames: [],
+          accountTypes: {}
+        }
+      };
+
+      // Basic app validation
+      if (!app.github.app) {
+        diagnostics.errors.push('GitHub App is not initialized');
+        return res.json(diagnostics);
+      }
+
+      // Validate each installation
+      for (let i = 0; i < app.github.installations.length; i++) {
+        const { installation, octokit } = app.github.installations[i];
+
+        const installationDiag: InstallationDiagnostic = {
+          index: i,
+          installationId: installation.id,
+          accountLogin: installation.account?.login || 'MISSING',
+          accountId: installation.account?.id || 'MISSING',
+          accountType: installation.account?.type || 'MISSING',
+          accountAvatarUrl: installation.account?.avatar_url || 'MISSING',
+          appId: installation.app_id,
+          appSlug: installation.app_slug,
+          targetType: installation.target_type,
+          permissions: installation.permissions || {},
+          events: installation.events || [],
+          createdAt: installation.created_at,
+          updatedAt: installation.updated_at,
+          suspendedAt: installation.suspended_at,
+          suspendedBy: installation.suspended_by,
+          hasOctokit: !!octokit,
+          octokitTest: null,
+          isValid: true,
+          validationErrors: []
+        };
+
+        // Validate required fields
+        if (!installation.account?.login) {
+          installationDiag.isValid = false;
+          installationDiag.validationErrors.push('Missing account.login (organization name)');
+        }
+
+        if (!installation.account?.id) {
+          installationDiag.isValid = false;
+          installationDiag.validationErrors.push('Missing account.id');
+        }
+
+        if (!installation.account?.type) {
+          installationDiag.isValid = false;
+          installationDiag.validationErrors.push('Missing account.type');
+        }
+
+        // Test Octokit functionality
+        if (octokit) {
+          try {
+            // Test basic API call with the installation's octokit
+            const authTest = await octokit.rest.apps.getAuthenticated();
+            installationDiag.octokitTest = {
+              success: true,
+              appName: authTest.data?.name || 'Unknown',
+              appOwner: (authTest.data?.owner && 'login' in authTest.data.owner) ? authTest.data.owner.login : 'Unknown',
+              permissions: authTest.data?.permissions || {}
+            };
+          } catch (error) {
+            installationDiag.octokitTest = {
+              success: false,
+              error: error instanceof Error ? error.message : 'Unknown error'
+            };
+            installationDiag.isValid = false;
+            installationDiag.validationErrors.push(`Octokit API test failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+          }
+        } else {
+          installationDiag.isValid = false;
+          installationDiag.validationErrors.push('Octokit instance is missing');
+        }
+
+        // Update summary
+        if (installationDiag.isValid) {
+          diagnostics.summary.validInstallations++;
+          if (installation.account?.login) {
+            diagnostics.summary.organizationNames.push(installation.account.login);
+          }
+        } else {
+          diagnostics.summary.invalidInstallations++;
+        }
+
+        // Track account types
+        const accountType = installation.account?.type || 'Unknown';
+        diagnostics.summary.accountTypes[accountType] = (diagnostics.summary.accountTypes[accountType] || 0) + 1;
+
+        diagnostics.installations.push(installationDiag);
+      }
+
+      // Additional app-level diagnostics
+      try {
+        const appInfo = await app.github.app.octokit.rest.apps.getAuthenticated();
+        diagnostics.appInfo = {
+          name: appInfo.data?.name || 'Unknown',
+          description: appInfo.data?.description || 'No description',
+          owner: (appInfo.data?.owner && 'login' in appInfo.data.owner) ? appInfo.data.owner.login : 'Unknown',
+          htmlUrl: appInfo.data?.html_url || 'Unknown',
+          permissions: appInfo.data?.permissions || {},
+          events: appInfo.data?.events || []
+        };
+      } catch (error) {
+        diagnostics.errors.push(`Failed to get app info: ${error instanceof Error ? error.message : 'Unknown error'}`);
+      }
+
+      // Sort organization names for easier reading
+      diagnostics.summary.organizationNames.sort();
+
+      res.json(diagnostics);
+    } catch (error) {
+      logger.error('Installation validation failed', error);
+      res.status(500).json({ 
+        error: 'Installation validation failed', 
+        details: error instanceof Error ? error.message : 'Unknown error' 
+      });
+    }
+  }
+
 
 }
 

backend/src/routes/index.ts

@@ -52,6 +52,7 @@ router.get('/setup/manifest', setupController.getManifest);
 router.post('/setup/existing-app', setupController.addExistingApp);
 router.post('/setup/db', setupController.setupDB);
 router.get('/setup/status', setupController.setupStatus);
+router.get('/setup/validate-installations', setupController.validateInstallations);
 
 router.get('/status', setupController.getStatus);
 

backend/src/services/value_modeling_doc.md

@@ -0,0 +1,118 @@
+**Value Modeling & Targeting Documentation**
+
+This document outlines the rationale and logic behind the calculated metrics and targets used in the "Value Modeling & Targeting" dashboard. Each section corresponds to a category of metrics displayed in the dashboard.
+
+---
+
+### Org Metrics
+
+**Seats**
+
+- **Logic**: Based on the average total active seats(licenses) across top 10 recent days for the organization.
+- **Max**: Set to known total developer headcount.
+
+**Adopted Devs**
+
+- **Logic**: Average of total active developers using AI tooling (e.g. Copilot) from top 10 recent days for the organization.
+- **Max**: Total known developer count.
+
+**Monthly Devs Reporting Time Savings**
+
+- **Logic**: Count of distinct users who responded to time-savings surveys in past 30 days.
+- **Target**: Double the current, indicating intent to increase reporting.
+
+**% of Seats Reporting Time Savings**
+
+- **Logic**: (Monthly reporting users / total seats) \* 100.
+- **Purpose**: Shows how broadly time savings are captured.
+
+**% of Seats Adopted**
+
+- **Logic**: (Adopted Devs / Total Seats) \* 100.
+- **Use**: Adoption penetration relative to seat assignments.
+
+**% of Max Adopted**
+
+- **Logic**: (Adopted Devs / Total Developer Count) \* 100.
+- **Use**: Indicates potential ceiling for adoption.
+
+---
+
+### Daily User Metrics
+
+**Daily IDE Suggestions**
+
+- **Logic**: Averaged from last 5 valid daily records.
+- **Target/Max**: Calibrated based on observed high-performing usage.
+
+**Daily IDE Acceptances**
+
+- **Logic**: Suggestions \* 30% (default assumed acceptance rate).
+- **Target/Max**: Reflects healthy usage from productive orgs.
+
+**Daily IDE Chat Turns**
+
+- **Logic**: Average of chat turns per day per user from recent week.
+- **Target/Max**: Reflects healthy usage from productive orgs.
+
+**Daily Dot-Com Chats**
+
+- **Logic**: Chat Turns \* 33% (estimated portion on dot-com).
+- **Target**: Not yet set pending more data.
+
+**Weekly PR Summaries**
+
+- **Logic**: Total PR summaries / daily active users from last week.
+
+**Weekly Time Saved**
+
+- **Logic**: Weekly average from time savings reports per developer.\
+  // Calculate weekly hours saved based on settings and average percent
+
+  const weeklyHours = hoursPerYear / 50; // Assuming 50 working weeks
+
+  const weeklyDevHours = weeklyHours \* (percentCoding / 100);
+
+  const avgWeeklyTimeSaved = weeklyDevHours \* (avgPercentTimeSaved / 100);
+
+---
+
+### Calculated Impacts
+
+**Monthly Time Savings (hrs)**
+
+- **Formula**: Adopted Devs \* Weekly Time Saved \* 4.
+- **Max**: 80 hours/month \* total seats (full work month).
+
+**Annual Time Savings (Dollars)**
+
+- **Formula**: Weekly Time Saved \* 50 weeks \* \$100/hr \* Adopted Devs.
+- **Note**: \$100/hr is assumed average developer cost.
+
+**Productivity / Throughput Boost**
+
+- **Formula**: ((40 + Weekly Time Saved) / 40 - 1) \* 100.
+- **Purpose**: Estimates effective increase in output per dev.
+
+---
+
+### Source of Calculations
+
+All calculations were derived from one or more of:
+
+- Recent metric exports (5 most recent days)
+- Monthly time-savings surveys
+- Developer seat and activity data
+- Assumed baselines (e.g., 40-hr weeks, \$100/hr, 70% acceptance)
+
+Targets are either:
+
+- Reflective of past top 10 org benchmarks
+- Strategically aspirational (2x current, known limits)
+
+---
+
+This model provides a structured framework for tracking usage, estimating impact, and guiding adoption investments.
+
+> Edits can include notes on thresholds, cohort segmentation, or more nuanced modeling (e.g., p50/p90 range breakdowns).
+

frontend/src/app/app.routes.ts

@@ -15,6 +15,7 @@ import { CopilotSeatComponent } from './main/copilot/copilot-seats/copilot-seat/
 import { DatabaseComponent } from './database/database.component';
 import { ErrorComponent } from './error/error.component';
 import { CopilotValueModelingComponent } from './main/copilot/copilot-value-modeling/copilot-value-modeling.component';
+import { MainDiagnosticsComponent } from './main/diagnostics/main-diagnostics.component';
 
 export const routes: Routes = [
   { path: 'setup', component: InstallComponent },
@@ -38,6 +39,7 @@ export const routes: Routes = [
       { path: 'copilot/surveys/:id', component: CopilotSurveyComponent, title: 'Survey' },
       { path: 'copilot/value-modeling', component: CopilotValueModelingComponent, title: 'Value Modeling' },
       { path: 'settings', component: SettingsComponent, title: 'Settings' },
+      { path: 'diagnostics', component: MainDiagnosticsComponent, title: 'Diagnostics' },
       { path: '', redirectTo: 'copilot', pathMatch: 'full' }
     ]
   },