feat: add ability to ask LLM questions about a report

Often times I find the AI Summary to be super helpful, but there are
also cases where the summary is lacking critical information, or giving
me too verbose information.

Every user might look for different things here. To address this need,
and leverage the full power of AI for e.g. quantifying findings (instead
of manual chore work, clicking through every app), this PR introduces an
AI assistant chat.

Users can ask questions about a report and get answers! This is an
extension of the AI Summary and can be extremely helpful.

Author: devversion

package-lock.json

@@ -16,7 +16,19 @@
             "browser": "src/main.ts",
             "polyfills": ["zone.js"],
             "tsConfig": "tsconfig.app.json",
-            "externalDependencies": ["@firebase/app", "@firebase/firestore"],
+            "externalDependencies": [
+              "@firebase/app",
+              "@firebase/firestore",
+              "tiktoken",
+              "genkit",
+              "@genkit-ai/compat-oai",
+              "@genkit-ai/googleai",
+              "@genkit-ai/mcp",
+              "@google/genai",
+              "@google/generative-ai",
+              "genkitx-anthropic",
+              "node-fetch"
+            ],
             "allowedCommonJsDependencies": [
               "prettier/plugins/angular.js",
               "prettier/plugins/typescript.js",

report-app/angular.json

@@ -7,9 +7,14 @@ import {
 import express from 'express';
 import {dirname, isAbsolute, join, resolve} from 'node:path';
 import {fileURLToPath} from 'node:url';
-import {FetchedLocalReports, fetchReportsFromDisk} from '../runner/reporting/report-local-disk';
-import {RunInfo} from '../runner/shared-interfaces';
+import {chatWithReportAI} from '../runner/reporting/report-ai-chat';
 import {convertV2ReportToV3Report} from '../runner/reporting/migrations/v2_to_v3';
+import {FetchedLocalReports, fetchReportsFromDisk} from '../runner/reporting/report-local-disk';
+import {AiChatRequest, RunInfo} from '../runner/shared-interfaces';
+
+// This will result in a lot of loading and would slow down the serving,
+// so it's loaded lazily below.
+import {type GenkitRunner} from '../runner/codegen/genkit/genkit-runner';
 
 const app = express();
 const reportsLoader = await getReportLoader();
@@ -19,6 +24,8 @@ const browserDistFolder = resolve(serverDistFolder, '../browser');
 const angularApp = new AngularNodeAppEngine();
 let localDataPromise: Promise<FetchedLocalReports> | null = null;
 
+app.use(express.json());
+
 // Endpoint for fetching all available report groups.
 app.get('/api/reports', async (_, res) => {
   const [remoteGroups, localData] = await Promise.all([
@@ -34,9 +41,7 @@ app.get('/api/reports', async (_, res) => {
   res.json(results);
 });
 
-// Endpoint for fetching a specific report group.
-app.get('/api/reports/:id', async (req, res) => {
-  const id = req.params.id;
+async function fetchAndMigrateReports(id: string): Promise<RunInfo[] | null> {
   const localData = await resolveLocalData(options.reportsRoot);
   let result: RunInfo[] | null = null;
 
@@ -46,10 +51,56 @@ app.get('/api/reports/:id', async (req, res) => {
     result = await reportsLoader.getGroupedReports(id);
   }
 
+  if (result === null) {
+    return null;
+  }
+
   // Convert potential older v2 reports.
-  result = result.map(r => convertV2ReportToV3Report(r));
+  return result.map(r => convertV2ReportToV3Report(r));
+}
+
+// Endpoint for fetching a specific report group.
+app.get('/api/reports/:id', async (req, res) => {
+  const id = req.params.id;
+  const result = await fetchAndMigrateReports(id);
+
+  res.json(result ?? []);
+});
+
+let llm: Promise<GenkitRunner> | null = null;
+
+/** Lazily initializes and returns the genkit runner. */
+async function getOrCreateGenkitLlmRunner() {
+  const llm = new (await import('../runner/codegen/genkit/genkit-runner')).GenkitRunner();
+  // Gracefully shut down the runner on exit.
+  process.on('SIGINT', () => llm!.dispose());
+  process.on('SIGTERM', () => llm!.dispose());
+  return llm;
+}
+
+// Endpoint for fetching a specific report group.
+app.post('/api/reports/:id/chat', async (req, res) => {
+  const id = req.params.id;
+  const reports = await fetchAndMigrateReports(id);
+
+  if (reports === null) {
+    res.status(404).send('Not found');
+    return;
+  }
 
-  res.json(result);
+  const {prompt, pastMessages, model} = req.body as AiChatRequest;
+  const assessments = reports.flatMap(run => run.results);
+  const abortController = new AbortController();
+  const summary = await chatWithReportAI(
+    await (llm ?? getOrCreateGenkitLlmRunner()),
+    prompt,
+    abortController.signal,
+    assessments,
+    pastMessages,
+    model,
+  );
+
+  res.json(summary);
 });
 
 app.use(
@@ -106,6 +157,7 @@ async function getReportLoader() {
     return {
       getGroupedReports: () => Promise.resolve([]),
       getGroupsList: () => Promise.resolve([]),
+      configureEndpoints: async () => {},
     } satisfies ReportLoader;
   }
 

report-app/report-server.ts

@@ -70,10 +70,7 @@ <h3 class="chart-title">
             <span>Build</span>
           </h3>
           <div class="summary-card-item">
-            <stacked-bar-chart
-              [data]="buildsAsGraphData(overview.stats.builds)"
-              [compact]="true"
-            />
+            <stacked-bar-chart [data]="buildsAsGraphData(overview.stats.builds)" [compact]="true" />
           </div>
         </div>
         @if (overview.stats.runtime) {
@@ -176,13 +173,15 @@ <h4>Repair System Prompt</h4>
     </expansion-panel>
 
     @if (report.details.summary.aiSummary !== undefined) {
-      <expansion-panel size="large" class="root-section">
-        <expansion-panel-header>
-          <img src="gemini.webp" alt="Gemini Logo" height="30" width="30" />
-          AI Summary
-        </expansion-panel-header>
-        <div [innerHTML]="report.details.summary.aiSummary"></div>
-      </expansion-panel>
+      <button class="fab" (click)="isAiAssistantVisible.set(true)">
+        <span class="material-symbols-outlined">smart_toy</span>
+      </button>
+
+      <app-ai-assistant
+        [class.hidden]="!isAiAssistantVisible()"
+        [reportGroupId]="reportGroupId()"
+        (close)="isAiAssistantVisible.set(false)"
+      />
     }
 
     @if (missingDeps().length > 0) {
@@ -232,9 +231,7 @@ <h4>Logs</h4>
     <h2>Generated applications</h2>
     @if (allFailedChecks().length > 0) {
       <details class="filter-dropdown" #dropdown>
-        <summary>
-          Filter by failed checks ({{ selectedChecks().size }} selected)
-        </summary>
+        <summary>Filter by failed checks ({{ selectedChecks().size }} selected)</summary>
         <div class="dropdown-content">
           <failed-checks-filter
             [allFailedChecks]="allFailedChecks()"
@@ -300,9 +297,7 @@ <h5>
                         @if (isSkippedAssessment(check)) {
                           <span class="status">➖</span>
                           <span class="name">{{ check.name }}</span>
-                          <span class="status-text points"
-                            >Skipped: {{ check.message }}</span
-                          >
+                          <span class="status-text points">Skipped: {{ check.message }}</span>
                         } @else {
                           @let isMax = check.successPercentage === 1;
                           <span
@@ -340,9 +335,11 @@ <h5>
                     [class.warn]="totalPercent < 90 && totalPercent >= 80"
                     [class.error]="totalPercent < 80"
                   >
-                    {{ result.score.totalPoints }} /
-                    {{ result.score.maxOverallPoints }} points ({{
+                    {{ result.score.totalPoints }} / {{ result.score.maxOverallPoints }} points ({{
                       totalPercent
+
+
+
                     }}%)
                   </span>
                 </div>
@@ -352,10 +349,8 @@ <h5>
                 <h4>Additional info</h4>
                 @for (attempt of result.attemptDetails; track attempt) {
                   @let isBuilt = attempt.buildResult.status === 'success';
-                  @let axeViolations =
-                    attempt.serveTestingResult?.axeViolations;
-                  @let hasAxeViolations =
-                    axeViolations && axeViolations.length > 0;
+                  @let axeViolations = attempt.serveTestingResult?.axeViolations;
+                  @let hasAxeViolations = axeViolations && axeViolations.length > 0;
 
                   <expansion-panel #expansionPanel>
                     <expansion-panel-header>
@@ -385,9 +380,7 @@ <h4>Additional info</h4>
                     @if (expansionPanel.opened()) {
                       @if (attempt.reasoning) {
                         <details class="thoughts-button">
-                          <summary class="neutral-button">
-                            See LLM Thoughts
-                          </summary>
+                          <summary class="neutral-button">See LLM Thoughts</summary>
                           <pre class="callout neutral code">{{
                             attempt.reasoning
                           }}</pre>
@@ -430,25 +423,16 @@ <h4>Generated Code</h4>
                             Format source code
                           </button>
                         </div>
-                        <app-code-viewer
-                          [code]="formatted().get(file) ?? file.code"
-                        />
+                        <app-code-viewer [code]="formatted().get(file) ?? file.code" />
                       }
                     }
                   </expansion-panel>
                 }
 
-                @if (
-                  result.userJourneys && result.userJourneys.result.length > 0
-                ) {
+                @if (result.userJourneys && result.userJourneys.result.length > 0) {
                   <expansion-panel>
-                    <expansion-panel-header
-                      >User Journeys</expansion-panel-header
-                    >
-                    @for (
-                      journey of result.userJourneys.result;
-                      track journey.name
-                    ) {
+                    <expansion-panel-header>User Journeys</expansion-panel-header>
+                    @for (journey of result.userJourneys.result; track journey.name) {
                       <h4>{{ journey.name }}</h4>
 
                       <ol>
@@ -467,7 +451,8 @@ <h4>Debugging Tools</h4>
                 <button
                   class="neutral-button"
                   title="Download a ZIP for debugging. You can upload the ZIP to AI Studio for further analysis of a specific app."
-                  (click)="downloadDebuggingZip(result)">
+                  (click)="downloadDebuggingZip(result)"
+                >
                   Download ZIP for debugging
                 </button>
 
@@ -487,8 +472,7 @@ <h4>Debugging Tools</h4>
                           <details class="details mcp-log-entry">
                             @let name = log.request.name;
                             <summary>
-                              Log Entry #{{ $index + 1
-                              }}{{ name ? ' - ' + name : '' }}
+                              Log Entry #{{ $index + 1}}{{ name ? ' - ' + name : '' }}
                             </summary>
                             <div class="mcp-log-content">
                               <h5>Request</h5>
@@ -512,8 +496,7 @@ <h5>Response</h5>
                 }
               </div>
 
-              @let finalRuntimeErrors =
-                finalAttempt.serveTestingResult?.runtimeErrors;
+              @let finalRuntimeErrors = finalAttempt.serveTestingResult?.runtimeErrors;
               @if (finalRuntimeErrors) {
                 <div class="app-details-section">
                   <h4>Runtime errors</h4>

report-app/src/app/pages/report-viewer/report-viewer.html

@@ -262,3 +262,26 @@ expansion-panel {
   list-style: none;
   padding: 0;
 }
+
+.fab {
+  position: fixed;
+  bottom: 20px;
+  right: 20px;
+  z-index: 999;
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  padding: 12px 18px;
+  border-radius: 28px;
+  background-color: #007bff;
+  color: white;
+  border: none;
+  box-shadow: 0 2px 10px rgba(0, 0, 0, 0.15);
+  cursor: pointer;
+  font-size: 1em;
+  font-weight: 500;
+}
+
+.hidden {
+  visibility: hidden;
+}

report-app/src/app/pages/report-viewer/report-viewer.scss

@@ -1,5 +1,6 @@
 import {Clipboard} from '@angular/cdk/clipboard';
 import {DatePipe, DecimalPipe} from '@angular/common';
+import {HttpClient} from '@angular/common/http';
 import {
   afterNextRender,
   Component,
@@ -17,6 +18,7 @@ import {
   BuildResultStatus,
 } from '../../../../../runner/workers/builder/builder-types';
 import {
+  AiChatResponse,
   AssessmentResult,
   IndividualAssessment,
   IndividualAssessmentState,
@@ -42,6 +44,8 @@ import {bucketToScoreVariable, formatScore, ScoreCssVariable} from '../../shared
 import {ExpansionPanel} from '../../shared/expansion-panel/expansion-panel';
 import {ExpansionPanelHeader} from '../../shared/expansion-panel/expansion-panel-header';
 import {ProviderLabel} from '../../shared/provider-label';
+import {firstValueFrom} from 'rxjs';
+import {AiAssistant} from '../../shared/ai-assistant/ai-assistant';
 
 const localReportRegex = /-l\d+$/;
 
@@ -58,6 +62,7 @@ const localReportRegex = /-l\d+$/;
     ExpansionPanelHeader,
     ProviderLabel,
     NgxJsonViewerModule,
+    AiAssistant,
   ],
   templateUrl: './report-viewer.html',
   styleUrls: ['./report-viewer.scss'],
@@ -68,6 +73,7 @@ const localReportRegex = /-l\d+$/;
 export class ReportViewer {
   private clipboard = inject(Clipboard);
   private reportsFetcher = inject(ReportsFetcher);
+  private http = inject(HttpClient);
 
   constructor() {
     // Scroll the page to the top since it seems to always land slightly scrolled down.
@@ -79,6 +85,7 @@ export class ReportViewer {
   protected formatted = signal<Map<LlmResponseFile, string>>(new Map());
   protected formatScore = formatScore;
   protected error = computed(() => this.selectedReport.error());
+  protected isAiAssistantVisible = signal(false);
 
   private selectedReport = resource({
     params: () => ({groupId: this.reportGroupId()}),