GreenDev: "floaty" in DevTools prototype

This is the prototype of the "DevTools Floaty" window idea that will
enable users to use a "point and click" approach to add additional
context to the active AI Agent in DevTools.

**Important**: all of this functionality is behind the GreenDev flag. We
have **no intention** of pushing this feature live in this state. This
is code landing to user test in Canary that will not ship without an
additional project to make this code fully production worthy. That is
why this CL has no tests, for example.

**What**

This CL is made up of a few parts:

1. The `Floaty` window. This window appears either via the icon in the
   AI Assistance panel, or by using the `f` key on the keyboard.
2. The `onFloatyClick` function, which can be called when the user has
  clicked on something that we want to add as context. If the floaty is
  in inspect mode, it will add this item to the context.
3. The `extraContext` in the AI Assistance chat view, which shows all
   the additional context items the user has added.
4. The `facts` added in AI Agent, which are added to the prompt that is
   sent to the LLM so it is aware that the user has explicitly added more
   items to the context.

Bug: 461428712
Change-Id: Ic21e18d266a994d4eb593d3b7ef4726b5d214cb0
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/7198584
Commit-Queue: Jack Franklin <jacktfranklin@chromium.org>
Reviewed-by: Finnur Thorarinsson <finnur@chromium.org>

Author: jackfranklin

config/gni/devtools_grd_files.gni

@@ -2494,6 +2494,7 @@ grd_files_unbundled_sources = [
   "front_end/ui/legacy/EmptyWidget.js",
   "front_end/ui/legacy/FilterBar.js",
   "front_end/ui/legacy/FilterSuggestionBuilder.js",
+  "front_end/ui/legacy/Floaty.js",
   "front_end/ui/legacy/ForwardedInputEventHandler.js",
   "front_end/ui/legacy/Fragment.js",
   "front_end/ui/legacy/GlassPane.js",
@@ -2641,6 +2642,7 @@ grd_files_unbundled_sources = [
   "front_end/ui/legacy/dropTarget.css.js",
   "front_end/ui/legacy/emptyWidget.css.js",
   "front_end/ui/legacy/filter.css.js",
+  "front_end/ui/legacy/floaty.css.js",
   "front_end/ui/legacy/glassPane.css.js",
   "front_end/ui/legacy/infobar.css.js",
   "front_end/ui/legacy/inspectorCommon.css.js",

front_end/entrypoints/main/MainImpl.ts

@@ -809,8 +809,8 @@ export class MainMenuItem implements UI.Toolbar.Provider {
   readonly #item: UI.Toolbar.ToolbarMenuButton;
   constructor() {
     this.#item = new UI.Toolbar.ToolbarMenuButton(
-        this.#handleContextMenu.bind(this), /* isIconDropdown */ true, /* useSoftMenu */ true, 'main-menu',
-        'dots-vertical');
+        this.#handleContextMenu.bind(this), /* isIconDropdown */ true,
+        /* useSoftMenu */ true, 'main-menu', 'dots-vertical');
     this.#item.element.classList.add('main-menu');
     this.#item.setTitle(i18nString(UIStrings.customizeAndControlDevtools));
   }

front_end/models/ai_assistance/AiConversation.ts

@@ -4,6 +4,9 @@
 
 import * as Host from '../../core/host/host.js';
 import * as Root from '../../core/root/root.js';
+import * as SDK from '../../core/sdk/sdk.js';
+import * as Trace from '../../models/trace/trace.js';
+import * as NetworkTimeCalculator from '../network_time_calculator/network_time_calculator.js';
 
 import {
   type AiAgent,
@@ -19,6 +22,10 @@ import {PerformanceAgent} from './agents/PerformanceAgent.js';
 import {StylingAgent} from './agents/StylingAgent.js';
 import {AiHistoryStorage, ConversationType, type SerializedConversation} from './AiHistoryStorage.js';
 import type {ChangeManager} from './ChangeManager.js';
+import {NetworkRequestFormatter} from './data_formatters/NetworkRequestFormatter.js';
+import {PerformanceInsightFormatter} from './data_formatters/PerformanceInsightFormatter.js';
+import {micros} from './data_formatters/UnitFormatters.js';
+import {AgentFocus} from './performance/AIContext.js';
 
 export const NOT_FOUND_IMAGE_DATA = '';
 const MAX_TITLE_LENGTH = 80;
@@ -240,15 +247,91 @@ export class AiConversation {
     return agent;
   }
 
+  #factsCache = new Map<ExtraContext, Host.AidaClient.RequestFact>();
+
+  async #createFactsForExtraContext(contexts: ExtraContext[]): Promise<void> {
+    for (const context of contexts) {
+      const cached = this.#factsCache.get(context);
+      if (cached) {
+        this.#agent.addFact(cached);
+        continue;
+      }
+
+      if (context instanceof SDK.DOMModel.DOMNode) {
+        const desc = await StylingAgent.describeElement(context);
+
+        const fact = {
+          text: `Relevant HTML element:\n${desc}`,
+          metadata: {
+            source: 'devtools-floaty',
+            score: 1,
+          }
+        };
+        this.#factsCache.set(context, fact);
+        this.#agent.addFact(fact);
+      } else if (context instanceof SDK.NetworkRequest.NetworkRequest) {
+        const calculator = new NetworkTimeCalculator.NetworkTransferTimeCalculator();
+        calculator.updateBoundaries(context);
+        const formatter = new NetworkRequestFormatter(context, calculator);
+        const desc = await formatter.formatNetworkRequest();
+
+        const fact = {
+          text: `Relevant network request:\n${desc}`,
+          metadata: {
+            source: 'devtools-floaty',
+            score: 1,
+          }
+        };
+        this.#factsCache.set(context, fact);
+        this.#agent.addFact(fact);
+      } else if ('insight' in context) {
+        const focus = AgentFocus.fromInsight(context.trace, context.insight);
+        const formatter = new PerformanceInsightFormatter(
+            focus,
+            context.insight,
+        );
+
+        const text = `Relevant Performance Insight:\n${formatter.formatInsight()}`;
+        const fact = {
+          text,
+          metadata: {
+            source: 'devtools-floaty',
+            score: 1,
+          }
+        };
+        this.#factsCache.set(context, fact);
+        this.#agent.addFact(fact);
+      } else {
+        // Must be a trace event
+        const time = Trace.Types.Timing.Micro(
+            context.event.ts - context.traceStartTime,
+        );
+
+        const desc = `Trace event: ${context.event.name}
+Time: ${micros(time)}`;
+
+        const fact = {
+          text: `Relevant trace event:\n${desc}`,
+          metadata: {
+            source: 'devtools-floaty',
+            score: 1,
+          }
+        };
+        this.#factsCache.set(context, fact);
+        this.#agent.addFact(fact);
+      }
+    }
+  }
+
   async *
       run(
           initialQuery: string,
-          options: {
-            selected: ConversationContext<unknown>|null,
-            signal?: AbortSignal,
-          },
+          options: {selected: ConversationContext<unknown>|null, signal?: AbortSignal, extraContext?: ExtraContext[]},
           multimodalInput?: MultimodalInput,
           ): AsyncGenerator<ResponseData, void, void> {
+    if (options.extraContext) {
+      await this.#createFactsForExtraContext(options.extraContext);
+    }
     for await (const data of this.#agent.run(initialQuery, options, multimodalInput)) {
       // We don't want to save partial responses to the conversation history.
       // TODO(crbug.com/463325400): We should save interleaved answers to the history as well.
@@ -267,3 +350,10 @@ export class AiConversation {
 function isAiAssistanceServerSideLoggingEnabled(): boolean {
   return !Root.Runtime.hostConfig.aidaAvailability?.disallowLogging;
 }
+
+// TODO: this is the same as the type in UI.Floaty but we cannot use UI
+// here. This is fine for prototyping but if we take this further we can
+// rearchitect.
+type ExtraContext = SDK.DOMModel.DOMNode|SDK.NetworkRequest.NetworkRequest|
+                    {event: Trace.Types.Events.Event, traceStartTime: Trace.Types.Timing.Micro}|
+                    {insight: Trace.Insights.Types.InsightModel, trace: Trace.TraceModel.ParsedTrace};

front_end/models/logs/BUILD.gn

@@ -37,6 +37,7 @@ devtools_entrypoint("bundle") {
     "../../panels/issues/*",
     "../../panels/network/*",
     "../../ui/components/request_link_icon/*",
+    "../../ui/legacy/*",
     "../ai_assistance/*",
     "../extensions/*",
   ]

front_end/models/trace/BUILD.gn

@@ -49,6 +49,7 @@ devtools_entrypoint("bundle") {
 
   visibility = [
     ":*",
+    "../../../mcp/*",
     "../../../test/interactions/*",
     "../../core/sdk:unittests",
     "../../legacy_test_runner/*",
@@ -60,12 +61,11 @@ devtools_entrypoint("bundle") {
     "../../services/tracing/*",
     "../../testing/*",
     "../../ui/components/docs/*",
-    "../../ui/legacy/components/utils/*",
+    "../../ui/legacy/*",
     "../ai_assistance/*",
     "../live-metrics/*",
     "../trace_source_maps_resolver/*",
     "./*",
-    "../../../mcp/*",
   ]
 
   visibility += devtools_models_visibility

front_end/models/trace/types/TraceEvents.ts

@@ -85,6 +85,9 @@ export interface Event {
   tdur?: Micro;
   dur?: Micro;
 }
+export function objectIsEvent(obj: object): obj is Event {
+  return 'cat' in obj && 'name' in obj && 'ts' in obj;
+}
 
 export interface Args {
   data?: ArgsData;

front_end/panels/ai_assistance/AiAssistancePanel.ts

@@ -505,6 +505,7 @@ export class AiAssistancePanel extends UI.Panel.Panel {
   #isTextInputEmpty = true;
   #timelinePanelInstance: TimelinePanel.TimelinePanel.TimelinePanel|null = null;
   #runAbortController = new AbortController();
+  #additionalContextItemsFromFloaty: UI.Floaty.FloatyContextSelection[] = [];
 
   constructor(private view: View = defaultView, {aidaClient, aidaAvailability, syncInfo}: {
     aidaClient: Host.AidaClient.AidaClient,
@@ -549,6 +550,7 @@ export class AiAssistancePanel extends UI.Panel.Panel {
       return {
         state: ViewState.CHAT_VIEW,
         props: {
+          additionalFloatyContext: this.#additionalContextItemsFromFloaty,
           blockedByCrossOrigin: this.#blockedByCrossOrigin,
           isLoading: this.#isLoading,
           messages: this.#messages,
@@ -649,6 +651,15 @@ export class AiAssistancePanel extends UI.Panel.Panel {
     }
   }
 
+  #bindFloatyListener(): void {
+    const additionalContexts = UI.Context.Context.instance().flavor(UI.Floaty.FloatyFlavor);
+    if (!additionalContexts) {
+      return;
+    }
+    this.#additionalContextItemsFromFloaty = additionalContexts.selectedContexts;
+    this.requestUpdate();
+  }
+
   #getDefaultConversationType(): AiAssistanceModel.AiHistoryStorage.ConversationType|undefined {
     const {hostConfig} = Root.Runtime;
     const viewManager = UI.ViewManager.ViewManager.instance();
@@ -765,6 +776,11 @@ export class AiAssistancePanel extends UI.Panel.Panel {
     this.#selectDefaultAgentIfNeeded();
 
     Host.userMetrics.actionTaken(Host.UserMetrics.Action.AiAssistancePanelOpened);
+
+    if (Root.Runtime.hostConfig.devToolsGreenDevUi?.enabled) {
+      UI.Context.Context.instance().addFlavorChangeListener(UI.Floaty.FloatyFlavor, this.#bindFloatyListener, this);
+      this.#bindFloatyListener();
+    }
   }
 
   override willHide(): void {
@@ -1409,6 +1425,7 @@ export class AiAssistancePanel extends UI.Panel.Panel {
             text, {
               signal,
               selected: context,
+              extraContext: this.#additionalContextItemsFromFloaty,
             },
             multimodalInput),
     );

front_end/panels/ai_assistance/components/ChatView.test.ts

@@ -40,6 +40,7 @@ describeWithEnvironment('ChatView', () => {
       disclaimerText: i18n.i18n.lockedString('disclaimer text'),
       isTextInputEmpty: true,
       markdownRenderer: new AiAssistancePanel.MarkdownRendererWithCodeBlock(),
+      additionalFloatyContext: [],
       ...options,
     };
   }

front_end/panels/ai_assistance/components/ChatView.ts

@@ -8,9 +8,11 @@ import '../../../ui/components/spinners/spinners.js';
 
 import * as Host from '../../../core/host/host.js';
 import * as i18n from '../../../core/i18n/i18n.js';
-import type * as Platform from '../../../core/platform/platform.js';
+import * as Platform from '../../../core/platform/platform.js';
+import * as Root from '../../../core/root/root.js';
 import * as SDK from '../../../core/sdk/sdk.js';
 import * as AiAssistanceModel from '../../../models/ai_assistance/ai_assistance.js';
+import * as Trace from '../../../models/trace/trace.js';
 import * as Workspace from '../../../models/workspace/workspace.js';
 import * as PanelsCommon from '../../../panels/common/common.js';
 import * as PanelUtils from '../../../panels/utils/utils.js';
@@ -265,6 +267,7 @@ export interface Props {
   isTextInputEmpty: boolean;
   uploadImageInputEnabled?: boolean;
   markdownRenderer: MarkdownLitRenderer;
+  additionalFloatyContext: UI.Floaty.FloatyContextSelection[];
 }
 
 export class ChatView extends HTMLElement {
@@ -517,7 +520,8 @@ export class ChatView extends HTMLElement {
         onTakeScreenshot: this.#props.onTakeScreenshot,
         onRemoveImageInput: this.#props.onRemoveImageInput,
         onTextInputChange: this.#props.onTextInputChange,
-        onImageUpload: this.#handleImageUpload
+        onImageUpload: this.#handleImageUpload,
+        additionalFloatyContext: this.#props.additionalFloatyContext,
       });
     };
 
@@ -1347,6 +1351,7 @@ function renderChatInput({
   isTextInputEmpty,
   uploadImageInputEnabled,
   disclaimerText,
+  additionalFloatyContext,
   onContextClick,
   onInspectElementClick,
   onSubmit,
@@ -1365,6 +1370,7 @@ function renderChatInput({
   selectedContext: AiAssistanceModel.AiAgent.ConversationContext<unknown>|null,
   inspectElementToggled: boolean,
   isTextInputEmpty: boolean,
+  additionalFloatyContext: UI.Floaty.FloatyContextSelection[],
   disclaimerText: string,
   onContextClick: () => void,
   onInspectElementClick: () => void,
@@ -1389,6 +1395,7 @@ function renderChatInput({
 
   // clang-format off
   return html` <form class="input-form" @submit=${onSubmit}>
+  ${renderFloatyExtraContext(additionalFloatyContext)}
     <div class=${chatInputContainerCls}>
       ${renderImageInput({
         multimodalInputEnabled,
@@ -1463,6 +1470,75 @@ function renderChatInput({
   // clang-format on
 }
 
+function renderFloatyExtraContext(contexts: UI.Floaty.FloatyContextSelection[]): Lit.LitTemplate {
+  if (!Root.Runtime.hostConfig.devToolsGreenDevUi?.enabled) {
+    return Lit.nothing;
+  }
+
+  // clang-format off
+  return html`
+  <ul class="floaty">
+    ${contexts.map(c => {
+      function onDelete(e: MouseEvent): void {
+        e.preventDefault();
+        UI.Floaty.onFloatyContextDelete(c);
+      }
+
+      return html`<li>
+        <span class="context-item">
+          ${renderFloatyContext(c)}
+        </span>
+        <devtools-button
+          class="floaty-delete-button"
+          @click=${onDelete}
+          .data=${{
+            variant: Buttons.Button.Variant.ICON,
+            iconName: 'cross',
+            title: 'Delete',
+            size: Buttons.Button.Size.SMALL,
+          } as Buttons.Button.ButtonData}
+        ></devtools-button>
+      </li>`;
+    })}
+    <li class="open-floaty">
+      <devtools-button
+        class="floaty-add-button"
+        @click=${UI.Floaty.onFloatyOpen}
+        .data=${{
+          variant: Buttons.Button.Variant.ICON,
+          iconName: 'select-element',
+          title: 'Open context picker',
+          size: Buttons.Button.Size.SMALL,
+        } as Buttons.Button.ButtonData}
+      ></devtools-button>
+    </li>
+  </ul>
+  `;
+  // clang-format on
+}
+
+function renderFloatyContext(context: UI.Floaty.FloatyContextSelection): Lit.TemplateResult {
+  if (context instanceof SDK.NetworkRequest.NetworkRequest) {
+    return html`${context.url()}`;
+  }
+
+  if (context instanceof SDK.DOMModel.DOMNode) {
+    return html`<devtools-widget .widgetConfig=${
+        UI.Widget.widgetConfig(PanelsCommon.DOMLinkifier.DOMNodeLink, {node: context})}>`;
+  }
+
+  if ('insight' in context) {
+    return html`${context.insight.title}`;
+  }
+
+  if ('event' in context && 'traceStartTime' in context) {
+    const time = Trace.Types.Timing.Micro(context.event.ts - context.traceStartTime);
+    return html`${context.event.name} @ ${i18n.TimeUtilities.formatMicroSecondsAsMillisFixed(time)}`;
+  }
+
+  Platform.assertNever(context, 'Unsupported context');
+}
+
 function renderMainContents({
   messages,
   isLoading,