feat: support chat agent rules and context management (#4578)

* feat: support process large image on tool result

* fix: typos

* chore: improve compress

* chore: reverse chat history

* chore: fix

* chore: update iconfont

* chore: add rules

* chore: update icons

* chore: update icons

* chore: update icons

* feat: support better llm context

* feat: support rules configuration

* feat: support add rules manualy

* feat: support global and cursor rules

* feat: support manage reference on header

* chore: support add context by click @

* feat: support remove context by chat input

* chore: fix types

* feat: improve some cases

* chore: use findApplicableRules function

* chore: fix build

* chore: update deps

* chore: update locks

Author: erha19

packages/ai-native/package.json

@@ -40,6 +40,7 @@
     "@opensumi/ide-outline": "workspace:*",
     "@opensumi/ide-overlay": "workspace:*",
     "@opensumi/ide-preferences": "workspace:*",
+    "@opensumi/ide-quick-open": "workspace:*",
     "@opensumi/ide-search": "workspace:*",
     "@opensumi/ide-terminal-next": "workspace:*",
     "@opensumi/ide-theme": "workspace:*",

packages/ai-native/src/browser/chat/chat-proxy.service.ts

@@ -1,18 +1,16 @@
 import { Autowired, Injectable } from '@opensumi/di';
-import { AppConfig, PreferenceService } from '@opensumi/ide-core-browser';
+import { PreferenceService } from '@opensumi/ide-core-browser';
 import {
   AIBackSerivcePath,
   CancellationToken,
   ChatAgentViewServiceToken,
   ChatFeatureRegistryToken,
-  ChatServiceToken,
   Deferred,
   Disposable,
   IAIBackService,
   IAIReporter,
   IApplicationService,
   IChatProgress,
-  getOperatingSystemName,
 } from '@opensumi/ide-core-common';
 import { AINativeSettingSectionsId } from '@opensumi/ide-core-common/lib/settings/ai-native';
 import { MonacoCommandRegistry } from '@opensumi/ide-editor/lib/browser/monaco-contrib/command/command.service';
@@ -27,10 +25,10 @@ import {
   IChatAgentService,
   IChatAgentWelcomeMessage,
 } from '../../common';
+import { DEFAULT_SYSTEM_PROMPT } from '../../common/prompts/system-prompt';
 import { ChatToolRender } from '../components/ChatToolRender';
 import { IChatAgentViewService } from '../types';
 
-import { ChatService } from './chat.api.service';
 import { ChatFeatureRegistry } from './chat.feature.registry';
 
 /**
@@ -53,9 +51,6 @@ export class ChatProxyService extends Disposable {
   @Autowired(MonacoCommandRegistry)
   private readonly monacoCommandRegistry: MonacoCommandRegistry;
 
-  @Autowired(ChatServiceToken)
-  private aiChatService: ChatService;
-
   @Autowired(IAIReporter)
   private readonly aiReporter: IAIReporter;
 
@@ -71,9 +66,6 @@ export class ChatProxyService extends Disposable {
   @Autowired(IMessageService)
   private readonly messageService: IMessageService;
 
-  @Autowired(AppConfig)
-  private readonly appConfig: AppConfig;
-
   private chatDeferred: Deferred<void> = new Deferred<void>();
 
   public getRequestOptions() {
@@ -112,79 +104,74 @@ export class ChatProxyService extends Disposable {
       initialProps: {},
     });
 
-    this.addDispose(
-      this.chatAgentService.registerAgent({
-        id: ChatProxyService.AGENT_ID,
-        metadata: {
-          systemPrompt:
-            this.preferenceService.get<string>(
+    this.applicationService.getBackendOS().then(() => {
+      this.addDispose(
+        this.chatAgentService.registerAgent({
+          id: ChatProxyService.AGENT_ID,
+          metadata: {
+            systemPrompt: this.preferenceService.get<string>(
               AINativeSettingSectionsId.SystemPrompt,
-              'You are a powerful AI coding assistant working in OpenSumi, a top IDE framework. You collaborate with a USER to solve coding tasks, which may involve creating, modifying, or debugging code, or answering questions. When the USER sends a message, relevant context (e.g., open files, cursor position, edit history, linter errors) may be attached. Use this information as needed.\n\n<tool_calling>\nYou have access to tools to assist with tasks. Follow these rules:\n1. Always adhere to the tool call schema and provide all required parameters.\n2. Only use tools explicitly provided; ignore unavailable ones.\n3. Avoid mentioning tool names to the USER (e.g., say "I will edit your file" instead of "I need to use the edit_file tool").\n4. Only call tools when necessary; respond directly if the task is general or you already know the answer.\n5. Explain to the USER why you’re using a tool before calling it.\n</tool_calling>\n\n<making_code_changes>\nWhen modifying code:\n1. Use code edit tools instead of outputting code unless explicitly requested.\n2. Limit tool calls to one per turn.\n3. Ensure generated code is immediately executable by including necessary imports, dependencies, and endpoints.\n4. For new projects, create a dependency management file (e.g., requirements.txt) and a README.\n5. For web apps, design a modern, user-friendly UI.\n6. Avoid generating non-textual or excessively long code.\n7. Read file contents before editing, unless appending a small change or creating a new file.\n8. Fix introduced linter errors if possible, but stop after 3 attempts and ask the USER for guidance.\n9. Reapply reasonable code edits if they weren’t followed initially.\n</making_code_changes>\n\nUse the appropriate tools to fulfill the USER’s request, ensuring all required parameters are provided or inferred from context.',
-            ) +
-            `\n\n<user_info>\nThe user's OS is ${getOperatingSystemName()}. The absolute path of the user's workspace is ${
-              this.appConfig.workspaceDir
-            }.\n</user_info>`,
-        },
-        invoke: async (
-          request: IChatAgentRequest,
-          progress: (part: IChatProgress) => void,
-          history: CoreMessage[],
-          token: CancellationToken,
-        ): Promise<IChatAgentResult> => {
-          this.chatDeferred = new Deferred<void>();
-
-          const { message, command } = request;
-          let prompt: string = message;
-
-          if (command) {
-            const commandHandler = this.chatFeatureRegistry.getSlashCommandHandler(command);
-            if (commandHandler && commandHandler.providerPrompt) {
-              const editor = this.monacoCommandRegistry.getActiveCodeEditor();
-              const slashCommandPrompt = await commandHandler.providerPrompt(message, editor);
-              prompt = slashCommandPrompt;
+              DEFAULT_SYSTEM_PROMPT,
+            ),
+          },
+          invoke: async (
+            request: IChatAgentRequest,
+            progress: (part: IChatProgress) => void,
+            history: CoreMessage[],
+            token: CancellationToken,
+          ): Promise<IChatAgentResult> => {
+            this.chatDeferred = new Deferred<void>();
+            const { message, command } = request;
+            let prompt: string = message;
+            if (command) {
+              const commandHandler = this.chatFeatureRegistry.getSlashCommandHandler(command);
+              if (commandHandler && commandHandler.providerPrompt) {
+                const editor = this.monacoCommandRegistry.getActiveCodeEditor();
+                const slashCommandPrompt = await commandHandler.providerPrompt(message, editor);
+                prompt = slashCommandPrompt;
+              }
             }
-          }
-
-          const stream = await this.aiBackService.requestStream(
-            prompt,
-            {
-              requestId: request.requestId,
-              sessionId: request.sessionId,
-              history,
-              images: request.images,
-              ...this.getRequestOptions(),
-            },
-            token,
-          );
-
-          listenReadable<IChatProgress>(stream, {
-            onData: (data) => {
-              progress(data);
-            },
-            onEnd: () => {
-              this.chatDeferred.resolve();
-            },
-            onError: (error) => {
-              this.messageService.error(error.message);
-              this.aiReporter.end(request.sessionId + '_' + request.requestId, {
-                message: error.message,
-                success: false,
-                command,
-              });
-            },
-          });
-
-          await this.chatDeferred.promise;
-          return {};
-        },
-        provideSlashCommands: async (token: CancellationToken): Promise<IChatAgentCommand[]> =>
-          this.chatFeatureRegistry
-            .getAllSlashCommand()
-            .map((s) => ({ ...s, name: s.name, description: s.description || '' })),
-        provideChatWelcomeMessage: async (token: CancellationToken): Promise<IChatAgentWelcomeMessage | undefined> =>
-          undefined,
-      }),
-    );
+
+            const stream = await this.aiBackService.requestStream(
+              prompt,
+              {
+                requestId: request.requestId,
+                sessionId: request.sessionId,
+                history,
+                images: request.images,
+                ...this.getRequestOptions(),
+              },
+              token,
+            );
+
+            listenReadable<IChatProgress>(stream, {
+              onData: (data) => {
+                progress(data);
+              },
+              onEnd: () => {
+                this.chatDeferred.resolve();
+              },
+              onError: (error) => {
+                this.messageService.error(error.message);
+                this.aiReporter.end(request.sessionId + '_' + request.requestId, {
+                  message: error.message,
+                  success: false,
+                  command,
+                });
+              },
+            });
+
+            await this.chatDeferred.promise;
+            return {};
+          },
+          provideSlashCommands: async (): Promise<IChatAgentCommand[]> =>
+            this.chatFeatureRegistry
+              .getAllSlashCommand()
+              .map((s) => ({ ...s, name: s.name, description: s.description || '' })),
+          provideChatWelcomeMessage: async (): Promise<IChatAgentWelcomeMessage | undefined> => undefined,
+        }),
+      );
+    });
 
     queueMicrotask(() => {
       this.chatAgentService.updateAgent(ChatProxyService.AGENT_ID, {});

packages/ai-native/src/browser/chat/chat.view.tsx

@@ -617,7 +617,12 @@ export const AIChatView = () => {
       const { message, images, agentId, command, reportExtra } = value;
       const { actionType, actionSource } = reportExtra || {};
 
-      const request = aiChatService.createRequest(message, agentId!, images, command);
+      const request = aiChatService.createRequest(
+        message.replaceAll(LLM_CONTEXT_KEY_REGEX, ''),
+        agentId!,
+        images,
+        command,
+      );
       if (!request) {
         return;
       }
@@ -641,7 +646,7 @@ export const AIChatView = () => {
         600 * 1000,
       );
       msgHistoryManager.addUserMessage({
-        content: message.replaceAll(LLM_CONTEXT_KEY_REGEX, ''),
+        content: message,
         images: images || [],
         agentId: agentId!,
         agentCommand: command!,
@@ -695,14 +700,9 @@ export const AIChatView = () => {
       let processedContent = message;
       const filePattern = /\{\{@file:(.*?)\}\}/g;
       const fileMatches = message.match(filePattern);
-      let isCleanContext = false;
       if (fileMatches) {
         for (const match of fileMatches) {
           const filePath = match.replace(/\{\{@file:(.*?)\}\}/, '$1');
-          if (filePath && !isCleanContext) {
-            isCleanContext = true;
-            llmContextService.cleanFileContext();
-          }
           const fileUri = new URI(filePath);
           const relativePath = (await workspaceService.asRelativePath(fileUri))?.path || fileUri.displayName;
           processedContent = processedContent.replace(match, `\`${LLM_CONTEXT_KEY.AttachedFile}${relativePath}\``);
@@ -738,6 +738,18 @@ export const AIChatView = () => {
           );
         }
       }
+      const rulePattern = /\{\{@rule:(.*?)\}\}/g;
+      const ruleMatches = processedContent.match(rulePattern);
+      if (ruleMatches) {
+        for (const match of ruleMatches) {
+          const ruleName = match.replace(/\{\{@rule:(.*?)\}\}/, '$1');
+          const ruleUri = new URI(ruleName);
+          processedContent = processedContent.replace(
+            match,
+            `\`${LLM_CONTEXT_KEY.AttachedFile}${ruleUri.displayName}\``,
+          );
+        }
+      }
       return handleAgentReply({ message: processedContent, images, agentId, command, reportExtra });
     },
     [handleAgentReply],