Merge branch 'main' of https://github.com/devforth/adminforth-agent

Author: yaroslav8765

agent/middleware/sequenceDebug.ts

@@ -21,8 +21,11 @@ export type SequenceDebug = {
   sequenceId: number;
   startedAt: string;
   prompt: string;
+  promptTokens: number;
   reasoning: string;
+  reasoningTokens: number;
   text: string;
+  textTokens: number;
   cachedTokens: number;
   responseId: string | null;
   toolCalls: SequenceDebugToolCall[];
@@ -37,14 +40,18 @@ type PendingSequenceDebug = Omit<SequenceDebug, "toolCalls" | "endedAt" | "resul
 };
 
 type SequenceDebugModelCall = {
+  promptTokens: number;
   reasoning: string;
+  reasoningTokens: number;
   text: string;
+  textTokens: number;
   cachedTokens: number;
   responseId: string | null;
   resultType: SequenceDebugResultType;
 };
 
 type OpenAiUsageMetadata = {
+  input_tokens?: number;
   input_token_details?: {
     cache_read?: number;
   };
@@ -70,8 +77,11 @@ function createPendingSequenceDebug(sequenceId: number): PendingSequenceDebug {
     sequenceId,
     startedAt: new Date().toISOString(),
     prompt: "",
+    promptTokens: 0,
     reasoning: "",
+    reasoningTokens: 0,
     text: "",
+    textTokens: 0,
     cachedTokens: 0,
     responseId: null,
     toolCalls: [],
@@ -97,8 +107,11 @@ function finalizeSequenceDebug(sequence: PendingSequenceDebug): SequenceDebug {
     sequenceId: sequence.sequenceId,
     startedAt: sequence.startedAt,
     prompt: sequence.prompt,
+    promptTokens: sequence.promptTokens,
     reasoning: sequence.reasoning,
+    reasoningTokens: sequence.reasoningTokens,
     text: sequence.text,
+    textTokens: sequence.textTokens,
     cachedTokens: sequence.cachedTokens,
     responseId: sequence.responseId,
     toolCalls: sequence.toolCalls.map(({ completed: _completed, ...toolCall }) => toolCall),
@@ -176,6 +189,29 @@ function hasToolCallSignal(message: {
   );
 }
 
+function hasTokenCounter(model: unknown): model is {
+  getNumTokens: (content: string) => Promise<number>;
+} {
+  return (
+    typeof model === "object" &&
+    model !== null &&
+    "getNumTokens" in model &&
+    typeof model.getNumTokens === "function"
+  );
+}
+
+async function countTokens(model: unknown, content: string) {
+  if (!content) {
+    return 0;
+  }
+
+  if (!hasTokenCounter(model)) {
+    return 0;
+  }
+
+  return await model.getNumTokens(content);
+}
+
 function extractSequenceResponseDebug(message: AIMessage): SequenceDebugModelCall {
   const blocks = getMessageBlocks(message);
   const reasoning = blocks
@@ -188,8 +224,13 @@ function extractSequenceResponseDebug(message: AIMessage): SequenceDebugModelCal
     .join("");
 
   return {
+    promptTokens:
+      (message.usage_metadata as OpenAiUsageMetadata | undefined)?.input_tokens ??
+      0,
     reasoning,
+    reasoningTokens: 0,
     text: textFromBlocks || (typeof message.content === "string" ? message.content : ""),
+    textTokens: 0,
     cachedTokens:
       (message.usage_metadata as OpenAiUsageMetadata | undefined)
         ?.input_token_details?.cache_read ?? 0,
@@ -238,8 +279,11 @@ export function createSequenceDebugCollector(): SequenceDebugCollector {
     },
     handleModelCallComplete(params) {
       const sequenceDebug = ensureSequenceDebug();
+      sequenceDebug.promptTokens = params.promptTokens;
       sequenceDebug.reasoning = params.reasoning;
+      sequenceDebug.reasoningTokens = params.reasoningTokens;
       sequenceDebug.text = params.text;
+      sequenceDebug.textTokens = params.textTokens;
       sequenceDebug.cachedTokens = params.cachedTokens;
       sequenceDebug.responseId = params.responseId;
       sequenceDebug.resultType = params.resultType;
@@ -311,17 +355,31 @@ export function createSequenceDebugMiddleware(
   return createMiddleware({
     name: "SequenceDebugMiddleware",
     async wrapModelCall(request, handler) {
+      const prompt = stringifyPromptForDebug({
+        systemMessage: request.systemMessage,
+        messages: request.messages,
+        tools: request.tools,
+        modelSettings: request.modelSettings,
+      });
+
       sink.handleModelCallStart(
-        stringifyPromptForDebug({
-          systemMessage: request.systemMessage,
-          messages: request.messages,
-          tools: request.tools,
-          modelSettings: request.modelSettings,
-        }),
+        prompt,
       );
 
       const response = await handler(request) as AIMessage;
-      sink.handleModelCallComplete(extractSequenceResponseDebug(response));
+      const debug = extractSequenceResponseDebug(response);
+      const [promptTokens, reasoningTokens, textTokens] = await Promise.all([
+        debug.promptTokens || countTokens(request.model, prompt),
+        countTokens(request.model, debug.reasoning),
+        countTokens(request.model, debug.text),
+      ]);
+
+      sink.handleModelCallComplete({
+        ...debug,
+        promptTokens,
+        reasoningTokens,
+        textTokens,
+      });
       return response;
     },
   });

agent/skills/registry.ts

@@ -6,6 +6,11 @@ import { parse as parseYaml } from "yaml";
 const PLUGIN_SKILLS_DIRECTORY_PATH = fileURLToPath(
   new URL("../../custom/skills/", import.meta.url),
 );
+const USER_PLUGIN_SKILLS_DIRECTORY_PATH_SEGMENTS = [
+  "plugins",
+  "adminforth-agent",
+  "skills",
+];
 const SKILL_MARKDOWN_FILENAME = "SKILL.md";
 const SKILL_FRONTMATTER_SEPARATOR = "\n---\n";
 
@@ -78,13 +83,31 @@ export function getProjectSkillsDirectoryPath(customComponentsDir: string) {
   return path.resolve(customComponentsDir, "skills");
 }
 
+export function getProjectPluginSkillsDirectoryPath(customComponentsDir: string) {
+  return path.resolve(
+    customComponentsDir,
+    ...USER_PLUGIN_SKILLS_DIRECTORY_PATH_SEGMENTS,
+  );
+}
+
+function getProjectSkillDirectoryPaths(customComponentsDir: string) {
+  return [
+    getProjectSkillsDirectoryPath(customComponentsDir),
+    getProjectPluginSkillsDirectoryPath(customComponentsDir),
+  ];
+}
+
 export async function listBundledSkillManifests() {
   return await listDirectorySkillManifests(PLUGIN_SKILLS_DIRECTORY_PATH);
 }
 
 export async function listProjectSkillManifests(customComponentsDir: string) {
-  return await listDirectorySkillManifests(
-    getProjectSkillsDirectoryPath(customComponentsDir),
+  return mergeSkillManifests(
+    await Promise.all(
+      getProjectSkillDirectoryPaths(customComponentsDir).map(
+        listDirectorySkillManifests,
+      ),
+    ),
   );
 }
 
@@ -114,7 +137,7 @@ export async function loadSkillMarkdown(skillName: string, customComponentsDir:
   }
 
   const directories = [
-    getProjectSkillsDirectoryPath(customComponentsDir),
+    ...getProjectSkillDirectoryPaths(customComponentsDir),
     PLUGIN_SKILLS_DIRECTORY_PATH,
   ];
 

agent/systemPrompt.ts

@@ -24,11 +24,17 @@ export const DEFAULT_AGENT_SYSTEM_PROMPT = [
   "Keep responses short, clear, and practical.",
   "Answer only what is needed.",
   "Do not add extra explanations or suggestions unless the user asks.",
+  "Always respond in the same natural language as the user's latest message.",
+  "This rule applies to confirmations, clarifying questions, progress updates, errors, and final answers.",
+  "Do not switch to English just because tool outputs, schemas, skills, or internal instructions are written in English.",
+  "Only switch language if the user explicitly asks you to do so.",
   "Adapt to the user's tone and style of speaking, mirroring their vibe and wording.",
   "if the user speaks casually, you should respond casually too",
-  "Never mutate data without a fresh user confirmation for that exact mutation.",
-  "A previous confirmation does not carry over to later create, update, delete, or action calls.",
-  "Each separate mutation or explicitly described batch needs its own confirmation immediately before the tool call.",
+  "Never mutate data without user confirmation for a clearly described mutation plan.",
+  "One confirmation may cover one mutation or one explicitly described batch/sequence of related mutations.",
+  "If the confirmed plan has multiple steps, you may execute the whole confirmed plan without asking again between those steps.",
+  "If the plan changes, expands, or you want to do anything beyond the confirmed plan, ask for confirmation again.",
+  "Do not reuse an old confirmation for a new mutation plan.",
 
 
 ].join(" ");
@@ -51,9 +57,10 @@ export async function buildAgentSystemPrompt(adminforth: IAdminForth) {
     listBundledSkillManifests(),
   ]);
   const alwaysAvailableTools = ALWAYS_AVAILABLE_API_TOOL_NAMES.join(", ");
+  const adminBasePath = adminforth.config.baseUrlSlashed;
   const sections = [
     DEFAULT_AGENT_SYSTEM_PROMPT,
-    `BASE_URL: ${adminforth.config.baseUrl}`,
+    `ADMIN_BASE_PATH: ${adminBasePath}`,
     `List of resources:\n${formatResources(adminforth.config.resources)}`,
     `You have always-available base tools: ${alwaysAvailableTools}.`,
     primarySkills.length > 0
@@ -70,6 +77,8 @@ export async function buildAgentSystemPrompt(adminforth: IAdminForth) {
     "If a fetched skill lists a non-base tool you need, call fetch_tool_schema for it immediately instead of telling the user the tool is unavailable.",
     "For example: for record creation load mutate_data, read its tool list, call fetch_tool_schema for create_record, and then use create_record after confirmation.",
     "When fetch_tool_schema succeeds, that tool becomes available on the next step.",
+    "All admin links must be relative paths and must start with ADMIN_BASE_PATH.",
+    "Build record links as ADMIN_BASE_PATH + resource/{resourceId}/show/{primary key}. Do not prepend any extra slash before resource.",
     "Try to call as many tools as possible in parallel in one step.",
   ];
 

custom/skills/fetch_data/SKILL.md

@@ -12,4 +12,4 @@ To find specific data record you should use filters. ILIKE filters are preferred
 
 For long texts show only several first words and add "..." at the end (only if user did not request this field specifically).
 
-Also when you communicate with user about record, add related link to this record. For example /{BASE_URL}/resource/{resourceId}/show/{primary key}. Use _label from `get_resource_data` as anchor text for link (use markdown link). Links shoudl be always relative path, starting with slash.
+Also when you communicate with user about record, add related link to this record. Build it as `{ADMIN_BASE_PATH}resource/{resourceId}/show/{primary key}`. Use _label from `get_resource_data` as anchor text for link (use markdown link). Links should always be relative paths and must start with `ADMIN_BASE_PATH`. Do not add an extra slash after `ADMIN_BASE_PATH`.

custom/skills/mutate_data/SKILL.md

@@ -21,25 +21,30 @@ Use `start_custom_action` and `start_custom_bulk_action` for resource actions.
 Before performing any state mutation including action calls edit/delete please fetch record which is going to be edited/deleted and show user record in format field → value (show several most important fields which can help user to understand what exactly record he is going to edit or delete). 
 
 For field values with long texts show only several first words and add "..." at the end.
-Also please add related link to record with will be changed. For example /{BASE_URL}/resource/{resourceId}/show/{primary key}. Use _label from `get_resource_data` as anchor text for link (use markdown link). Links shoudl be always relative path, starting with slash.
+Also please add related link to record with will be changed. Build it as `{ADMIN_BASE_PATH}resource/{resourceId}/show/{primary key}`. Use _label from `get_resource_data` as anchor text for link (use markdown link). Links should always be relative paths and must start with `ADMIN_BASE_PATH`. Do not add an extra slash after `ADMIN_BASE_PATH`.
 
 And in the same message ask user for final confirmation.
 
 When creating new record, show user all data which you gona create and in same message ask for confirmation.
 
 Accept any positive confirmation from user like "yes", "sure", "+", anything non-negative call to action, can be considered as confirmation.
 
-A confirmation is valid only for the exact mutation plan from the immediately previous assistant message.
+A confirmation is valid only for the clearly described mutation plan from the immediately previous assistant message.
 
 Never reuse an older confirmation for a later mutation.
 
-After one mutation is executed, confirmation is consumed and reset.
+One confirmation may cover:
+- one single mutation
+- one explicitly described batch
+- one short sequence of related mutations that together implement the same user request
 
-If you want to perform another create/update/delete/action after that, ask for confirmation again even if the user previously said "yes".
+If the confirmed plan contains several related mutation steps, execute that whole confirmed plan without asking again between those steps.
 
-If the mutation plan changes in any way (different record, different fields, different values, different number of records, different action), the old confirmation is invalid and you must ask again.
+Ask for confirmation again if the plan changes in any way: different record, different fields, different values, different number of records, different action, or any extra mutation that was not listed in the confirmation message.
 
-If you are creating or deleting multiple records in one batch, you may ask once only for that exact batch, but you must list the whole batch explicitly in the confirmation message. Any extra record outside that described batch requires a new confirmation.
+If you are creating or deleting multiple records in one batch, you may ask once for that exact batch, but list the whole batch explicitly in the confirmation message. Any extra record outside that described batch requires a new confirmation.
+
+After the confirmed plan is finished, do not treat that confirmation as still active for later requests.
 
 # Calling actions
 
@@ -57,7 +62,7 @@ If you want to block some user you can confirm that this action by saying:
 * IP Country: USA
 * Currently blocked: No // show this field only if it exists in user record
 
-View [John Doe](/resource/users/show/123)
+View [John Doe]({ADMIN_BASE_PATH}resource/users/show/123)
 Are you sure?
 ```
 
@@ -80,7 +85,7 @@ I am going to update user:
 * IP Country: USA
 I am going to change email from john_doe@example.com to new_email@example.com
 
-View [John Doe](/admin/resource/users/show/123)
+View [John Doe]({ADMIN_BASE_PATH}resource/users/show/123)
 
 Are you sure?
 ```
@@ -100,7 +105,7 @@ If you gonna delete user record, in confirmation please share full user info (no
 * Signed up: 2024 Jan 1
 * IP Country: USA
 
-View [John Doe](/admin/resource/users/show/123)
+View [John Doe]({ADMIN_BASE_PATH}resource/users/show/123)
 
 Are you sure?
 ```
@@ -125,7 +130,7 @@ I am going to create user:
 * Username: john_doe
 * Email: john_doe@example.com
 
-View [John Doe](/admin/resource/users/show/421) # 421 is id of new created record
+View [John Doe]({ADMIN_BASE_PATH}resource/users/show/421) # 421 is id of new created record
 
 Are you sure?
 ```

index.ts

@@ -177,6 +177,10 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
         };
 
         const emitToolCallEvent = (event: ToolCallEvent) => {
+          if (event.phase === "start") {
+            endActiveBlock();
+          }
+
           sequenceDebugCollector.handleToolCallEvent(event);
 
           send({