diff --git a/backend/src/ai-core/tools/database-tools.ts b/backend/src/ai-core/tools/database-tools.ts index 18f8bb043..1b308bd15 100644 --- a/backend/src/ai-core/tools/database-tools.ts +++ b/backend/src/ai-core/tools/database-tools.ts @@ -53,6 +53,23 @@ export function createDatabaseTools(isMongoDB: boolean): AIToolDefinition[] { }, }; + const searchDocumentationTool: AIToolDefinition = { + name: 'searchDocumentation', + description: + 'Searches the official Rocketadmin documentation at https://docs.rocketadmin.com and returns the most relevant pages with their titles, URLs, and content snippets. Use this when the user asks how to use Rocketadmin features (connections, dashboards, permissions, groups, master password, widgets, integrations, settings, SSO, secrets, etc.) or when a question is about the product rather than the data in the connected database.', + parameters: { + type: 'object', + properties: { + query: { + type: 'string', + description: 'A short search query describing what to look up in the Rocketadmin documentation.', + }, + }, + required: ['query'], + additionalProperties: false, + }, + }; + const tools: AIToolDefinition[] = [getTableStructureTool]; if (isMongoDB) { @@ -61,6 +78,8 @@ export function createDatabaseTools(isMongoDB: boolean): AIToolDefinition[] { tools.push(executeRawSqlTool); } + tools.push(searchDocumentationTool); + return tools; } diff --git a/backend/src/ai-core/tools/documentation-search.ts b/backend/src/ai-core/tools/documentation-search.ts new file mode 100644 index 000000000..ed2d0a8fc --- /dev/null +++ b/backend/src/ai-core/tools/documentation-search.ts @@ -0,0 +1,98 @@ +import axios from 'axios'; + +const ALGOLIA_APP_ID = '31P3X3M1EE'; +const ALGOLIA_SEARCH_API_KEY = 'fe7422b190b4ec77f8e60c80a3a3ed8a'; +const ALGOLIA_INDEX_NAME = 'rocketadmin-docs'; +const ALGOLIA_SEARCH_URL = `https://${ALGOLIA_APP_ID}-dsn.algolia.net/1/indexes/${ALGOLIA_INDEX_NAME}/query`; + +const DEFAULT_HITS_PER_PAGE = 5; +const MAX_HITS_PER_PAGE = 10; +const MAX_CONTENT_LENGTH = 800; +const REQUEST_TIMEOUT_MS = 10000; + +export interface DocumentationSearchHit { + title: string; + url: string; + content: string; +} + +interface AlgoliaHierarchy { + lvl0?: string | null; + lvl1?: string | null; + lvl2?: string | null; + lvl3?: string | null; + lvl4?: string | null; + lvl5?: string | null; + lvl6?: string | null; +} + +interface AlgoliaHit { + url?: string; + content?: string | null; + hierarchy?: AlgoliaHierarchy; + type?: string; +} + +interface AlgoliaSearchResponse { + hits: AlgoliaHit[]; +} + +export async function searchDocumentation(query: string, hitsPerPage?: number): Promise { + const trimmedQuery = query?.trim(); + if (!trimmedQuery) { + return []; + } + + const limit = Math.min(Math.max(hitsPerPage ?? DEFAULT_HITS_PER_PAGE, 1), MAX_HITS_PER_PAGE); + + const response = await axios.post( + ALGOLIA_SEARCH_URL, + { + query: trimmedQuery, + hitsPerPage: limit, + attributesToRetrieve: ['hierarchy', 'content', 'url', 'type'], + attributesToSnippet: ['content:50'], + }, + { + headers: { + 'X-Algolia-Application-Id': ALGOLIA_APP_ID, + 'X-Algolia-API-Key': ALGOLIA_SEARCH_API_KEY, + 'Content-Type': 'application/json', + }, + timeout: REQUEST_TIMEOUT_MS, + }, + ); + + const hits = response.data?.hits ?? []; + return hits.map(buildHit).filter((hit) => hit.url && (hit.title || hit.content)); +} + +function buildHit(hit: AlgoliaHit): DocumentationSearchHit { + const title = formatHierarchy(hit.hierarchy); + const rawContent = (hit.content ?? '').replace(/\s+/g, ' ').trim(); + const content = rawContent.length > MAX_CONTENT_LENGTH ? `${rawContent.slice(0, MAX_CONTENT_LENGTH)}…` : rawContent; + return { + title, + url: hit.url ?? '', + content, + }; +} + +function formatHierarchy(hierarchy: AlgoliaHierarchy | undefined): string { + if (!hierarchy) { + return ''; + } + const parts = [ + hierarchy.lvl0, + hierarchy.lvl1, + hierarchy.lvl2, + hierarchy.lvl3, + hierarchy.lvl4, + hierarchy.lvl5, + hierarchy.lvl6, + ] + .filter((part): part is string => Boolean(part)) + .map((part) => part.replace(/​|‌|‍/g, '').trim()) + .filter(Boolean); + return parts.join(' › '); +} diff --git a/backend/src/ai-core/tools/prompts.ts b/backend/src/ai-core/tools/prompts.ts index b0f7081fe..18f09997b 100644 --- a/backend/src/ai-core/tools/prompts.ts +++ b/backend/src/ai-core/tools/prompts.ts @@ -16,7 +16,7 @@ Current date and time: ${currentDatetime} Tool responses are encoded in TOON (Token-Oriented Object Notation) format - a compact, human-readable format similar to YAML with CSV-style tabular arrays. Parse it naturally. -Please follow these steps EXACTLY: +Please follow these steps EXACTLY when answering data questions: 1. First, always use the getTableStructure tool to analyze the table schema and understand available columns 2. If the question requires data from related tables, note their relationships 3. Generate an appropriate query that answers the user's question precisely @@ -25,8 +25,13 @@ Please follow these steps EXACTLY: 6. After receiving query results, explain them to the user in a clear, conversational way 7. Include explanations of your approach when helpful +When the user asks how to use Rocketadmin itself (features, configuration, connections, dashboards, permissions, groups, master password, widgets, integrations, SSO, secrets, settings, API, etc.) rather than asking about the data in their database: +- Call the searchDocumentation tool with a concise query that captures the user's question +- Base your answer on the returned snippets and cite the relevant documentation URLs in your response +- You may combine searchDocumentation with the data tools when a question needs both product knowledge and data from the database + IMPORTANT: -- You MUST execute your generated queries using the appropriate tool - this is required for every question +- You MUST execute your generated queries using the appropriate tool - this is required for every data question - After generating a SQL query, immediately call executeRawSql with that query - For MongoDB databases, call executeAggregationPipeline with the aggregation pipeline - The user cannot see the query results until you execute it with the appropriate tool diff --git a/backend/src/entities/ai/use-cases/request-info-from-table-with-ai-v7.use.case.ts b/backend/src/entities/ai/use-cases/request-info-from-table-with-ai-v7.use.case.ts index e3ae9cd99..314dd6f3f 100644 --- a/backend/src/entities/ai/use-cases/request-info-from-table-with-ai-v7.use.case.ts +++ b/backend/src/entities/ai/use-cases/request-info-from-table-with-ai-v7.use.case.ts @@ -10,6 +10,7 @@ import { AIToolCall, AIToolDefinition } from '../../../ai-core/interfaces/ai-pro import { AIProviderType } from '../../../ai-core/interfaces/ai-service.interface.js'; import { AICoreService } from '../../../ai-core/services/ai-core.service.js'; import { createDatabaseTools } from '../../../ai-core/tools/database-tools.js'; +import { searchDocumentation } from '../../../ai-core/tools/documentation-search.js'; import { createDatabaseQuerySystemPrompt } from '../../../ai-core/tools/prompts.js'; import { isValidMongoDbCommand, isValidSQLQuery, wrapQueryWithLimit } from '../../../ai-core/tools/query-validators.js'; import { MessageBuilder } from '../../../ai-core/utils/message-builder.js'; @@ -275,6 +276,16 @@ export class RequestInfoFromTableWithAIUseCaseV7 break; } + case 'searchDocumentation': { + const query = toolCall.arguments.query as string; + if (!query) { + throw new Error('Missing required function argument "query"'); + } + const docsResults = await searchDocumentation(query); + result = encodeToToon({ query, results: docsResults }); + break; + } + default: result = encodeError({ error: `Unknown tool: ${toolCall.name}` }); } diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-ai-search-documentation-tool-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-ai-search-documentation-tool-e2e.test.ts new file mode 100644 index 000000000..3915dd9a3 --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-ai-search-documentation-tool-e2e.test.ts @@ -0,0 +1,257 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { faker } from '@faker-js/faker'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import { ValidationError } from 'class-validator'; +import cookieParser from 'cookie-parser'; +import nock from 'nock'; +import request from 'supertest'; +import { DataSource } from 'typeorm'; +import { AICoreService } from '../../../src/ai-core/services/ai-core.service.js'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { BaseType } from '../../../src/common/data-injection.tokens.js'; +import { AiChatMessageEntity } from '../../../src/entities/ai/ai-conversation-history/ai-chat-messages/ai-chat-message.entity.js'; +import { MessageRole } from '../../../src/entities/ai/ai-conversation-history/ai-chat-messages/message-role.enum.js'; +import { WinstonLogger } from '../../../src/entities/logging/winston-logger.js'; +import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { createTestPostgresTableWithSchema } from '../../utils/create-test-table.js'; +import { dropTestTables } from '../../utils/drop-test-tables.js'; +import { getTestData } from '../../utils/get-test-data.js'; +import { + createInitialTestUser, + registerUserAndReturnUserInfo, +} from '../../utils/register-user-and-return-user-info.js'; +import { setSaasEnvVariable } from '../../utils/set-saas-env-variable.js'; +import { TestUtils } from '../../utils/test.utils.js'; + +const ALGOLIA_ORIGIN = 'https://31P3X3M1EE-dsn.algolia.net'; +const ALGOLIA_PATH = '/1/indexes/rocketadmin-docs/query'; + +const mockFactory = new MockFactory(); +let app: INestApplication; +let _testUtils: TestUtils; +const testTables: Array = []; + +let iterationCounter = 0; +let capturedSearchQuery: string | null = null; + +function buildIterableStream(chunks: Array>) { + return { + async *[Symbol.asyncIterator]() { + for (const chunk of chunks) { + yield chunk; + } + }, + }; +} + +const mockAICoreService = { + streamChatWithToolsAndProvider: async (_provider: unknown, _messages: unknown, tools: Array = []) => { + if (!Array.isArray(tools) || tools.length === 0) { + return buildIterableStream([{ type: 'text', content: 'Master password help' }]); + } + iterationCounter += 1; + if (iterationCounter === 1) { + return buildIterableStream([ + { + type: 'tool_call', + toolCall: { + id: 'doc-search-call-1', + name: 'searchDocumentation', + arguments: { query: 'how to enable master password' }, + }, + }, + ]); + } + return buildIterableStream([ + { + type: 'text', + content: + 'Based on the docs: enable master password from the connection settings page. See https://docs.rocketadmin.com/Reference/MasterPassword for the full guide.', + }, + ]); + }, + complete: async () => 'mocked', + chat: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChat: async () => buildIterableStream([{ type: 'text', content: 'mocked' }]), + chatWithTools: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithTools: async () => buildIterableStream([{ type: 'text', content: 'mocked' }]), + chatWithToolsAndProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + getDefaultProvider: () => 'bedrock', + setDefaultProvider: () => {}, + getAvailableProviders: () => [], +}; + +test.before(async () => { + setSaasEnvVariable(); + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }) + .overrideProvider(AICoreService) + .useValue(mockAICoreService) + .compile(); + + _testUtils = moduleFixture.get(TestUtils); + + app = moduleFixture.createNestApplication(); + app.use(cookieParser()); + app.useGlobalFilters(new AllExceptionsFilter(app.get(WinstonLogger))); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + await createInitialTestUser(app); + app.getHttpServer().listen(0); +}); + +test.after.always(async () => { + try { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgresSchema; + await dropTestTables(testTables, connectionToTestDB); + await Cacher.clearAllCache(); + nock.cleanAll(); + nock.enableNetConnect(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +test.beforeEach(() => { + iterationCounter = 0; + capturedSearchQuery = null; + nock.cleanAll(); +}); + +test.serial('searchDocumentation tool: AI tool_call triggers Algolia call and final answer is persisted', async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgresSchema; + const { token } = await registerUserAndReturnUserInfo(app); + const { testTableName } = await createTestPostgresTableWithSchema(connectionToTestDB); + testTables.push(testTableName); + + const algoliaScope = nock(ALGOLIA_ORIGIN) + .post(ALGOLIA_PATH, (body) => { + capturedSearchQuery = typeof body.query === 'string' ? body.query : null; + return true; + }) + .reply(200, { + hits: [ + { + url: 'https://docs.rocketadmin.com/Reference/MasterPassword', + content: 'Master password protects the connection encryption key.', + hierarchy: { lvl0: 'Reference', lvl1: 'Master password' }, + }, + ], + }); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + t.is(createConnectionResponse.status, 201); + const connectionRO = JSON.parse(createConnectionResponse.text); + + await request(app.getHttpServer()) + .post(`/connection/properties/${connectionRO.id}`) + .send({ allow_ai_requests: true }) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + + const aiResponse = await request(app.getHttpServer()) + .post(`/ai/v4/request/${connectionRO.id}?tableName=${testTableName}`) + .send({ user_message: 'How do I turn on the master password?' }) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + + t.is(aiResponse.status, 201); + t.true(algoliaScope.isDone(), 'expected the use case to call the Algolia search endpoint'); + t.is(capturedSearchQuery, 'how to enable master password'); + + const threadId = aiResponse.headers['x-ai-thread-id']; + t.truthy(threadId); + + const dataSource = app.get(BaseType.DATA_SOURCE); + const messageRepository = dataSource.getRepository(AiChatMessageEntity); + const messages = await messageRepository.find({ + where: { ai_chat_id: threadId }, + order: { created_at: 'ASC' }, + }); + + t.is(messages.length, 2); + t.is(messages[0].role, MessageRole.user); + t.is(messages[0].message, 'How do I turn on the master password?'); + t.is(messages[1].role, MessageRole.ai); + t.true(messages[1].message.includes('docs.rocketadmin.com/Reference/MasterPassword')); +}); + +test.serial('searchDocumentation tool: empty query argument is rejected without calling Algolia', async (t) => { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgresSchema; + const { token } = await registerUserAndReturnUserInfo(app); + const { testTableName } = await createTestPostgresTableWithSchema(connectionToTestDB); + testTables.push(testTableName); + + const algoliaScope = nock(ALGOLIA_ORIGIN).post(ALGOLIA_PATH).reply(200, { hits: [] }); + + const createConnectionResponse = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + const connectionRO = JSON.parse(createConnectionResponse.text); + + await request(app.getHttpServer()) + .post(`/connection/properties/${connectionRO.id}`) + .send({ allow_ai_requests: true }) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + + const localMock = { + ...mockAICoreService, + streamChatWithToolsAndProvider: async (_provider: unknown, _messages: unknown, tools: Array = []) => { + if (!Array.isArray(tools) || tools.length === 0) { + return buildIterableStream([{ type: 'text', content: 'docs?' }]); + } + iterationCounter += 1; + if (iterationCounter === 1) { + return buildIterableStream([ + { + type: 'tool_call', + toolCall: { + id: 'doc-search-call-2', + name: 'searchDocumentation', + arguments: { query: '' }, + }, + }, + ]); + } + return buildIterableStream([{ type: 'text', content: 'Sorry, I could not search the documentation.' }]); + }, + }; + const original = mockAICoreService.streamChatWithToolsAndProvider; + mockAICoreService.streamChatWithToolsAndProvider = localMock.streamChatWithToolsAndProvider; + + try { + const aiResponse = await request(app.getHttpServer()) + .post(`/ai/v4/request/${connectionRO.id}?tableName=${testTableName}`) + .send({ user_message: 'docs?' }) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + + t.is(aiResponse.status, 201); + t.false(algoliaScope.isDone(), 'Algolia should not have been called when the query is empty'); + } finally { + mockAICoreService.streamChatWithToolsAndProvider = original; + } +}); diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-documentation-search.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-documentation-search.test.ts new file mode 100644 index 000000000..f0c9d3bc9 --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-documentation-search.test.ts @@ -0,0 +1,113 @@ +import test from 'ava'; +import nock from 'nock'; +import { searchDocumentation } from '../../../src/ai-core/tools/documentation-search.js'; + +const ALGOLIA_ORIGIN = 'https://31P3X3M1EE-dsn.algolia.net'; +const ALGOLIA_PATH = '/1/indexes/rocketadmin-docs/query'; + +test.before(() => { + nock.disableNetConnect(); +}); + +test.after.always(() => { + nock.cleanAll(); + nock.enableNetConnect(); +}); + +test.afterEach(() => { + nock.cleanAll(); +}); + +test.serial('returns an empty list when query is blank without calling Algolia', async (t) => { + const scope = nock(ALGOLIA_ORIGIN).post(ALGOLIA_PATH).reply(200, { hits: [] }); + + const blankResults = await searchDocumentation(' '); + t.deepEqual(blankResults, []); + t.false(scope.isDone(), 'no HTTP call should have been made for a blank query'); +}); + +test.serial('parses Algolia hits into title/url/content triples', async (t) => { + nock(ALGOLIA_ORIGIN) + .post(ALGOLIA_PATH, (body) => body.query === 'master password' && body.hitsPerPage === 5) + .matchHeader('x-algolia-application-id', '31P3X3M1EE') + .matchHeader('x-algolia-api-key', 'fe7422b190b4ec77f8e60c80a3a3ed8a') + .reply(200, { + hits: [ + { + url: 'https://docs.rocketadmin.com/Reference/MasterPassword#how-it-works', + content: 'The master password should be distributed to all users.', + hierarchy: { + lvl0: 'Reference', + lvl1: 'Master password', + lvl2: null, + lvl3: 'How it works​', + lvl4: null, + lvl5: null, + lvl6: null, + }, + }, + { + url: 'https://docs.rocketadmin.com/Reference/permissions', + content: ' Permissions control which users see which tables. ', + hierarchy: { lvl0: 'Reference', lvl1: 'Permissions' }, + }, + ], + }); + + const results = await searchDocumentation('master password'); + + t.is(results.length, 2); + t.is(results[0].url, 'https://docs.rocketadmin.com/Reference/MasterPassword#how-it-works'); + t.is(results[0].title, 'Reference › Master password › How it works'); + t.is(results[0].content, 'The master password should be distributed to all users.'); + t.is(results[1].title, 'Reference › Permissions'); + t.is(results[1].content, 'Permissions control which users see which tables.'); +}); + +test.serial('truncates very long content snippets', async (t) => { + const longContent = 'x'.repeat(2000); + nock(ALGOLIA_ORIGIN) + .post(ALGOLIA_PATH) + .reply(200, { + hits: [ + { + url: 'https://docs.rocketadmin.com/quickstart', + content: longContent, + hierarchy: { lvl0: 'Quickstart' }, + }, + ], + }); + + const [hit] = await searchDocumentation('quickstart'); + t.true(hit.content.length <= 801); + t.true(hit.content.endsWith('…')); +}); + +test.serial('clamps hitsPerPage to the allowed range', async (t) => { + let receivedHitsPerPage: number | undefined; + nock(ALGOLIA_ORIGIN) + .post(ALGOLIA_PATH, (body) => { + receivedHitsPerPage = body.hitsPerPage; + return true; + }) + .reply(200, { hits: [] }); + + await searchDocumentation('anything', 999); + t.is(receivedHitsPerPage, 10); +}); + +test.serial('drops hits that have no url or content', async (t) => { + nock(ALGOLIA_ORIGIN) + .post(ALGOLIA_PATH) + .reply(200, { + hits: [ + { url: '', content: 'no url so should drop', hierarchy: { lvl0: 'X' } }, + { url: 'https://docs.rocketadmin.com/x', content: '', hierarchy: {} }, + { url: 'https://docs.rocketadmin.com/keep', content: 'kept', hierarchy: { lvl0: 'Keep' } }, + ], + }); + + const results = await searchDocumentation('mixed'); + t.is(results.length, 1); + t.is(results[0].url, 'https://docs.rocketadmin.com/keep'); +});