diff --git a/backend/package.json b/backend/package.json index a23a7fb35..28ffa9324 100644 --- a/backend/package.json +++ b/backend/package.json @@ -79,6 +79,7 @@ "langchain": "^1.2.34", "lru-cache": "^11.2.7", "nanoid": "5.1.7", + "node-sql-parser": "^5.3.0", "nodemailer": "^8.0.4", "nunjucks": "^3.2.4", "openai": "^6.32.0", diff --git a/backend/src/ai-core/tools/prompts.ts b/backend/src/ai-core/tools/prompts.ts index 62ce9778b..6383613e7 100644 --- a/backend/src/ai-core/tools/prompts.ts +++ b/backend/src/ai-core/tools/prompts.ts @@ -114,6 +114,16 @@ export function convertDbTypeToReadableString(dataType: ConnectionTypesEnum): st case ConnectionTypesEnum.ibmdb2: case ConnectionTypesEnum.agent_ibmdb2: return 'IBM DB2'; + case ConnectionTypesEnum.clickhouse: + case ConnectionTypesEnum.agent_clickhouse: + return 'ClickHouse'; + case ConnectionTypesEnum.dynamodb: + return 'DynamoDB'; + case ConnectionTypesEnum.cassandra: + case ConnectionTypesEnum.agent_cassandra: + return 'Cassandra'; + case ConnectionTypesEnum.elasticsearch: + return 'Elasticsearch'; default: return 'Unknown Database'; } diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index 94211cae2..bd5360d0a 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -8,6 +8,7 @@ import { GlobalDatabaseContext } from './common/application/global-database-cont import { BaseType, UseCaseType } from './common/data-injection.tokens.js'; import { AIModule } from './entities/ai/ai.module.js'; import { ApiKeyModule } from './entities/api-key/api-key.module.js'; +import { CedarAuthorizationModule } from './entities/cedar-authorization/cedar-authorization.module.js'; import { CompanyFaviconModule } from './entities/company-favicon/company-favicon.module.js'; import { CompanyInfoModule } from './entities/company-info/company-info.module.js'; import { CompanyLogoModule } from './entities/company-logo/company-logo.module.js'; @@ -30,6 +31,7 @@ import { TableActionModule } from './entities/table-actions/table-actions-module import { TableCategoriesModule } from './entities/table-categories/table-categories.module.js'; import { TableFiltersModule } from './entities/table-filters/table-filters.module.js'; import { TableLogsModule } from './entities/table-logs/table-logs.module.js'; +import { TableSchemaModule } from './entities/table-schema/table-schema.module.js'; import { TableSettingsModule } from './entities/table-settings/common-table-settings/table-settings.module.js'; import { PersonalTableSettingsModule } from './entities/table-settings/personal-table-settings/personal-table-settings.module.js'; import { UserModule } from './entities/user/user.module.js'; @@ -37,10 +39,9 @@ import { UserActionModule } from './entities/user-actions/user-action.module.js' import { UserSecretModule } from './entities/user-secret/user-secret.module.js'; import { SignInAuditModule } from './entities/user-sign-in-audit/sign-in-audit.module.js'; import { DashboardModule } from './entities/visualizations/dashboard/dashboards.module.js'; -import { PanelPositionModule } from './entities/visualizations/panel-position/panel-position.module.js'; import { PanelModule } from './entities/visualizations/panel/panel.module.js'; +import { PanelPositionModule } from './entities/visualizations/panel-position/panel-position.module.js'; import { TableWidgetModule } from './entities/widget/table-widget.module.js'; -import { CedarAuthorizationModule } from './entities/cedar-authorization/cedar-authorization.module.js'; import { SaaSGatewayModule } from './microservices/gateways/saas-gateway.ts/saas-gateway.module.js'; import { SaasModule } from './microservices/saas-microservice/saas.module.js'; import { AppLoggerMiddleware } from './middlewares/logging-middleware/app-logger-middlewate.js'; @@ -94,6 +95,7 @@ import { GetHelloUseCase } from './use-cases-app/get-hello.use.case.js'; SharedJobsModule, TableCategoriesModule, UserSecretModule, + TableSchemaModule, SignInAuditModule, PersonalTableSettingsModule, S3WidgetModule, diff --git a/backend/src/common/application/global-database-context.interface.ts b/backend/src/common/application/global-database-context.interface.ts index 4b523951b..74aa75ed9 100644 --- a/backend/src/common/application/global-database-context.interface.ts +++ b/backend/src/common/application/global-database-context.interface.ts @@ -38,6 +38,8 @@ import { ITableFiltersCustomRepository } from '../../entities/table-filters/repo import { TableFiltersEntity } from '../../entities/table-filters/table-filters.entity.js'; import { TableInfoEntity } from '../../entities/table-info/table-info.entity.js'; import { ITableLogsRepository } from '../../entities/table-logs/repository/table-logs-repository.interface.js'; +import { ITableSchemaChangeRepository } from '../../entities/table-schema/repository/table-schema-change.repository.interface.js'; +import { TableSchemaChangeEntity } from '../../entities/table-schema/table-schema-change.entity.js'; import { ITableSettingsRepository } from '../../entities/table-settings/common-table-settings/repository/table-settings.repository.interface.js'; import { TableSettingsEntity } from '../../entities/table-settings/common-table-settings/table-settings.entity.js'; import { PersonalTableSettingsEntity } from '../../entities/table-settings/personal-table-settings/personal-table-settings.entity.js'; @@ -57,10 +59,10 @@ import { ISignInAuditRepository } from '../../entities/user-sign-in-audit/reposi import { SignInAuditEntity } from '../../entities/user-sign-in-audit/sign-in-audit.entity.js'; import { DashboardEntity } from '../../entities/visualizations/dashboard/dashboard.entity.js'; import { IDashboardRepository } from '../../entities/visualizations/dashboard/repository/dashboard.repository.interface.js'; +import { PanelEntity } from '../../entities/visualizations/panel/panel.entity.js'; +import { IPanelRepository } from '../../entities/visualizations/panel/repository/saved-db-query.repository.interface.js'; import { PanelPositionEntity } from '../../entities/visualizations/panel-position/panel-position.entity.js'; import { IPanelPositionRepository } from '../../entities/visualizations/panel-position/repository/panel-position.repository.interface.js'; -import { IPanelRepository } from '../../entities/visualizations/panel/repository/saved-db-query.repository.interface.js'; -import { PanelEntity } from '../../entities/visualizations/panel/panel.entity.js'; import { ITableWidgetsRepository } from '../../entities/widget/repository/table-widgets-repository.interface.js'; import { TableWidgetEntity } from '../../entities/widget/table-widget.entity.js'; import { IDatabaseContext } from '../database-context.interface.js'; @@ -106,4 +108,5 @@ export interface IGlobalDatabaseContext extends IDatabaseContext { panelPositionRepository: Repository & IPanelPositionRepository; userAiChatRepository: Repository & IUserAiChatRepository; aiChatMessageRepository: Repository & IAiChatMessageRepository; + tableSchemaChangeRepository: Repository & ITableSchemaChangeRepository; } diff --git a/backend/src/common/application/global-database-context.ts b/backend/src/common/application/global-database-context.ts index c8703645c..65ba690bb 100644 --- a/backend/src/common/application/global-database-context.ts +++ b/backend/src/common/application/global-database-context.ts @@ -65,6 +65,9 @@ import { TableInfoEntity } from '../../entities/table-info/table-info.entity.js' import { tableLogsCustomRepositoryExtension } from '../../entities/table-logs/repository/table-logs-custom-repository-extension.js'; import { ITableLogsRepository } from '../../entities/table-logs/repository/table-logs-repository.interface.js'; import { TableLogsEntity } from '../../entities/table-logs/table-logs.entity.js'; +import { customTableSchemaChangeRepositoryExtension } from '../../entities/table-schema/repository/custom-table-schema-change-repository-extension.js'; +import { ITableSchemaChangeRepository } from '../../entities/table-schema/repository/table-schema-change.repository.interface.js'; +import { TableSchemaChangeEntity } from '../../entities/table-schema/table-schema-change.entity.js'; import { ITableSettingsRepository } from '../../entities/table-settings/common-table-settings/repository/table-settings.repository.interface.js'; import { tableSettingsCustomRepositoryExtension } from '../../entities/table-settings/common-table-settings/repository/table-settings-custom-repository-extension.js'; import { TableSettingsEntity } from '../../entities/table-settings/common-table-settings/table-settings.entity.js'; @@ -101,12 +104,12 @@ import { SignInAuditEntity } from '../../entities/user-sign-in-audit/sign-in-aud import { DashboardEntity } from '../../entities/visualizations/dashboard/dashboard.entity.js'; import { IDashboardRepository } from '../../entities/visualizations/dashboard/repository/dashboard.repository.interface.js'; import { dashboardCustomRepositoryExtension } from '../../entities/visualizations/dashboard/repository/dashboard-custom-repository-extension.js'; +import { PanelEntity } from '../../entities/visualizations/panel/panel.entity.js'; +import { IPanelRepository } from '../../entities/visualizations/panel/repository/saved-db-query.repository.interface.js'; +import { panelCustomRepositoryExtension } from '../../entities/visualizations/panel/repository/saved-db-query-custom-repository-extension.js'; import { PanelPositionEntity } from '../../entities/visualizations/panel-position/panel-position.entity.js'; import { IPanelPositionRepository } from '../../entities/visualizations/panel-position/repository/panel-position.repository.interface.js'; import { panelPositionCustomRepositoryExtension } from '../../entities/visualizations/panel-position/repository/panel-position-custom-repository-extension.js'; -import { IPanelRepository } from '../../entities/visualizations/panel/repository/saved-db-query.repository.interface.js'; -import { panelCustomRepositoryExtension } from '../../entities/visualizations/panel/repository/saved-db-query-custom-repository-extension.js'; -import { PanelEntity } from '../../entities/visualizations/panel/panel.entity.js'; import { tableWidgetsCustomRepositoryExtension } from '../../entities/widget/repository/table-widgets-custom-repsitory-extension.js'; import { ITableWidgetsRepository } from '../../entities/widget/repository/table-widgets-repository.interface.js'; import { TableWidgetEntity } from '../../entities/widget/table-widget.entity.js'; @@ -157,6 +160,7 @@ export class GlobalDatabaseContext implements IGlobalDatabaseContext { private _panelPositionRepository: Repository & IPanelPositionRepository; private _userAiChatRepository: Repository & IUserAiChatRepository; private _aiChatMessageRepository: Repository & IAiChatMessageRepository; + private _tableSchemaChangeRepository: Repository & ITableSchemaChangeRepository; public constructor( @Inject(BaseType.DATA_SOURCE) @@ -264,6 +268,9 @@ export class GlobalDatabaseContext implements IGlobalDatabaseContext { this._aiChatMessageRepository = this.appDataSource .getRepository(AiChatMessageEntity) .extend(aiChatMessageRepositoryExtension); + this._tableSchemaChangeRepository = this.appDataSource + .getRepository(TableSchemaChangeEntity) + .extend(customTableSchemaChangeRepositoryExtension); } public get userRepository(): Repository & IUserRepository { @@ -428,6 +435,10 @@ export class GlobalDatabaseContext implements IGlobalDatabaseContext { return this._aiChatMessageRepository; } + public get tableSchemaChangeRepository(): Repository & ITableSchemaChangeRepository { + return this._tableSchemaChangeRepository; + } + public startTransaction(): Promise { this._queryRunner = this.appDataSource.createQueryRunner(); this._queryRunner.startTransaction(); diff --git a/backend/src/common/data-injection.tokens.ts b/backend/src/common/data-injection.tokens.ts index 79058a533..26d637a1a 100644 --- a/backend/src/common/data-injection.tokens.ts +++ b/backend/src/common/data-injection.tokens.ts @@ -214,4 +214,15 @@ export enum UseCaseType { IS_CONFIGURED = 'IS_CONFIGURED', CREATE_INITIAL_USER = 'CREATE_INITIAL_USER', + + GENERATE_SCHEMA_CHANGE = 'GENERATE_SCHEMA_CHANGE', + APPROVE_SCHEMA_CHANGE = 'APPROVE_SCHEMA_CHANGE', + REJECT_SCHEMA_CHANGE = 'REJECT_SCHEMA_CHANGE', + ROLLBACK_SCHEMA_CHANGE = 'ROLLBACK_SCHEMA_CHANGE', + LIST_SCHEMA_CHANGES = 'LIST_SCHEMA_CHANGES', + GET_SCHEMA_CHANGE = 'GET_SCHEMA_CHANGE', + APPROVE_BATCH_SCHEMA_CHANGE = 'APPROVE_BATCH_SCHEMA_CHANGE', + REJECT_BATCH_SCHEMA_CHANGE = 'REJECT_BATCH_SCHEMA_CHANGE', + ROLLBACK_BATCH_SCHEMA_CHANGE = 'ROLLBACK_BATCH_SCHEMA_CHANGE', + GET_BATCH_SCHEMA_CHANGE = 'GET_BATCH_SCHEMA_CHANGE', } diff --git a/backend/src/decorators/slug-uuid.decorator.ts b/backend/src/decorators/slug-uuid.decorator.ts index 9ad8450af..baebb2662 100644 --- a/backend/src/decorators/slug-uuid.decorator.ts +++ b/backend/src/decorators/slug-uuid.decorator.ts @@ -15,7 +15,9 @@ export type SlugUuidParameter = | 'companyId' | 'threadId' | 'filterId' - | 'chatId'; + | 'chatId' + | 'changeId' + | 'batchId'; export const SlugUuid = createParamDecorator( (parameterName: SlugUuidParameter = 'slug', ctx: ExecutionContext): string => { const request: IRequestWithCognitoInfo = ctx.switchToHttp().getRequest(); @@ -32,6 +34,8 @@ export const SlugUuid = createParamDecorator( 'threadId', 'filterId', 'chatId', + 'changeId', + 'batchId', ]; if (!availableSlagParameters.includes(parameterName)) { throw new BadRequestException(Messages.UUID_INVALID); diff --git a/backend/src/entities/table-schema/ai/run-schema-change-ai-loop.ts b/backend/src/entities/table-schema/ai/run-schema-change-ai-loop.ts new file mode 100644 index 000000000..bbd62afa4 --- /dev/null +++ b/backend/src/entities/table-schema/ai/run-schema-change-ai-loop.ts @@ -0,0 +1,257 @@ +import { BaseMessage } from '@langchain/core/messages'; +import { Logger } from '@nestjs/common'; +import { IDataAccessObject } from '@rocketadmin/shared-code/dist/src/shared/interfaces/data-access-object.interface.js'; +import { IDataAccessObjectAgent } from '@rocketadmin/shared-code/dist/src/shared/interfaces/data-access-object-agent.interface.js'; +import { + AICoreService, + AIProviderConfig, + AIProviderType, + AIToolCall, + AIToolDefinition, + encodeError, + encodeToToon, + MessageBuilder, +} from '../../../ai-core/index.js'; +import { + isDynamoDbSchemaChangeType, + isElasticsearchSchemaChangeType, + isMongoSchemaChangeType, + SchemaChangeTypeEnum, +} from '../table-schema-change-enums.js'; +import { + GET_TABLE_STRUCTURE_TOOL_NAME, + PROPOSE_DYNAMODB_SCHEMA_CHANGE_TOOL_NAME, + PROPOSE_ELASTICSEARCH_SCHEMA_CHANGE_TOOL_NAME, + PROPOSE_MONGO_SCHEMA_CHANGE_TOOL_NAME, + ProposeSchemaChangeArgs, + TERMINAL_PROPOSAL_TOOL_NAMES, +} from './schema-change-tools.js'; + +export interface RunSchemaChangeAiLoopOptions { + aiCoreService: AICoreService; + provider: AIProviderType; + messages: BaseMessage[]; + tools: AIToolDefinition[]; + dao: IDataAccessObject | IDataAccessObjectAgent; + userEmail: string | undefined; + maxDepth?: number; + logger?: Logger; +} + +export interface SchemaChangeAiLoopResult { + proposals: ProposeSchemaChangeArgs[]; + responseId: string | null; +} + +export async function runSchemaChangeAiLoop(opts: RunSchemaChangeAiLoopOptions): Promise { + const { aiCoreService, provider, dao, userEmail, logger } = opts; + const maxDepth = opts.maxDepth ?? 6; + + let currentMessages = [...opts.messages]; + let currentConfig: AIProviderConfig = {}; + let lastResponseId: string | null = null; + + for (let depth = 0; depth < maxDepth; depth++) { + const stream = await aiCoreService.streamChatWithToolsAndProvider( + provider, + currentMessages, + opts.tools, + currentConfig, + ); + + const pendingToolCalls: AIToolCall[] = []; + let accumulatedContent = ''; + + for await (const chunk of stream) { + if (chunk.type === 'text' && chunk.content) { + accumulatedContent += chunk.content; + } + if (chunk.type === 'tool_call' && chunk.toolCall) { + pendingToolCalls.push(chunk.toolCall); + } + if (chunk.responseId) { + lastResponseId = chunk.responseId; + } + } + + logger?.log( + `AI loop depth=${depth + 1}: toolCalls=[${pendingToolCalls.map((t) => t.name).join(', ')}], textLen=${accumulatedContent.length}`, + ); + + const proposalCall = pendingToolCalls.find((tc) => TERMINAL_PROPOSAL_TOOL_NAMES.has(tc.name)); + if (proposalCall) { + const proposals = coerceAndValidateProposals(proposalCall); + return { proposals, responseId: lastResponseId }; + } + + if (pendingToolCalls.length === 0) { + const hint = accumulatedContent + ? `AI replied with text but no tool call: "${accumulatedContent.slice(0, 200)}"` + : 'AI produced no tool calls and no text.'; + throw new Error( + `${hint} The model must call proposeSchemaChange (SQL), proposeMongoSchemaChange (Mongo), proposeDynamoDbSchemaChange (DynamoDB), or proposeElasticsearchSchemaChange (Elasticsearch) with structured arguments.`, + ); + } + + const toolResults = await executeInspectionToolCalls(pendingToolCalls, dao, userEmail, logger); + + if (provider === AIProviderType.OPENAI && lastResponseId) { + currentConfig = { ...currentConfig, previousResponseId: lastResponseId }; + } + + const continuation = MessageBuilder.fromMessages(currentMessages); + continuation.ai(accumulatedContent, pendingToolCalls); + for (const result of toolResults) { + continuation.toolResult(result.toolCallId, result.result); + } + currentMessages = continuation.build(); + } + + throw new Error(`AI did not produce a proposal within ${maxDepth} iterations.`); +} + +function coerceAndValidateProposals(toolCall: AIToolCall): ProposeSchemaChangeArgs[] { + const args = (toolCall.arguments ?? {}) as Record; + + if (Object.keys(args).length === 0) { + throw new Error( + `AI returned ${toolCall.name} with empty arguments — the underlying tool-call JSON likely failed to parse.`, + ); + } + + const proposalsRaw = args.proposals; + if (!Array.isArray(proposalsRaw) || proposalsRaw.length === 0) { + throw new Error( + `AI returned ${toolCall.name} without a non-empty "proposals" array. Tool calls must wrap one or more changes in proposals.`, + ); + } + + const isMongoTool = toolCall.name === PROPOSE_MONGO_SCHEMA_CHANGE_TOOL_NAME; + const isDynamoDbTool = toolCall.name === PROPOSE_DYNAMODB_SCHEMA_CHANGE_TOOL_NAME; + const isElasticsearchTool = toolCall.name === PROPOSE_ELASTICSEARCH_SCHEMA_CHANGE_TOOL_NAME; + + return proposalsRaw.map((entry, index) => + coerceSingleProposal(entry, toolCall.name, index, isMongoTool, isDynamoDbTool, isElasticsearchTool), + ); +} + +function coerceSingleProposal( + entry: unknown, + toolName: string, + index: number, + isMongoTool: boolean, + isDynamoDbTool: boolean, + isElasticsearchTool: boolean, +): ProposeSchemaChangeArgs { + if (!entry || typeof entry !== 'object') { + throw new Error(`${toolName} proposals[${index}] is not an object.`); + } + const raw = entry as Record; + + const targetTableName = asNonEmptyString(raw.targetTableName, `proposals[${index}].targetTableName`); + const summary = asString(raw.summary) ?? ''; + const reasoning = asString(raw.reasoning) ?? ''; + const changeType = asChangeType(raw.changeType, index); + const isReversible = asBoolean(raw.isReversible, `proposals[${index}].isReversible`); + + if (isMongoTool !== isMongoSchemaChangeType(changeType)) { + throw new Error( + `Tool ${toolName} proposals[${index}] has changeType "${changeType}" which does not match. Mongo tool requires MONGO_* changeType and vice versa.`, + ); + } + if (isDynamoDbTool !== isDynamoDbSchemaChangeType(changeType)) { + throw new Error( + `Tool ${toolName} proposals[${index}] has changeType "${changeType}" which does not match. DynamoDB tool requires DYNAMODB_* changeType and vice versa.`, + ); + } + if (isElasticsearchTool !== isElasticsearchSchemaChangeType(changeType)) { + throw new Error( + `Tool ${toolName} proposals[${index}] has changeType "${changeType}" which does not match. Elasticsearch tool requires ELASTICSEARCH_* changeType and vice versa.`, + ); + } + + if (isMongoTool || isDynamoDbTool || isElasticsearchTool) { + const forwardOp = asNonEmptyString(raw.forwardOp, `proposals[${index}].forwardOp`); + const rollbackOp = asNonEmptyString(raw.rollbackOp, `proposals[${index}].rollbackOp`); + return { + forwardSql: forwardOp, + rollbackSql: rollbackOp, + changeType, + targetTableName, + isReversible, + summary, + reasoning, + }; + } + + const forwardSql = asNonEmptyString(raw.forwardSql, `proposals[${index}].forwardSql`); + const rollbackSql = asNonEmptyString(raw.rollbackSql, `proposals[${index}].rollbackSql`); + return { forwardSql, rollbackSql, changeType, targetTableName, isReversible, summary, reasoning }; +} + +function asString(value: unknown): string | null { + return typeof value === 'string' ? value : null; +} + +function asNonEmptyString(value: unknown, fieldName: string): string { + const str = asString(value); + if (!str || str.trim().length === 0) { + throw new Error(`proposeSchemaChange is missing required string field "${fieldName}".`); + } + return str; +} + +function asBoolean(value: unknown, fieldName: string): boolean { + if (typeof value === 'boolean') return value; + if (typeof value === 'string') { + if (value.toLowerCase() === 'true') return true; + if (value.toLowerCase() === 'false') return false; + } + throw new Error(`proposeSchemaChange field "${fieldName}" must be a boolean; received ${JSON.stringify(value)}.`); +} + +function asChangeType(value: unknown, index: number): SchemaChangeTypeEnum { + if (typeof value !== 'string') { + throw new Error( + `proposeSchemaChange proposals[${index}].changeType must be a string; received ${JSON.stringify(value)}.`, + ); + } + if ((Object.values(SchemaChangeTypeEnum) as string[]).includes(value)) { + return value as SchemaChangeTypeEnum; + } + throw new Error( + `proposeSchemaChange proposals[${index}].changeType "${value}" is not one of ${Object.values(SchemaChangeTypeEnum).join(', ')}.`, + ); +} + +async function executeInspectionToolCalls( + toolCalls: AIToolCall[], + dao: IDataAccessObject | IDataAccessObjectAgent, + userEmail: string | undefined, + logger?: Logger, +): Promise> { + const results: Array<{ toolCallId: string; result: string }> = []; + for (const tc of toolCalls) { + let result: string; + try { + if (tc.name === GET_TABLE_STRUCTURE_TOOL_NAME) { + const tableName = tc.arguments.tableName as string; + if (!tableName) throw new Error('Missing argument "tableName"'); + const structure = await dao.getTableStructure(tableName, userEmail); + const foreignKeys = await dao.getTableForeignKeys(tableName, userEmail); + result = encodeToToon({ tableName, structure, foreignKeys }); + } else if (TERMINAL_PROPOSAL_TOOL_NAMES.has(tc.name)) { + result = encodeError({ + error: `${tc.name} is a terminal tool; it should be the only tool call in the final round.`, + }); + } else { + result = encodeError({ error: `Unknown tool: ${tc.name}` }); + } + } catch (err) { + logger?.error(`Tool call ${tc.name} failed: ${(err as Error).message}`); + result = encodeError({ error: (err as Error).message }); + } + results.push({ toolCallId: tc.id, result }); + } + return results; +} diff --git a/backend/src/entities/table-schema/ai/schema-change-prompts.ts b/backend/src/entities/table-schema/ai/schema-change-prompts.ts new file mode 100644 index 000000000..8d4866d69 --- /dev/null +++ b/backend/src/entities/table-schema/ai/schema-change-prompts.ts @@ -0,0 +1,268 @@ +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; +import { convertDbTypeToReadableString } from '../../../ai-core/tools/prompts.js'; +import { isDynamoDbDialect, isElasticsearchDialect, isMongoDialect } from '../utils/assert-dialect-supported.js'; + +export function buildSchemaChangePrompt( + connectionType: ConnectionTypesEnum, + existingTables: string[], + schema: string | null, +): string { + if (isMongoDialect(connectionType)) { + return buildMongoSchemaChangePrompt(existingTables); + } + if (isDynamoDbDialect(connectionType)) { + return buildDynamoDbSchemaChangePrompt(existingTables); + } + if (isElasticsearchDialect(connectionType)) { + return buildElasticsearchSchemaChangePrompt(existingTables); + } + return buildSqlSchemaChangePrompt(connectionType, existingTables, schema); +} + +function buildSqlSchemaChangePrompt( + connectionType: ConnectionTypesEnum, + existingTables: string[], + schema: string | null, +): string { + const dialect = convertDbTypeToReadableString(connectionType); + const tableList = existingTables.length > 0 ? existingTables.join(', ') : '(none)'; + const schemaLine = schema ? `Schema: "${schema}".` : ''; + + return `You are a DDL generator for ${dialect}. +${schemaLine} +Existing tables in this database: ${tableList}. + +Your job: translate the user's natural-language request into ONE OR MORE DDL statements for ${dialect}, each paired with a rollback statement. A single user request may legitimately require multiple changes (e.g. "create products, users and orders tables", "add columns a, b, c", "create users plus an index on email"). Emit one proposal per logical change. + +Workflow: +1. If the user's request references an existing table, call getTableStructure FIRST to inspect it. You may call it multiple times for different tables. +2. Once you have enough context, call proposeSchemaChange EXACTLY ONCE with a "proposals" array containing every change required by the user's request. Do not write free-text explanations outside the tool call. + +Multi-proposal rules: +- Order proposals so that any change depending on another comes AFTER its dependency. Tables referenced by foreign keys must be created BEFORE the tables that reference them. +- Each proposal element must independently satisfy the single-DDL-statement and dialect rules below; the array carries the ordered set, not a multi-statement script. +- Do NOT bundle unrelated changes the user did not ask for. Only emit proposals that fulfil the user's request. +- For a single-change request, supply a "proposals" array of length 1 — same content as before. + +Rules for the generated SQL: +- Target dialect is ${dialect}. Use the correct identifier quoting (double quotes for PostgreSQL/Oracle/DB2, backticks for MySQL/ClickHouse, square brackets or double quotes for Microsoft SQL Server) and the correct syntax for data types, autoincrement, and constraints. +- For Microsoft SQL Server, use IDENTITY for autoincrement, NVARCHAR/VARCHAR for strings, and name primary/foreign keys explicitly (e.g. CONSTRAINT PK_tbl PRIMARY KEY ...) so they are referenceable in rollback DROP CONSTRAINT statements. +- For Oracle DB, use NUMBER/VARCHAR2, and GENERATED BY DEFAULT AS IDENTITY for autoincrement. ALTER TABLE ... MODIFY is the column-change syntax. +- For IBM DB2, use BIGINT GENERATED BY DEFAULT AS IDENTITY and VARCHAR. ALTER TABLE ... ALTER COLUMN ... SET DATA TYPE is the column-change syntax. +- For ClickHouse: every CREATE TABLE MUST include a table engine (prefer \`ENGINE = MergeTree()\`) followed by an \`ORDER BY ()\` clause — ClickHouse has no conventional PRIMARY KEY; the sort/primary key is the \`ORDER BY\` tuple. Use types like \`UInt32\`, \`UInt64\`, \`Int64\`, \`String\`, \`Float64\`, \`DateTime\`, \`Date\`, \`UUID\`, \`Nullable(T)\` (wrap a type to allow NULL). There is no autoincrement — use a plain numeric type the user populates. Column additions use \`ALTER TABLE t ADD COLUMN c T\`, drops use \`ALTER TABLE t DROP COLUMN c\`, type changes use \`ALTER TABLE t MODIFY COLUMN c T\`. Do NOT emit \`ON CLUSTER\` clauses; the target is a single-node server. There are NO true transactions, so rollback is a best-effort compensating DDL (e.g. add-column forward / drop-column rollback). Do NOT propose foreign keys (ClickHouse does not enforce them). Indexes in ClickHouse are DATA SKIPPING indexes created with \`ALTER TABLE t ADD INDEX idx_name col TYPE minmax GRANULARITY 4\`; rollback is \`ALTER TABLE t DROP INDEX idx_name\`. Avoid \`DROP TABLE IF EXISTS\` unless the user asked. +- For Cassandra (CQL): every CREATE TABLE MUST declare a \`PRIMARY KEY\` inline, either as a column-level \`PRIMARY KEY\` on one column or as a trailing \`PRIMARY KEY ((partition_key_cols), clustering_key_cols)\` clause. Use CQL types: \`UUID\`, \`TIMEUUID\`, \`TEXT\`, \`VARCHAR\`, \`ASCII\`, \`INT\`, \`BIGINT\`, \`SMALLINT\`, \`TINYINT\`, \`FLOAT\`, \`DOUBLE\`, \`DECIMAL\`, \`BOOLEAN\`, \`TIMESTAMP\`, \`DATE\`, \`TIME\`, \`BLOB\`, \`INET\`, \`LIST\`, \`SET\`, \`MAP\`. There is NO autoincrement — prefer \`UUID\` partition keys. Do NOT propose foreign keys (Cassandra does not enforce them). Do NOT propose \`ALTER COLUMN\` type-change DDL — CQL only supports renaming primary-key columns and adding/dropping non-primary-key columns. Column additions use \`ALTER TABLE t ADD c T\` (no \`COLUMN\` keyword), drops use \`ALTER TABLE t DROP c\`. Indexes are \`CREATE INDEX idx_name ON t (col)\` with rollback \`DROP INDEX idx_name\`. There are NO transactions; rollback is a best-effort compensating DDL. Do NOT emit \`CREATE KEYSPACE\`, \`DROP KEYSPACE\`, \`USE\`, or materialized-view DDL. +- Both forwardSql and rollbackSql MUST be single DDL statements. No semicolons terminating a chain. No multi-statement scripts. +- Never emit DROP DATABASE, DROP SCHEMA, GRANT, REVOKE, TRUNCATE, DELETE, UPDATE, INSERT, or any DML. The proposer is for schema changes only. +- For CREATE TABLE X, the rollback is DROP TABLE X. +- For ALTER TABLE T ADD COLUMN C, the rollback is ALTER TABLE T DROP COLUMN C. +- For ALTER TABLE T DROP COLUMN C, isReversible=false (data is lost) and rollback is a best-effort ALTER TABLE T ADD COLUMN C with the prior type if you inspected the structure. +- For ALTER COLUMN type changes: only propose compatible widening transitions (e.g. VARCHAR(N) -> VARCHAR(M) with M>N, VARCHAR -> TEXT, INT -> BIGINT). Refuse incompatible narrowing (e.g. TEXT -> INTEGER) or precision-losing transitions by returning a proposeSchemaChange with empty forwardSql and a clear reasoning explaining the block. When the user explicitly accepts the risk of precision loss, set isReversible=false and produce a best-effort rollback. +- Foreign keys must reference existing tables. If they don't, refuse with a clear reasoning in the proposeSchemaChange arguments and still emit your best attempt; the server will validate. +- When adding a named index, rollback is DROP INDEX with the same name. When adding a named foreign key, rollback is ALTER TABLE ... DROP CONSTRAINT / DROP FOREIGN KEY by that name. +- Do not use IF EXISTS or IF NOT EXISTS unless the user explicitly asked. +- Do not create indexes implicitly on foreign keys unless the user asked. + +Classification (changeType): +- CREATE_TABLE, DROP_TABLE +- ADD_COLUMN, DROP_COLUMN, ALTER_COLUMN +- ADD_INDEX, DROP_INDEX +- ADD_FOREIGN_KEY, DROP_FOREIGN_KEY +- ADD_PRIMARY_KEY, DROP_PRIMARY_KEY +- OTHER (last resort) + +Set targetTableName to the unqualified table name the change acts on. +Set isReversible=true only if executing rollbackSql would fully restore the prior state.`; +} + +function buildDynamoDbSchemaChangePrompt(existingTables: string[]): string { + const tableList = existingTables.length > 0 ? existingTables.join(', ') : '(none)'; + + return `You are a DynamoDB schema-change generator. +Existing tables in this database: ${tableList}. + +DynamoDB is a NoSQL key-value store with a fixed primary key schema per table and no conventional DDL. Schema changes are expressed as structured JSON operations the server will apply via the AWS SDK (CreateTable, DeleteTable, UpdateTable, UpdateTimeToLive). A single user request may legitimately require multiple changes (e.g. "create products, users and orders tables"). + +Workflow: +1. If the user's request references an existing table, call getTableStructure FIRST to see its key schema and sampled attributes. You may call it multiple times. +2. Once you have enough context, call proposeDynamoDbSchemaChange EXACTLY ONCE with a "proposals" array containing every change required by the user's request. Do not write free-text explanations outside the tool call. + +Multi-proposal rules: +- Order proposals so dependent changes come after their prerequisites. +- Each element must independently satisfy the per-operation rules below. +- Do NOT bundle unrelated changes the user did not ask for. +- For a single-change request, supply a "proposals" array of length 1. + +forwardOp and rollbackOp are JSON strings. Each must decode to exactly one object of one of these shapes: + + { + "operation": "createTable", + "tableName": "", + "attributeDefinitions": [{ "attributeName": "id", "attributeType": "S" | "N" | "B" }, ...], // required, must cover every key/GSI/LSI key attribute + "keySchema": [ + { "attributeName": "id", "keyType": "HASH" }, + { "attributeName": "sort", "keyType": "RANGE" } // optional RANGE key + ], + "billingMode": "PAY_PER_REQUEST" | "PROVISIONED", // default PAY_PER_REQUEST + "provisionedThroughput": { "readCapacityUnits": 5, "writeCapacityUnits": 5 }, // required when billingMode=PROVISIONED + "globalSecondaryIndexes": [{ "indexName": "...", "keySchema": [...], "projection": { "projectionType": "ALL" | "KEYS_ONLY" | "INCLUDE", "nonKeyAttributes": ["a", "b"] }, "provisionedThroughput"?: {...} }], + "localSecondaryIndexes": [{ "indexName": "...", "keySchema": [...], "projection": {...} }], + "streamSpecification": { "streamEnabled": true, "streamViewType": "NEW_AND_OLD_IMAGES" } // optional + } + + { + "operation": "deleteTable", + "tableName": "" + } + + { + "operation": "updateTable", + "tableName": "", + "attributeDefinitions"?: [...], // required when adding a GSI with new key attributes + "billingMode"?: "PAY_PER_REQUEST" | "PROVISIONED", + "provisionedThroughput"?: { "readCapacityUnits": N, "writeCapacityUnits": N }, + "globalSecondaryIndexUpdates"?: [ + { "create": { "indexName": "...", "keySchema": [...], "projection": {...}, "provisionedThroughput"?: {...} } }, + { "update": { "indexName": "...", "provisionedThroughput": {...} } }, + { "delete": { "indexName": "..." } } + ], + "streamSpecification"?: { "streamEnabled": true, "streamViewType": "NEW_IMAGE" } + } + + { + "operation": "updateTimeToLive", + "tableName": "", + "timeToLiveSpecification": { "enabled": true, "attributeName": "expiresAt" } + } + +Rules: +- "tableName" must be the unqualified name and must match the top-level targetTableName. +- attributeDefinitions in createTable MUST include every attribute referenced by keySchema, GSI keySchema, and LSI keySchema. +- scalar attributeType is "S" (string), "N" (number), or "B" (binary). +- Prefer billingMode="PAY_PER_REQUEST" unless the user asks for provisioned capacity. +- For createTable, rollback is deleteTable on the same tableName. An initial create-then-rollback pair is reversible (set isReversible=true). +- For deleteTable, isReversible=false (items are lost). Rollback is a best-effort createTable with the prior key schema inferred from getTableStructure if possible; if not, still emit a best-effort createTable and set isReversible=false. +- For updateTable adding a GSI, rollback is updateTable with a matching delete GSI action on the same indexName (isReversible=true). For updateTable deleting a GSI, rollback is the create action for that GSI with the prior key schema/projection if known (else isReversible=false). +- For updateTable changing billingMode or provisionedThroughput, rollback is the inverse updateTable with the prior values (isReversible=true if the prior values are known from getTableStructure). +- For updateTimeToLive enabling TTL on attribute X, rollback is updateTimeToLive with enabled=false on the same attribute X (isReversible=true). AWS requires specifying attributeName on disable as well. +- Never propose dropping an entire region/replica, IAM actions, or tag operations — those are out of scope for this tool. + +Classification (changeType): +- DYNAMODB_CREATE_TABLE +- DYNAMODB_DROP_TABLE +- DYNAMODB_UPDATE_TABLE +- DYNAMODB_UPDATE_TTL +- OTHER (last resort) + +Set targetTableName to the table name the change acts on. +Set isReversible=true only if executing rollbackOp would fully restore the prior state.`; +} + +function buildElasticsearchSchemaChangePrompt(existingIndices: string[]): string { + const indexList = existingIndices.length > 0 ? existingIndices.join(', ') : '(none)'; + + return `You are an Elasticsearch schema-change generator. +Existing indices in this cluster: ${indexList}. + +Elasticsearch is a search engine where each "table" is an index with a mapping (the JSON schema of fields). Schema changes are expressed as structured JSON operations the server will apply via the official @elastic/elasticsearch client (indices.create, indices.delete, indices.putMapping). A single user request may legitimately require multiple changes (e.g. "create indices for products, users and orders"). + +Workflow: +1. If the user's request references an existing index, call getTableStructure FIRST to inspect its sampled fields and inferred types. You may call it multiple times. +2. Once you have enough context, call proposeElasticsearchSchemaChange EXACTLY ONCE with a "proposals" array containing every change required by the user's request. Do not write free-text explanations outside the tool call. + +Multi-proposal rules: +- Order proposals so dependent changes come after their prerequisites. +- Each element must independently satisfy the per-operation rules below. +- Do NOT bundle unrelated changes the user did not ask for. +- For a single-change request, supply a "proposals" array of length 1. + +forwardOp and rollbackOp are JSON strings. Each must decode to exactly one object of one of these shapes: + + { + "operation": "createIndex", + "indexName": "", + "mappings": { "properties": { "fieldName": { "type": "keyword" }, ... } }, // optional + "settings": { "number_of_shards": 1, "number_of_replicas": 0 } // optional + } + + { + "operation": "deleteIndex", + "indexName": "" + } + + { + "operation": "updateMapping", + "indexName": "", + "properties": { "newFieldName": { "type": "keyword" }, ... } // required, only ADDING fields is allowed + } + +Rules: +- "indexName" must be the unqualified name and must match the top-level targetTableName. +- Index names that start with "." or "_" are reserved system indices and MUST NOT be touched. +- Allowed Elasticsearch field types include: "keyword", "text", "long", "integer", "short", "byte", "double", "float", "date", "boolean", "ip", "object", "nested", "geo_point", "binary". +- For createIndex, rollback is deleteIndex on the same indexName. For an initial create-then-rollback pair, isReversible=true. If user data is then indexed and the rollback would lose it, that is the user's risk; the rollback semantics here only describe the schema-level inverse. +- For deleteIndex, isReversible=false (documents are lost). Rollback is a best-effort createIndex with the prior mapping inferred from getTableStructure if possible; if not, still emit a best-effort createIndex with no mapping. +- For updateMapping, ONLY ADDING NEW FIELDS is allowed — Elasticsearch does not let you change the type of an existing field without reindexing. If the user requests a remap of an existing field, set isReversible=false and explain in reasoning that they need a reindex; you may still emit a best-effort updateMapping that adds new fields. Rollback for updateMapping has no first-class inverse (Elasticsearch cannot drop a single field from a mapping); emit a deleteIndex on the index as the rollback ONLY when the user explicitly accepts losing the entire index, otherwise echo a no-op updateMapping with the same properties (effectively idempotent) and set isReversible=false. +- Never propose dropping/renaming the entire cluster, modifying templates, ILM policies, security/IAM, or alias management — those are out of scope for this tool. + +Classification (changeType): +- ELASTICSEARCH_CREATE_INDEX +- ELASTICSEARCH_DELETE_INDEX +- ELASTICSEARCH_UPDATE_MAPPING +- OTHER (last resort) + +Set targetTableName to the index name the change acts on. +Set isReversible=true only if executing rollbackOp would fully restore the prior state.`; +} + +function buildMongoSchemaChangePrompt(existingCollections: string[]): string { + const collectionList = existingCollections.length > 0 ? existingCollections.join(', ') : '(none)'; + + return `You are a MongoDB schema-change generator. +Existing collections in this database: ${collectionList}. + +MongoDB does not use SQL DDL. Instead, schema changes are structured JSON operations that the server will execute via the official MongoDB driver. A single user request may legitimately require multiple changes (e.g. "create products, users and orders collections", or "add a unique email index plus a products collection"). + +Workflow: +1. If the user's request references an existing collection, call getTableStructure FIRST to inspect a sample of documents. You may call it multiple times for different collections. +2. Once you have enough context, call proposeMongoSchemaChange EXACTLY ONCE with a "proposals" array containing every change required by the user's request. Do not write free-text explanations outside the tool call. + +Multi-proposal rules: +- Order proposals so dependent changes come after their prerequisites (e.g. createCollection before createIndex on that collection). +- Each element must independently satisfy the per-operation rules below. +- Do NOT bundle unrelated changes the user did not ask for. +- For a single-change request, supply a "proposals" array of length 1. + +forwardOp and rollbackOp are JSON strings. Each must decode to exactly one object of the form: + + { + "operation": "createCollection" | "dropCollection" | "setValidator" | "createIndex" | "dropIndex", + "collectionName": "", + "validatorSchema": , // required for setValidator (use null to clear) + "validationLevel": "off" | "strict" | "moderate", // optional for setValidator, default "strict" + "validationAction": "warn" | "error", // optional for setValidator, default "error" + "indexName": "", // required for dropIndex; required and unique for createIndex + "indexSpec": { "field1": 1, "field2": -1 }, // required for createIndex; uses 1 / -1 / "text" / "2dsphere" + "indexOptions": { "unique": true, "sparse": false } // optional for createIndex + } + +Rules: +- "collectionName" must be the unqualified name. +- Each JSON string must be a single object (not an array, not multiple operations). +- Never propose drop/rename of the entire database, user management, or $where / $function / $accumulator inside a validator. +- For createCollection, rollback is dropCollection on the same collectionName and is NOT reversible (documents inserted afterward would be lost if we recreated it — but an empty collection drop is safe; set isReversible=true for the initial create-then-drop pair). +- For dropCollection, isReversible=false (data is lost). Rollback is a best-effort createCollection with no validator. +- For setValidator, rollback is another setValidator carrying the prior validator (or null to clear). Call getTableStructure to infer the previous shape if you need to. +- For createIndex, rollback is dropIndex with the same indexName. Always provide an explicit indexName in indexOptions.name AND at the top level so rollback is unambiguous. +- For dropIndex, rollback is createIndex using the dropped index's spec/options. If you don't know the original spec, set isReversible=false. + +Classification (changeType): +- MONGO_CREATE_COLLECTION +- MONGO_DROP_COLLECTION +- MONGO_SET_VALIDATOR +- MONGO_CREATE_INDEX +- MONGO_DROP_INDEX +- OTHER (last resort) + +Set targetTableName to the collection name the change acts on. +Set isReversible=true only if executing rollbackOp would fully restore the prior state.`; +} diff --git a/backend/src/entities/table-schema/ai/schema-change-tools.ts b/backend/src/entities/table-schema/ai/schema-change-tools.ts new file mode 100644 index 000000000..0346f2c19 --- /dev/null +++ b/backend/src/entities/table-schema/ai/schema-change-tools.ts @@ -0,0 +1,322 @@ +import { AIToolDefinition } from '../../../ai-core/index.js'; +import { SchemaChangeTypeEnum } from '../table-schema-change-enums.js'; + +export const PROPOSE_SCHEMA_CHANGE_TOOL_NAME = 'proposeSchemaChange'; +export const PROPOSE_MONGO_SCHEMA_CHANGE_TOOL_NAME = 'proposeMongoSchemaChange'; +export const PROPOSE_DYNAMODB_SCHEMA_CHANGE_TOOL_NAME = 'proposeDynamoDbSchemaChange'; +export const PROPOSE_ELASTICSEARCH_SCHEMA_CHANGE_TOOL_NAME = 'proposeElasticsearchSchemaChange'; +export const GET_TABLE_STRUCTURE_TOOL_NAME = 'getTableStructure'; + +export const TERMINAL_PROPOSAL_TOOL_NAMES: ReadonlySet = new Set([ + PROPOSE_SCHEMA_CHANGE_TOOL_NAME, + PROPOSE_MONGO_SCHEMA_CHANGE_TOOL_NAME, + PROPOSE_DYNAMODB_SCHEMA_CHANGE_TOOL_NAME, + PROPOSE_ELASTICSEARCH_SCHEMA_CHANGE_TOOL_NAME, +]); + +export interface ProposeSchemaChangeArgs { + forwardSql: string; + rollbackSql: string; + changeType: SchemaChangeTypeEnum; + targetTableName: string; + isReversible: boolean; + summary: string; + reasoning: string; +} + +export function createSchemaChangeTools(): AIToolDefinition[] { + return [createGetTableStructureTool(), createProposeSqlSchemaChangeTool()]; +} + +export function createMongoSchemaChangeTools(): AIToolDefinition[] { + return [createGetTableStructureTool(), createProposeMongoSchemaChangeTool()]; +} + +export function createDynamoDbSchemaChangeTools(): AIToolDefinition[] { + return [createGetTableStructureTool(), createProposeDynamoDbSchemaChangeTool()]; +} + +export function createElasticsearchSchemaChangeTools(): AIToolDefinition[] { + return [createGetTableStructureTool(), createProposeElasticsearchSchemaChangeTool()]; +} + +function createGetTableStructureTool(): AIToolDefinition { + return { + name: GET_TABLE_STRUCTURE_TOOL_NAME, + description: + 'Inspect an existing table or MongoDB collection. For SQL databases returns column names, data types, nullability, primary keys, and foreign keys. For MongoDB returns inferred field types from sampled documents. Call this BEFORE proposing changes that reference an existing table/collection.', + parameters: { + type: 'object', + properties: { + tableName: { + type: 'string', + description: 'The name of the table or collection to inspect.', + }, + }, + required: ['tableName'], + additionalProperties: false, + }, + }; +} + +function createProposeSqlSchemaChangeTool(): AIToolDefinition { + return { + name: PROPOSE_SCHEMA_CHANGE_TOOL_NAME, + description: + 'Emit the final proposed DDL. Call this exactly ONCE. Provide a "proposals" array containing one or more change proposals. For a single-change request, supply one element. For requests that imply multiple related changes (e.g. "create products, users and orders tables", or creating multiple columns at once), supply MULTIPLE elements in dependency order — parent tables must come BEFORE tables that reference them via foreign keys. Each element must be a single DDL statement for the declared dialect.', + parameters: { + type: 'object', + properties: { + proposals: { + type: 'array', + minItems: 1, + description: + 'Ordered list of schema changes to apply. For a single change, length 1. For multi-table or multi-column requests, list every change in dependency order.', + items: { + type: 'object', + properties: { + forwardSql: { + type: 'string', + description: + 'Single DDL statement to apply (e.g. CREATE TABLE "x" ...). No trailing semicolon required. Must be one statement.', + }, + rollbackSql: { + type: 'string', + description: + 'Single DDL statement that undoes forwardSql (e.g. DROP TABLE "x" for a CREATE TABLE). Must be one statement. If truly non-reversible, still emit a best-effort compensating statement and set isReversible=false.', + }, + changeType: { + type: 'string', + enum: Object.values(SchemaChangeTypeEnum).filter( + (v) => !v.startsWith('MONGO_') && !v.startsWith('DYNAMODB_') && !v.startsWith('ELASTICSEARCH_'), + ), + description: 'Classification of the change. Match the AST of forwardSql.', + }, + targetTableName: { + type: 'string', + description: 'The unqualified name of the table being created, altered, or dropped.', + }, + isReversible: { + type: 'boolean', + description: + 'False when rolling back would lose data or cannot exactly restore state (e.g. DROP TABLE on populated table, DROP COLUMN). True otherwise.', + }, + summary: { + type: 'string', + description: 'One-sentence human-readable description of the change.', + }, + reasoning: { + type: 'string', + description: 'Brief explanation: what it does, why this rollback, any caveats.', + }, + }, + required: [ + 'forwardSql', + 'rollbackSql', + 'changeType', + 'targetTableName', + 'isReversible', + 'summary', + 'reasoning', + ], + additionalProperties: false, + }, + }, + }, + required: ['proposals'], + additionalProperties: false, + }, + }; +} + +function createProposeDynamoDbSchemaChangeTool(): AIToolDefinition { + return { + name: PROPOSE_DYNAMODB_SCHEMA_CHANGE_TOOL_NAME, + description: + 'Emit the final DynamoDB schema changes. Call this exactly ONCE. Provide a "proposals" array containing one or more changes. Each element supplies forwardOp and rollbackOp as JSON strings; each JSON string MUST decode to a single object describing one structured operation (createTable / deleteTable / updateTable / updateTimeToLive). For multi-table requests, include multiple proposals in dependency order.', + parameters: { + type: 'object', + properties: { + proposals: { + type: 'array', + minItems: 1, + description: 'Ordered list of DynamoDB changes to apply.', + items: { + type: 'object', + properties: { + forwardOp: { + type: 'string', + description: + 'JSON string describing the forward operation. Single object with "operation" and "tableName" fields plus op-specific fields (keySchema, attributeDefinitions, billingMode, provisionedThroughput, globalSecondaryIndexes, globalSecondaryIndexUpdates, timeToLiveSpecification, etc.).', + }, + rollbackOp: { + type: 'string', + description: + 'JSON string describing the compensating operation. Must be a single object in the same form as forwardOp.', + }, + changeType: { + type: 'string', + enum: Object.values(SchemaChangeTypeEnum).filter((v) => v.startsWith('DYNAMODB_')), + description: 'Classification of the DynamoDB change.', + }, + targetTableName: { + type: 'string', + description: 'The DynamoDB table name the change acts on.', + }, + isReversible: { + type: 'boolean', + description: + 'False when rolling back cannot exactly restore state (e.g. deleteTable on a populated table). True otherwise.', + }, + summary: { type: 'string', description: 'One-sentence human-readable description of the change.' }, + reasoning: { + type: 'string', + description: 'Brief explanation: what it does, why this rollback, any caveats.', + }, + }, + required: [ + 'forwardOp', + 'rollbackOp', + 'changeType', + 'targetTableName', + 'isReversible', + 'summary', + 'reasoning', + ], + additionalProperties: false, + }, + }, + }, + required: ['proposals'], + additionalProperties: false, + }, + }; +} + +function createProposeElasticsearchSchemaChangeTool(): AIToolDefinition { + return { + name: PROPOSE_ELASTICSEARCH_SCHEMA_CHANGE_TOOL_NAME, + description: + 'Emit the final Elasticsearch schema changes. Call this exactly ONCE. Provide a "proposals" array. Each element supplies forwardOp and rollbackOp as JSON strings; each JSON string MUST decode to a single object describing one structured operation (createIndex / deleteIndex / updateMapping). For multi-index requests, include multiple proposals in dependency order.', + parameters: { + type: 'object', + properties: { + proposals: { + type: 'array', + minItems: 1, + description: 'Ordered list of Elasticsearch changes to apply.', + items: { + type: 'object', + properties: { + forwardOp: { + type: 'string', + description: + 'JSON string describing the forward operation. Single object with "operation" and "indexName" fields plus op-specific fields (mappings, settings, properties).', + }, + rollbackOp: { + type: 'string', + description: + 'JSON string describing the compensating operation. Must be a single object in the same form as forwardOp.', + }, + changeType: { + type: 'string', + enum: Object.values(SchemaChangeTypeEnum).filter((v) => v.startsWith('ELASTICSEARCH_')), + description: 'Classification of the Elasticsearch change.', + }, + targetTableName: { + type: 'string', + description: 'The Elasticsearch index name the change acts on.', + }, + isReversible: { + type: 'boolean', + description: + 'False when rolling back cannot exactly restore state (e.g. deleteIndex on a populated index). True otherwise.', + }, + summary: { type: 'string', description: 'One-sentence human-readable description of the change.' }, + reasoning: { + type: 'string', + description: 'Brief explanation: what it does, why this rollback, any caveats.', + }, + }, + required: [ + 'forwardOp', + 'rollbackOp', + 'changeType', + 'targetTableName', + 'isReversible', + 'summary', + 'reasoning', + ], + additionalProperties: false, + }, + }, + }, + required: ['proposals'], + additionalProperties: false, + }, + }; +} + +function createProposeMongoSchemaChangeTool(): AIToolDefinition { + return { + name: PROPOSE_MONGO_SCHEMA_CHANGE_TOOL_NAME, + description: + 'Emit the final MongoDB schema changes. Call this exactly ONCE. Provide a "proposals" array. Each element supplies forwardOp and rollbackOp as JSON strings; each JSON string MUST decode to a single object describing one structured operation (createCollection / dropCollection / setValidator / createIndex / dropIndex). For multi-collection requests, include multiple proposals in dependency order.', + parameters: { + type: 'object', + properties: { + proposals: { + type: 'array', + minItems: 1, + description: 'Ordered list of MongoDB changes to apply.', + items: { + type: 'object', + properties: { + forwardOp: { + type: 'string', + description: + 'JSON string describing the forward operation. Single object with "operation" and "collectionName" fields plus op-specific fields (indexSpec, indexName, validatorSchema, etc.).', + }, + rollbackOp: { + type: 'string', + description: + 'JSON string describing the compensating operation. Must be a single object in the same form as forwardOp.', + }, + changeType: { + type: 'string', + enum: Object.values(SchemaChangeTypeEnum).filter((v) => v.startsWith('MONGO_')), + description: 'Classification of the Mongo change.', + }, + targetTableName: { + type: 'string', + description: 'The unqualified collection name the change acts on.', + }, + isReversible: { + type: 'boolean', + description: + 'False when rolling back cannot exactly restore state (e.g. dropCollection on a populated collection). True otherwise.', + }, + summary: { type: 'string', description: 'One-sentence human-readable description of the change.' }, + reasoning: { + type: 'string', + description: 'Brief explanation: what it does, why this rollback, any caveats.', + }, + }, + required: [ + 'forwardOp', + 'rollbackOp', + 'changeType', + 'targetTableName', + 'isReversible', + 'summary', + 'reasoning', + ], + additionalProperties: false, + }, + }, + }, + required: ['proposals'], + additionalProperties: false, + }, + }; +} diff --git a/backend/src/entities/table-schema/application/data-structures/approve-batch-schema-change.ds.ts b/backend/src/entities/table-schema/application/data-structures/approve-batch-schema-change.ds.ts new file mode 100644 index 000000000..26254db3b --- /dev/null +++ b/backend/src/entities/table-schema/application/data-structures/approve-batch-schema-change.ds.ts @@ -0,0 +1,6 @@ +export class ApproveBatchSchemaChangeDs { + batchId: string; + userId: string; + masterPassword?: string; + confirmedDestructive?: boolean; +} diff --git a/backend/src/entities/table-schema/application/data-structures/approve-schema-change.ds.ts b/backend/src/entities/table-schema/application/data-structures/approve-schema-change.ds.ts new file mode 100644 index 000000000..d5e930504 --- /dev/null +++ b/backend/src/entities/table-schema/application/data-structures/approve-schema-change.ds.ts @@ -0,0 +1,7 @@ +export class ApproveSchemaChangeDs { + changeId: string; + userId: string; + masterPassword?: string; + userModifiedSql?: string; + confirmedDestructive?: boolean; +} diff --git a/backend/src/entities/table-schema/application/data-structures/generate-schema-change.ds.ts b/backend/src/entities/table-schema/application/data-structures/generate-schema-change.ds.ts new file mode 100644 index 000000000..a73504841 --- /dev/null +++ b/backend/src/entities/table-schema/application/data-structures/generate-schema-change.ds.ts @@ -0,0 +1,6 @@ +export class GenerateSchemaChangeDs { + connectionId: string; + userPrompt: string; + userId: string; + masterPassword?: string; +} diff --git a/backend/src/entities/table-schema/application/data-structures/get-batch-schema-change.ds.ts b/backend/src/entities/table-schema/application/data-structures/get-batch-schema-change.ds.ts new file mode 100644 index 000000000..4717fa8d2 --- /dev/null +++ b/backend/src/entities/table-schema/application/data-structures/get-batch-schema-change.ds.ts @@ -0,0 +1,4 @@ +export class GetBatchSchemaChangeDs { + batchId: string; + userId: string; +} diff --git a/backend/src/entities/table-schema/application/data-structures/get-schema-change.ds.ts b/backend/src/entities/table-schema/application/data-structures/get-schema-change.ds.ts new file mode 100644 index 000000000..128cbb0d9 --- /dev/null +++ b/backend/src/entities/table-schema/application/data-structures/get-schema-change.ds.ts @@ -0,0 +1,4 @@ +export class GetSchemaChangeDs { + changeId: string; + userId: string; +} diff --git a/backend/src/entities/table-schema/application/data-structures/list-schema-changes.ds.ts b/backend/src/entities/table-schema/application/data-structures/list-schema-changes.ds.ts new file mode 100644 index 000000000..21972a8e4 --- /dev/null +++ b/backend/src/entities/table-schema/application/data-structures/list-schema-changes.ds.ts @@ -0,0 +1,6 @@ +export class ListSchemaChangesDs { + connectionId: string; + userId: string; + limit: number; + offset: number; +} diff --git a/backend/src/entities/table-schema/application/data-structures/reject-batch-schema-change.ds.ts b/backend/src/entities/table-schema/application/data-structures/reject-batch-schema-change.ds.ts new file mode 100644 index 000000000..cabbb1735 --- /dev/null +++ b/backend/src/entities/table-schema/application/data-structures/reject-batch-schema-change.ds.ts @@ -0,0 +1,4 @@ +export class RejectBatchSchemaChangeDs { + batchId: string; + userId: string; +} diff --git a/backend/src/entities/table-schema/application/data-structures/reject-schema-change.ds.ts b/backend/src/entities/table-schema/application/data-structures/reject-schema-change.ds.ts new file mode 100644 index 000000000..ec8322739 --- /dev/null +++ b/backend/src/entities/table-schema/application/data-structures/reject-schema-change.ds.ts @@ -0,0 +1,4 @@ +export class RejectSchemaChangeDs { + changeId: string; + userId: string; +} diff --git a/backend/src/entities/table-schema/application/data-structures/rollback-batch-schema-change.ds.ts b/backend/src/entities/table-schema/application/data-structures/rollback-batch-schema-change.ds.ts new file mode 100644 index 000000000..4d04616b1 --- /dev/null +++ b/backend/src/entities/table-schema/application/data-structures/rollback-batch-schema-change.ds.ts @@ -0,0 +1,5 @@ +export class RollbackBatchSchemaChangeDs { + batchId: string; + userId: string; + masterPassword?: string; +} diff --git a/backend/src/entities/table-schema/application/data-structures/rollback-schema-change.ds.ts b/backend/src/entities/table-schema/application/data-structures/rollback-schema-change.ds.ts new file mode 100644 index 000000000..9ea3d36b9 --- /dev/null +++ b/backend/src/entities/table-schema/application/data-structures/rollback-schema-change.ds.ts @@ -0,0 +1,5 @@ +export class RollbackSchemaChangeDs { + changeId: string; + userId: string; + masterPassword?: string; +} diff --git a/backend/src/entities/table-schema/application/data-transfer-objects/approve-batch-schema-change.dto.ts b/backend/src/entities/table-schema/application/data-transfer-objects/approve-batch-schema-change.dto.ts new file mode 100644 index 000000000..d40762efb --- /dev/null +++ b/backend/src/entities/table-schema/application/data-transfer-objects/approve-batch-schema-change.dto.ts @@ -0,0 +1,15 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsBoolean, IsOptional } from 'class-validator'; + +export class ApproveBatchSchemaChangeDto { + @ApiProperty({ + type: Boolean, + required: false, + default: false, + description: + 'Must be true to approve a batch containing any non-reversible change. Without it the server rejects the request and lists the offending change ids.', + }) + @IsOptional() + @IsBoolean() + confirmedDestructive?: boolean; +} diff --git a/backend/src/entities/table-schema/application/data-transfer-objects/approve-schema-change.dto.ts b/backend/src/entities/table-schema/application/data-transfer-objects/approve-schema-change.dto.ts new file mode 100644 index 000000000..a4a0c37fa --- /dev/null +++ b/backend/src/entities/table-schema/application/data-transfer-objects/approve-schema-change.dto.ts @@ -0,0 +1,26 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsBoolean, IsOptional, IsString, MaxLength } from 'class-validator'; + +export class ApproveSchemaChangeDto { + @ApiProperty({ + type: String, + required: false, + description: 'User-edited SQL to run in place of the AI-proposed forward SQL.', + maxLength: 8000, + }) + @IsOptional() + @IsString() + @MaxLength(8000) + userModifiedSql?: string; + + @ApiProperty({ + type: Boolean, + required: false, + default: false, + description: + 'Must be true to approve a change where isReversible=false. Without it the server rejects the request.', + }) + @IsOptional() + @IsBoolean() + confirmedDestructive?: boolean; +} diff --git a/backend/src/entities/table-schema/application/data-transfer-objects/generate-schema-change.dto.ts b/backend/src/entities/table-schema/application/data-transfer-objects/generate-schema-change.dto.ts new file mode 100644 index 000000000..c7d9ed7f2 --- /dev/null +++ b/backend/src/entities/table-schema/application/data-transfer-objects/generate-schema-change.dto.ts @@ -0,0 +1,18 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { IsNotEmpty, IsString, MaxLength, MinLength } from 'class-validator'; + +export class GenerateSchemaChangeDto { + @ApiProperty({ + type: String, + required: true, + description: 'Natural-language description of the desired schema change.', + example: 'Create a products table with id, name and price columns.', + minLength: 1, + maxLength: 2000, + }) + @IsString() + @IsNotEmpty() + @MinLength(1) + @MaxLength(2000) + userPrompt: string; +} diff --git a/backend/src/entities/table-schema/application/data-transfer-objects/schema-change-batch-response.dto.ts b/backend/src/entities/table-schema/application/data-transfer-objects/schema-change-batch-response.dto.ts new file mode 100644 index 000000000..f0c9281fb --- /dev/null +++ b/backend/src/entities/table-schema/application/data-transfer-objects/schema-change-batch-response.dto.ts @@ -0,0 +1,16 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { SchemaChangeResponseDto } from './schema-change-response.dto.js'; + +export class SchemaChangeBatchResponseDto { + @ApiProperty({ + description: + 'Identifier shared by every change generated from a single user prompt. Use it to approve/reject/rollback the entire batch in one call.', + }) + batchId: string; + + @ApiProperty({ + type: [SchemaChangeResponseDto], + description: 'Generated changes ordered by orderInBatch (dependency order — parents first).', + }) + changes: SchemaChangeResponseDto[]; +} diff --git a/backend/src/entities/table-schema/application/data-transfer-objects/schema-change-list-response.dto.ts b/backend/src/entities/table-schema/application/data-transfer-objects/schema-change-list-response.dto.ts new file mode 100644 index 000000000..d43a112d8 --- /dev/null +++ b/backend/src/entities/table-schema/application/data-transfer-objects/schema-change-list-response.dto.ts @@ -0,0 +1,11 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { PaginationDs } from '../../../table/application/data-structures/pagination.ds.js'; +import { SchemaChangeResponseDto } from './schema-change-response.dto.js'; + +export class SchemaChangeListResponseDto { + @ApiProperty({ type: SchemaChangeResponseDto, isArray: true }) + data: SchemaChangeResponseDto[]; + + @ApiProperty({ type: PaginationDs }) + pagination: PaginationDs; +} diff --git a/backend/src/entities/table-schema/application/data-transfer-objects/schema-change-response.dto.ts b/backend/src/entities/table-schema/application/data-transfer-objects/schema-change-response.dto.ts new file mode 100644 index 000000000..fe4189337 --- /dev/null +++ b/backend/src/entities/table-schema/application/data-transfer-objects/schema-change-response.dto.ts @@ -0,0 +1,74 @@ +import { ApiProperty } from '@nestjs/swagger'; +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; +import { SchemaChangeStatusEnum, SchemaChangeTypeEnum } from '../../table-schema-change-enums.js'; + +export class SchemaChangeResponseDto { + @ApiProperty() + id: string; + + @ApiProperty() + connectionId: string; + + @ApiProperty({ required: false, nullable: true }) + batchId: string | null; + + @ApiProperty() + orderInBatch: number; + + @ApiProperty({ required: false, nullable: true }) + authorId: string | null; + + @ApiProperty({ required: false, nullable: true }) + previousChangeId: string | null; + + @ApiProperty() + forwardSql: string; + + @ApiProperty({ required: false, nullable: true }) + rollbackSql: string | null; + + @ApiProperty({ required: false, nullable: true }) + userModifiedSql: string | null; + + @ApiProperty({ enum: SchemaChangeStatusEnum }) + status: SchemaChangeStatusEnum; + + @ApiProperty({ enum: SchemaChangeTypeEnum }) + changeType: SchemaChangeTypeEnum; + + @ApiProperty() + targetTableName: string; + + @ApiProperty({ enum: ConnectionTypesEnum }) + databaseType: ConnectionTypesEnum; + + @ApiProperty({ required: false, nullable: true }) + executionError: string | null; + + @ApiProperty() + isReversible: boolean; + + @ApiProperty() + autoRollbackAttempted: boolean; + + @ApiProperty() + autoRollbackSucceeded: boolean; + + @ApiProperty() + userPrompt: string; + + @ApiProperty({ required: false, nullable: true }) + aiSummary: string | null; + + @ApiProperty({ required: false, nullable: true }) + aiReasoning: string | null; + + @ApiProperty() + createdAt: Date; + + @ApiProperty({ required: false, nullable: true }) + appliedAt: Date | null; + + @ApiProperty({ required: false, nullable: true }) + rolledBackAt: Date | null; +} diff --git a/backend/src/entities/table-schema/repository/custom-table-schema-change-repository-extension.ts b/backend/src/entities/table-schema/repository/custom-table-schema-change-repository-extension.ts new file mode 100644 index 000000000..a85da68bb --- /dev/null +++ b/backend/src/entities/table-schema/repository/custom-table-schema-change-repository-extension.ts @@ -0,0 +1,100 @@ +import { Repository } from 'typeorm'; +import { TableSchemaChangeEntity } from '../table-schema-change.entity.js'; +import { SchemaChangeStatusEnum } from '../table-schema-change-enums.js'; +import { + ITableSchemaChangeRepository, + UpdateSchemaChangeStatusMeta, +} from './table-schema-change.repository.interface.js'; + +export const customTableSchemaChangeRepositoryExtension: ITableSchemaChangeRepository & + ThisType & ITableSchemaChangeRepository> = { + async createPendingChange(data: Partial): Promise { + const entity = this.create({ + ...data, + status: SchemaChangeStatusEnum.PENDING, + }); + return await this.save(entity); + }, + + async createPendingBatch(items: Partial[]): Promise { + const entities = items.map((item) => + this.create({ + ...item, + status: SchemaChangeStatusEnum.PENDING, + }), + ); + return await this.save(entities); + }, + + async findByIdWithRelations(id: string): Promise { + return await this.findOne({ + where: { id }, + relations: ['connection', 'author', 'previousChange'], + }); + }, + + async findByBatchId(batchId: string): Promise { + return await this.find({ + where: { batchId }, + order: { orderInBatch: 'ASC' }, + }); + }, + + async findByConnectionPaginated( + connectionId: string, + options: { limit: number; offset: number }, + ): Promise<[TableSchemaChangeEntity[], number]> { + return await this.findAndCount({ + where: { connectionId }, + order: { createdAt: 'DESC' }, + skip: options.offset, + take: options.limit, + }); + }, + + async updateStatus( + id: string, + status: SchemaChangeStatusEnum, + meta?: UpdateSchemaChangeStatusMeta, + ): Promise { + const patch: Partial = { status }; + if (meta) { + if (meta.executionError !== undefined) { + patch.executionError = meta.executionError; + } + if (meta.appliedAt !== undefined) { + patch.appliedAt = meta.appliedAt; + } + if (meta.rolledBackAt !== undefined) { + patch.rolledBackAt = meta.rolledBackAt; + } + if (meta.autoRollbackAttempted !== undefined) { + patch.autoRollbackAttempted = meta.autoRollbackAttempted; + } + if (meta.autoRollbackSucceeded !== undefined) { + patch.autoRollbackSucceeded = meta.autoRollbackSucceeded; + } + if (meta.userModifiedSql !== undefined) { + patch.userModifiedSql = meta.userModifiedSql; + } + } + await this.update({ id }, patch); + return await this.findOne({ where: { id } }); + }, + + async transitionStatusIfMatches( + id: string, + expectedStatus: SchemaChangeStatusEnum, + nextStatus: SchemaChangeStatusEnum, + ): Promise { + const result = await this.update({ id, status: expectedStatus }, { status: nextStatus }); + return (result.affected ?? 0) > 0; + }, + + async findLatestAppliedChange(connectionId: string): Promise { + return await this.findOne({ + where: { connectionId, status: SchemaChangeStatusEnum.APPLIED }, + order: { appliedAt: 'DESC' }, + }); + }, +}; diff --git a/backend/src/entities/table-schema/repository/table-schema-change.repository.interface.ts b/backend/src/entities/table-schema/repository/table-schema-change.repository.interface.ts new file mode 100644 index 000000000..1f2bc58d6 --- /dev/null +++ b/backend/src/entities/table-schema/repository/table-schema-change.repository.interface.ts @@ -0,0 +1,53 @@ +import { Repository } from 'typeorm'; +import { TableSchemaChangeEntity } from '../table-schema-change.entity.js'; +import { SchemaChangeStatusEnum } from '../table-schema-change-enums.js'; + +export interface UpdateSchemaChangeStatusMeta { + executionError?: string | null; + appliedAt?: Date | null; + rolledBackAt?: Date | null; + autoRollbackAttempted?: boolean; + autoRollbackSucceeded?: boolean; + userModifiedSql?: string | null; +} + +export interface ITableSchemaChangeRepository { + createPendingChange( + this: Repository, + data: Partial, + ): Promise; + + createPendingBatch( + this: Repository, + items: Partial[], + ): Promise; + + findByIdWithRelations(this: Repository, id: string): Promise; + + findByBatchId(this: Repository, batchId: string): Promise; + + findByConnectionPaginated( + this: Repository, + connectionId: string, + options: { limit: number; offset: number }, + ): Promise<[TableSchemaChangeEntity[], number]>; + + updateStatus( + this: Repository, + id: string, + status: SchemaChangeStatusEnum, + meta?: UpdateSchemaChangeStatusMeta, + ): Promise; + + transitionStatusIfMatches( + this: Repository, + id: string, + expectedStatus: SchemaChangeStatusEnum, + nextStatus: SchemaChangeStatusEnum, + ): Promise; + + findLatestAppliedChange( + this: Repository, + connectionId: string, + ): Promise; +} diff --git a/backend/src/entities/table-schema/table-schema-change-enums.ts b/backend/src/entities/table-schema/table-schema-change-enums.ts new file mode 100644 index 000000000..b1c020315 --- /dev/null +++ b/backend/src/entities/table-schema/table-schema-change-enums.ts @@ -0,0 +1,49 @@ +export enum SchemaChangeStatusEnum { + PENDING = 'PENDING', + APPROVED = 'APPROVED', + APPLYING = 'APPLYING', + APPLIED = 'APPLIED', + FAILED = 'FAILED', + REJECTED = 'REJECTED', + ROLLED_BACK = 'ROLLED_BACK', +} + +export enum SchemaChangeTypeEnum { + CREATE_TABLE = 'CREATE_TABLE', + DROP_TABLE = 'DROP_TABLE', + ADD_COLUMN = 'ADD_COLUMN', + DROP_COLUMN = 'DROP_COLUMN', + ALTER_COLUMN = 'ALTER_COLUMN', + ADD_INDEX = 'ADD_INDEX', + DROP_INDEX = 'DROP_INDEX', + ADD_FOREIGN_KEY = 'ADD_FOREIGN_KEY', + DROP_FOREIGN_KEY = 'DROP_FOREIGN_KEY', + ADD_PRIMARY_KEY = 'ADD_PRIMARY_KEY', + DROP_PRIMARY_KEY = 'DROP_PRIMARY_KEY', + MONGO_CREATE_COLLECTION = 'MONGO_CREATE_COLLECTION', + MONGO_DROP_COLLECTION = 'MONGO_DROP_COLLECTION', + MONGO_SET_VALIDATOR = 'MONGO_SET_VALIDATOR', + MONGO_CREATE_INDEX = 'MONGO_CREATE_INDEX', + MONGO_DROP_INDEX = 'MONGO_DROP_INDEX', + DYNAMODB_CREATE_TABLE = 'DYNAMODB_CREATE_TABLE', + DYNAMODB_DROP_TABLE = 'DYNAMODB_DROP_TABLE', + DYNAMODB_UPDATE_TABLE = 'DYNAMODB_UPDATE_TABLE', + DYNAMODB_UPDATE_TTL = 'DYNAMODB_UPDATE_TTL', + ELASTICSEARCH_CREATE_INDEX = 'ELASTICSEARCH_CREATE_INDEX', + ELASTICSEARCH_DELETE_INDEX = 'ELASTICSEARCH_DELETE_INDEX', + ELASTICSEARCH_UPDATE_MAPPING = 'ELASTICSEARCH_UPDATE_MAPPING', + ROLLBACK = 'ROLLBACK', + OTHER = 'OTHER', +} + +export function isMongoSchemaChangeType(changeType: SchemaChangeTypeEnum): boolean { + return changeType.startsWith('MONGO_'); +} + +export function isDynamoDbSchemaChangeType(changeType: SchemaChangeTypeEnum): boolean { + return changeType.startsWith('DYNAMODB_'); +} + +export function isElasticsearchSchemaChangeType(changeType: SchemaChangeTypeEnum): boolean { + return changeType.startsWith('ELASTICSEARCH_'); +} diff --git a/backend/src/entities/table-schema/table-schema-change.entity.ts b/backend/src/entities/table-schema/table-schema-change.entity.ts new file mode 100644 index 000000000..a527f3eeb --- /dev/null +++ b/backend/src/entities/table-schema/table-schema-change.entity.ts @@ -0,0 +1,99 @@ +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; +import { Column, Entity, Index, JoinColumn, ManyToOne, PrimaryGeneratedColumn, Relation } from 'typeorm'; +import { ConnectionEntity } from '../connection/connection.entity.js'; +import { UserEntity } from '../user/user.entity.js'; +import { SchemaChangeStatusEnum, SchemaChangeTypeEnum } from './table-schema-change-enums.js'; + +@Entity('table_schema_change') +@Index('IDX_tsc_connection_created', ['connectionId', 'createdAt']) +@Index('IDX_tsc_previous_change', ['previousChangeId']) +@Index('IDX_tsc_batch_order', ['batchId', 'orderInBatch']) +export class TableSchemaChangeEntity { + @PrimaryGeneratedColumn('uuid') + id: string; + + @Column({ type: 'varchar', length: 38 }) + connectionId: string; + + @Column({ type: 'uuid', nullable: true }) + batchId: string | null; + + @Column({ type: 'int', default: 0 }) + orderInBatch: number; + + @ManyToOne(() => ConnectionEntity, { onDelete: 'CASCADE' }) + @JoinColumn({ name: 'connectionId' }) + connection: Relation; + + @Column({ type: 'uuid', nullable: true }) + authorId: string | null; + + @ManyToOne(() => UserEntity, { onDelete: 'SET NULL', nullable: true }) + @JoinColumn({ name: 'authorId' }) + author: Relation | null; + + @Column({ type: 'uuid', nullable: true }) + previousChangeId: string | null; + + @ManyToOne(() => TableSchemaChangeEntity, { onDelete: 'SET NULL', nullable: true }) + @JoinColumn({ name: 'previousChangeId' }) + previousChange: Relation | null; + + @Column({ type: 'text' }) + forwardSql: string; + + @Column({ type: 'text', nullable: true }) + rollbackSql: string | null; + + @Column({ type: 'text', nullable: true }) + userModifiedSql: string | null; + + @Column({ + type: 'enum', + enum: SchemaChangeStatusEnum, + default: SchemaChangeStatusEnum.PENDING, + }) + status: SchemaChangeStatusEnum; + + @Column({ type: 'enum', enum: SchemaChangeTypeEnum }) + changeType: SchemaChangeTypeEnum; + + @Column({ type: 'varchar', length: 255 }) + targetTableName: string; + + @Column({ type: 'enum', enum: ConnectionTypesEnum }) + databaseType: ConnectionTypesEnum; + + @Column({ type: 'text', nullable: true }) + executionError: string | null; + + @Column({ type: 'boolean', default: false }) + isReversible: boolean; + + @Column({ type: 'boolean', default: false }) + autoRollbackAttempted: boolean; + + @Column({ type: 'boolean', default: false }) + autoRollbackSucceeded: boolean; + + @Column({ type: 'text' }) + userPrompt: string; + + @Column({ type: 'text', nullable: true }) + aiSummary: string | null; + + @Column({ type: 'text', nullable: true }) + aiReasoning: string | null; + + @Column({ type: 'varchar', length: 128, nullable: true }) + aiModelUsed: string | null; + + @Column({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' }) + createdAt: Date; + + @Column({ type: 'timestamp', nullable: true }) + appliedAt: Date | null; + + @Column({ type: 'timestamp', nullable: true }) + rolledBackAt: Date | null; +} diff --git a/backend/src/entities/table-schema/table-schema.controller.ts b/backend/src/entities/table-schema/table-schema.controller.ts new file mode 100644 index 000000000..806de0450 --- /dev/null +++ b/backend/src/entities/table-schema/table-schema.controller.ts @@ -0,0 +1,232 @@ +import { + Body, + Controller, + Get, + HttpCode, + HttpStatus, + Inject, + Injectable, + Post, + Query, + UseGuards, + UseInterceptors, +} from '@nestjs/common'; +import { ApiBearerAuth, ApiBody, ApiOperation, ApiParam, ApiQuery, ApiResponse, ApiTags } from '@nestjs/swagger'; +import { UseCaseType } from '../../common/data-injection.tokens.js'; +import { MasterPassword } from '../../decorators/master-password.decorator.js'; +import { SlugUuid } from '../../decorators/slug-uuid.decorator.js'; +import { UserId } from '../../decorators/user-id.decorator.js'; +import { + ConnectionEditGuard, + SchemaChangeBatchOwnershipGuard, + SchemaChangeOwnershipGuard, +} from '../../guards/index.js'; +import { SentryInterceptor } from '../../interceptors/sentry.interceptor.js'; +import { ApproveBatchSchemaChangeDto } from './application/data-transfer-objects/approve-batch-schema-change.dto.js'; +import { ApproveSchemaChangeDto } from './application/data-transfer-objects/approve-schema-change.dto.js'; +import { GenerateSchemaChangeDto } from './application/data-transfer-objects/generate-schema-change.dto.js'; +import { SchemaChangeBatchResponseDto } from './application/data-transfer-objects/schema-change-batch-response.dto.js'; +import { SchemaChangeListResponseDto } from './application/data-transfer-objects/schema-change-list-response.dto.js'; +import { SchemaChangeResponseDto } from './application/data-transfer-objects/schema-change-response.dto.js'; +import { + IApproveBatchSchemaChange, + IApproveSchemaChange, + IGenerateSchemaChange, + IGetBatchSchemaChange, + IGetSchemaChange, + IListSchemaChanges, + IRejectBatchSchemaChange, + IRejectSchemaChange, + IRollbackBatchSchemaChange, + IRollbackSchemaChange, +} from './use-cases/table-schema-use-cases.interface.js'; + +@UseInterceptors(SentryInterceptor) +@Controller() +@ApiBearerAuth() +@ApiTags('Table Schema Changes') +@Injectable() +export class TableSchemaController { + constructor( + @Inject(UseCaseType.GENERATE_SCHEMA_CHANGE) + private readonly generateSchemaChangeUseCase: IGenerateSchemaChange, + @Inject(UseCaseType.APPROVE_SCHEMA_CHANGE) + private readonly approveSchemaChangeUseCase: IApproveSchemaChange, + @Inject(UseCaseType.REJECT_SCHEMA_CHANGE) + private readonly rejectSchemaChangeUseCase: IRejectSchemaChange, + @Inject(UseCaseType.ROLLBACK_SCHEMA_CHANGE) + private readonly rollbackSchemaChangeUseCase: IRollbackSchemaChange, + @Inject(UseCaseType.LIST_SCHEMA_CHANGES) + private readonly listSchemaChangesUseCase: IListSchemaChanges, + @Inject(UseCaseType.GET_SCHEMA_CHANGE) + private readonly getSchemaChangeUseCase: IGetSchemaChange, + @Inject(UseCaseType.APPROVE_BATCH_SCHEMA_CHANGE) + private readonly approveBatchSchemaChangeUseCase: IApproveBatchSchemaChange, + @Inject(UseCaseType.REJECT_BATCH_SCHEMA_CHANGE) + private readonly rejectBatchSchemaChangeUseCase: IRejectBatchSchemaChange, + @Inject(UseCaseType.ROLLBACK_BATCH_SCHEMA_CHANGE) + private readonly rollbackBatchSchemaChangeUseCase: IRollbackBatchSchemaChange, + @Inject(UseCaseType.GET_BATCH_SCHEMA_CHANGE) + private readonly getBatchSchemaChangeUseCase: IGetBatchSchemaChange, + ) {} + + @ApiOperation({ + summary: + 'Generate one or more schema changes from a natural-language prompt. The response is always a batch envelope; single-change prompts return a length-1 array.', + }) + @ApiParam({ name: 'connectionId', type: String }) + @ApiBody({ type: GenerateSchemaChangeDto }) + @ApiResponse({ status: 201, type: SchemaChangeBatchResponseDto }) + @UseGuards(ConnectionEditGuard) + @Post('/table-schema/:connectionId/generate') + @HttpCode(HttpStatus.CREATED) + async generate( + @SlugUuid('connectionId') connectionId: string, + @UserId() userId: string, + @MasterPassword() masterPassword: string, + @Body() body: GenerateSchemaChangeDto, + ): Promise { + return await this.generateSchemaChangeUseCase.execute({ + connectionId, + userPrompt: body.userPrompt, + userId, + masterPassword, + }); + } + + @ApiOperation({ summary: 'Approve and apply a pending schema change.' }) + @ApiParam({ name: 'changeId', type: String }) + @ApiBody({ type: ApproveSchemaChangeDto, required: false }) + @ApiResponse({ status: 200, type: SchemaChangeResponseDto }) + @UseGuards(SchemaChangeOwnershipGuard) + @Post('/table-schema/change/:changeId/approve') + @HttpCode(HttpStatus.OK) + async approve( + @SlugUuid('changeId') changeId: string, + @UserId() userId: string, + @MasterPassword() masterPassword: string, + @Body() body: ApproveSchemaChangeDto, + ): Promise { + return await this.approveSchemaChangeUseCase.execute({ + changeId, + userId, + masterPassword, + userModifiedSql: body?.userModifiedSql, + confirmedDestructive: body?.confirmedDestructive, + }); + } + + @ApiOperation({ summary: 'Reject a pending schema change.' }) + @ApiParam({ name: 'changeId', type: String }) + @ApiResponse({ status: 200, type: SchemaChangeResponseDto }) + @UseGuards(SchemaChangeOwnershipGuard) + @Post('/table-schema/change/:changeId/reject') + @HttpCode(HttpStatus.OK) + async reject(@SlugUuid('changeId') changeId: string, @UserId() userId: string): Promise { + return await this.rejectSchemaChangeUseCase.execute({ changeId, userId }); + } + + @ApiOperation({ summary: 'Roll back a previously applied schema change.' }) + @ApiParam({ name: 'changeId', type: String }) + @ApiResponse({ status: 200, type: SchemaChangeResponseDto }) + @UseGuards(SchemaChangeOwnershipGuard) + @Post('/table-schema/change/:changeId/rollback') + @HttpCode(HttpStatus.OK) + async rollback( + @SlugUuid('changeId') changeId: string, + @UserId() userId: string, + @MasterPassword() masterPassword: string, + ): Promise { + return await this.rollbackSchemaChangeUseCase.execute({ changeId, userId, masterPassword }); + } + + @ApiOperation({ summary: 'List schema-change history for a connection (newest first).' }) + @ApiParam({ name: 'connectionId', type: String }) + @ApiQuery({ name: 'limit', required: false, type: Number, description: 'Items per page (default: 20, max: 100).' }) + @ApiQuery({ name: 'offset', required: false, type: Number, description: 'Rows to skip (default: 0).' }) + @ApiResponse({ status: 200, type: SchemaChangeListResponseDto }) + @UseGuards(ConnectionEditGuard) + @Get('/table-schema/:connectionId/changes') + async list( + @SlugUuid('connectionId') connectionId: string, + @UserId() userId: string, + @Query('limit') rawLimit?: string, + @Query('offset') rawOffset?: string, + ): Promise { + const limit = Math.min(Math.max(Number.parseInt(rawLimit ?? '20', 10) || 20, 1), 100); + const offset = Math.max(Number.parseInt(rawOffset ?? '0', 10) || 0, 0); + return await this.listSchemaChangesUseCase.execute({ connectionId, userId, limit, offset }); + } + + @ApiOperation({ summary: 'Get a single schema change.' }) + @ApiParam({ name: 'changeId', type: String }) + @ApiResponse({ status: 200, type: SchemaChangeResponseDto }) + @UseGuards(SchemaChangeOwnershipGuard) + @Get('/table-schema/change/:changeId') + async get(@SlugUuid('changeId') changeId: string, @UserId() userId: string): Promise { + return await this.getSchemaChangeUseCase.execute({ changeId, userId }); + } + + @ApiOperation({ + summary: + 'Approve and apply every pending change in a batch in dependency order. Halts on first failure and rolls back already-applied items in reverse.', + }) + @ApiParam({ name: 'batchId', type: String }) + @ApiBody({ type: ApproveBatchSchemaChangeDto, required: false }) + @ApiResponse({ status: 200, type: SchemaChangeBatchResponseDto }) + @UseGuards(SchemaChangeBatchOwnershipGuard) + @Post('/table-schema/batch/:batchId/approve') + @HttpCode(HttpStatus.OK) + async approveBatch( + @SlugUuid('batchId') batchId: string, + @UserId() userId: string, + @MasterPassword() masterPassword: string, + @Body() body: ApproveBatchSchemaChangeDto, + ): Promise { + return await this.approveBatchSchemaChangeUseCase.execute({ + batchId, + userId, + masterPassword, + confirmedDestructive: body?.confirmedDestructive, + }); + } + + @ApiOperation({ summary: 'Reject every pending change in a batch.' }) + @ApiParam({ name: 'batchId', type: String }) + @ApiResponse({ status: 200, type: SchemaChangeBatchResponseDto }) + @UseGuards(SchemaChangeBatchOwnershipGuard) + @Post('/table-schema/batch/:batchId/reject') + @HttpCode(HttpStatus.OK) + async rejectBatch( + @SlugUuid('batchId') batchId: string, + @UserId() userId: string, + ): Promise { + return await this.rejectBatchSchemaChangeUseCase.execute({ batchId, userId }); + } + + @ApiOperation({ summary: 'Roll back every applied change in a batch in reverse order.' }) + @ApiParam({ name: 'batchId', type: String }) + @ApiResponse({ status: 200, type: SchemaChangeBatchResponseDto }) + @UseGuards(SchemaChangeBatchOwnershipGuard) + @Post('/table-schema/batch/:batchId/rollback') + @HttpCode(HttpStatus.OK) + async rollbackBatch( + @SlugUuid('batchId') batchId: string, + @UserId() userId: string, + @MasterPassword() masterPassword: string, + ): Promise { + return await this.rollbackBatchSchemaChangeUseCase.execute({ batchId, userId, masterPassword }); + } + + @ApiOperation({ summary: 'Get every change in a batch (ordered by orderInBatch).' }) + @ApiParam({ name: 'batchId', type: String }) + @ApiResponse({ status: 200, type: SchemaChangeBatchResponseDto }) + @UseGuards(SchemaChangeBatchOwnershipGuard) + @Get('/table-schema/batch/:batchId') + async getBatch( + @SlugUuid('batchId') batchId: string, + @UserId() userId: string, + ): Promise { + return await this.getBatchSchemaChangeUseCase.execute({ batchId, userId }); + } +} diff --git a/backend/src/entities/table-schema/table-schema.module.ts b/backend/src/entities/table-schema/table-schema.module.ts new file mode 100644 index 000000000..7d6d889c8 --- /dev/null +++ b/backend/src/entities/table-schema/table-schema.module.ts @@ -0,0 +1,92 @@ +import { MiddlewareConsumer, Module, NestModule, RequestMethod } from '@nestjs/common'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { AuthMiddleware } from '../../authorization/auth.middleware.js'; +import { GlobalDatabaseContext } from '../../common/application/global-database-context.js'; +import { BaseType, UseCaseType } from '../../common/data-injection.tokens.js'; +import { SchemaChangeBatchOwnershipGuard, SchemaChangeOwnershipGuard } from '../../guards/index.js'; +import { ConnectionEntity } from '../connection/connection.entity.js'; +import { LogOutEntity } from '../log-out/log-out.entity.js'; +import { UserEntity } from '../user/user.entity.js'; +import { TableSchemaController } from './table-schema.controller.js'; +import { TableSchemaChangeEntity } from './table-schema-change.entity.js'; +import { ApproveAndApplySchemaChangeUseCase } from './use-cases/approve-and-apply-schema-change.use-case.js'; +import { ApproveBatchSchemaChangesUseCase } from './use-cases/approve-batch-schema-changes.use-case.js'; +import { GenerateSchemaChangeUseCase } from './use-cases/generate-schema-change.use-case.js'; +import { GetBatchSchemaChangesUseCase } from './use-cases/get-batch-schema-changes.use-case.js'; +import { GetSchemaChangeUseCase } from './use-cases/get-schema-change.use-case.js'; +import { ListSchemaChangesUseCase } from './use-cases/list-schema-changes.use-case.js'; +import { RejectBatchSchemaChangesUseCase } from './use-cases/reject-batch-schema-changes.use-case.js'; +import { RejectSchemaChangeUseCase } from './use-cases/reject-schema-change.use-case.js'; +import { RollbackBatchSchemaChangesUseCase } from './use-cases/rollback-batch-schema-changes.use-case.js'; +import { RollbackSchemaChangeUseCase } from './use-cases/rollback-schema-change.use-case.js'; + +@Module({ + imports: [TypeOrmModule.forFeature([TableSchemaChangeEntity, ConnectionEntity, UserEntity, LogOutEntity])], + providers: [ + { + provide: BaseType.GLOBAL_DB_CONTEXT, + useClass: GlobalDatabaseContext, + }, + { + provide: UseCaseType.GENERATE_SCHEMA_CHANGE, + useClass: GenerateSchemaChangeUseCase, + }, + { + provide: UseCaseType.APPROVE_SCHEMA_CHANGE, + useClass: ApproveAndApplySchemaChangeUseCase, + }, + { + provide: UseCaseType.REJECT_SCHEMA_CHANGE, + useClass: RejectSchemaChangeUseCase, + }, + { + provide: UseCaseType.ROLLBACK_SCHEMA_CHANGE, + useClass: RollbackSchemaChangeUseCase, + }, + { + provide: UseCaseType.LIST_SCHEMA_CHANGES, + useClass: ListSchemaChangesUseCase, + }, + { + provide: UseCaseType.GET_SCHEMA_CHANGE, + useClass: GetSchemaChangeUseCase, + }, + { + provide: UseCaseType.APPROVE_BATCH_SCHEMA_CHANGE, + useClass: ApproveBatchSchemaChangesUseCase, + }, + { + provide: UseCaseType.REJECT_BATCH_SCHEMA_CHANGE, + useClass: RejectBatchSchemaChangesUseCase, + }, + { + provide: UseCaseType.ROLLBACK_BATCH_SCHEMA_CHANGE, + useClass: RollbackBatchSchemaChangesUseCase, + }, + { + provide: UseCaseType.GET_BATCH_SCHEMA_CHANGE, + useClass: GetBatchSchemaChangesUseCase, + }, + SchemaChangeOwnershipGuard, + SchemaChangeBatchOwnershipGuard, + ], + controllers: [TableSchemaController], +}) +export class TableSchemaModule implements NestModule { + public configure(consumer: MiddlewareConsumer): void { + consumer + .apply(AuthMiddleware) + .forRoutes( + { path: '/table-schema/:connectionId/generate', method: RequestMethod.POST }, + { path: '/table-schema/:connectionId/changes', method: RequestMethod.GET }, + { path: '/table-schema/change/:changeId', method: RequestMethod.GET }, + { path: '/table-schema/change/:changeId/approve', method: RequestMethod.POST }, + { path: '/table-schema/change/:changeId/reject', method: RequestMethod.POST }, + { path: '/table-schema/change/:changeId/rollback', method: RequestMethod.POST }, + { path: '/table-schema/batch/:batchId', method: RequestMethod.GET }, + { path: '/table-schema/batch/:batchId/approve', method: RequestMethod.POST }, + { path: '/table-schema/batch/:batchId/reject', method: RequestMethod.POST }, + { path: '/table-schema/batch/:batchId/rollback', method: RequestMethod.POST }, + ); + } +} diff --git a/backend/src/entities/table-schema/use-cases/approve-and-apply-schema-change.use-case.ts b/backend/src/entities/table-schema/use-cases/approve-and-apply-schema-change.use-case.ts new file mode 100644 index 000000000..e7a01d16a --- /dev/null +++ b/backend/src/entities/table-schema/use-cases/approve-and-apply-schema-change.use-case.ts @@ -0,0 +1,175 @@ +import { + BadRequestException, + ConflictException, + Inject, + Injectable, + Logger, + NotFoundException, + Scope, +} from '@nestjs/common'; +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { Messages } from '../../../exceptions/text/messages.js'; +import { ApproveSchemaChangeDs } from '../application/data-structures/approve-schema-change.ds.js'; +import { SchemaChangeResponseDto } from '../application/data-transfer-objects/schema-change-response.dto.js'; +import { + isDynamoDbSchemaChangeType, + isElasticsearchSchemaChangeType, + isMongoSchemaChangeType, + SchemaChangeStatusEnum, +} from '../table-schema-change-enums.js'; +import { assertDialectSupported } from '../utils/assert-dialect-supported.js'; +import { validateProposedDynamoDbOp } from '../utils/dynamodb-schema-op.js'; +import { validateProposedElasticsearchOp } from '../utils/elasticsearch-schema-op.js'; +import { executeSchemaChange } from '../utils/execute-schema-change.js'; +import { mapSchemaChangeToResponseDto } from '../utils/map-schema-change-to-response-dto.js'; +import { validateProposedMongoOp } from '../utils/mongo-schema-op.js'; +import { validateProposedDdl } from '../utils/validate-proposed-ddl.js'; +import { IApproveSchemaChange } from './table-schema-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class ApproveAndApplySchemaChangeUseCase + extends AbstractUseCase + implements IApproveSchemaChange +{ + private readonly logger = new Logger(ApproveAndApplySchemaChangeUseCase.name); + + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + ) { + super(); + } + + protected async implementation(inputData: ApproveSchemaChangeDs): Promise { + const { changeId, masterPassword, userModifiedSql, confirmedDestructive } = inputData; + + const change = await this._dbContext.tableSchemaChangeRepository.findByIdWithRelations(changeId); + if (!change) { + throw new NotFoundException('Schema change not found.'); + } + + if (change.status !== SchemaChangeStatusEnum.PENDING && change.status !== SchemaChangeStatusEnum.APPROVED) { + throw new ConflictException(`Schema change is ${change.status}; only PENDING or APPROVED can be applied.`); + } + + if (!change.isReversible && !confirmedDestructive) { + throw new BadRequestException({ + message: 'This change is not reversible. Re-submit with confirmedDestructive=true to proceed.', + type: 'destructive_confirmation_required', + }); + } + + const connectionType = change.databaseType as ConnectionTypesEnum; + assertDialectSupported(connectionType); + + const connection = await this._dbContext.connectionRepository.findAndDecryptConnection( + change.connectionId, + masterPassword, + ); + if (!connection) { + throw new NotFoundException(Messages.CONNECTION_NOT_FOUND); + } + + const isMongo = isMongoSchemaChangeType(change.changeType); + const isDynamoDb = isDynamoDbSchemaChangeType(change.changeType); + const isElasticsearch = isElasticsearchSchemaChangeType(change.changeType); + + let sqlToRun = change.forwardSql; + if (userModifiedSql && userModifiedSql.trim().length > 0) { + if (isMongo) { + validateProposedMongoOp({ + opJson: userModifiedSql, + changeType: change.changeType, + targetTableName: change.targetTableName, + }); + } else if (isDynamoDb) { + validateProposedDynamoDbOp({ + opJson: userModifiedSql, + changeType: change.changeType, + targetTableName: change.targetTableName, + }); + } else if (isElasticsearch) { + validateProposedElasticsearchOp({ + opJson: userModifiedSql, + changeType: change.changeType, + targetTableName: change.targetTableName, + }); + } else { + validateProposedDdl({ + sql: userModifiedSql, + connectionType, + changeType: change.changeType, + targetTableName: change.targetTableName, + }); + } + sqlToRun = userModifiedSql; + await this._dbContext.tableSchemaChangeRepository.updateStatus(change.id, change.status, { + userModifiedSql, + }); + } + + const transitioned = await this._dbContext.tableSchemaChangeRepository.transitionStatusIfMatches( + change.id, + change.status, + SchemaChangeStatusEnum.APPLYING, + ); + if (!transitioned) { + throw new ConflictException('Schema change state changed concurrently; refresh and retry.'); + } + + try { + await executeSchemaChange({ + connection, + connectionType, + changeType: change.changeType, + targetTableName: change.targetTableName, + sql: sqlToRun, + }); + const updated = await this._dbContext.tableSchemaChangeRepository.updateStatus( + change.id, + SchemaChangeStatusEnum.APPLIED, + { appliedAt: new Date(), executionError: null }, + ); + return mapSchemaChangeToResponseDto(updated!); + } catch (err) { + const error = err as Error; + this.logger.error(`Forward execution failed for change ${change.id}: ${error.message}`); + + let autoRollbackSucceeded = false; + let rollbackError: string | null = null; + if (change.rollbackSql) { + try { + await executeSchemaChange({ + connection, + connectionType, + changeType: change.changeType, + targetTableName: change.targetTableName, + sql: change.rollbackSql, + allowAnyOperation: true, + }); + autoRollbackSucceeded = true; + } catch (rbErr) { + rollbackError = (rbErr as Error).message; + this.logger.error(`Auto-rollback failed for change ${change.id}: ${rollbackError}`); + } + } + + const composedError = rollbackError ? `${error.message} | auto-rollback failed: ${rollbackError}` : error.message; + + await this._dbContext.tableSchemaChangeRepository.updateStatus(change.id, SchemaChangeStatusEnum.FAILED, { + executionError: composedError, + autoRollbackAttempted: !!change.rollbackSql, + autoRollbackSucceeded, + }); + + throw new BadRequestException({ + message: `Schema change failed: ${error.message}`, + autoRollbackAttempted: !!change.rollbackSql, + autoRollbackSucceeded, + }); + } + } +} diff --git a/backend/src/entities/table-schema/use-cases/approve-batch-schema-changes.use-case.ts b/backend/src/entities/table-schema/use-cases/approve-batch-schema-changes.use-case.ts new file mode 100644 index 000000000..ee1034dd2 --- /dev/null +++ b/backend/src/entities/table-schema/use-cases/approve-batch-schema-changes.use-case.ts @@ -0,0 +1,199 @@ +import { + BadRequestException, + ConflictException, + Inject, + Injectable, + Logger, + NotFoundException, + Scope, +} from '@nestjs/common'; +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { Messages } from '../../../exceptions/text/messages.js'; +import { ApproveBatchSchemaChangeDs } from '../application/data-structures/approve-batch-schema-change.ds.js'; +import { SchemaChangeBatchResponseDto } from '../application/data-transfer-objects/schema-change-batch-response.dto.js'; +import { TableSchemaChangeEntity } from '../table-schema-change.entity.js'; +import { SchemaChangeStatusEnum } from '../table-schema-change-enums.js'; +import { assertDialectSupported } from '../utils/assert-dialect-supported.js'; +import { executeSchemaChange } from '../utils/execute-schema-change.js'; +import { mapSchemaChangeToResponseDto } from '../utils/map-schema-change-to-response-dto.js'; +import { IApproveBatchSchemaChange } from './table-schema-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class ApproveBatchSchemaChangesUseCase + extends AbstractUseCase + implements IApproveBatchSchemaChange +{ + private readonly logger = new Logger(ApproveBatchSchemaChangesUseCase.name); + + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + ) { + super(); + } + + protected async implementation(inputData: ApproveBatchSchemaChangeDs): Promise { + const { batchId, masterPassword, confirmedDestructive } = inputData; + + const items = await this._dbContext.tableSchemaChangeRepository.findByBatchId(batchId); + if (items.length === 0) { + throw new NotFoundException('Schema change batch not found.'); + } + + const pending = items.filter( + (it) => it.status === SchemaChangeStatusEnum.PENDING || it.status === SchemaChangeStatusEnum.APPROVED, + ); + if (pending.length === 0) { + throw new ConflictException('No items in this batch can be approved (they must be PENDING or APPROVED).'); + } + + const destructiveItems = pending.filter((it) => !it.isReversible); + if (destructiveItems.length > 0 && !confirmedDestructive) { + throw new BadRequestException({ + message: `This batch contains ${destructiveItems.length} non-reversible change(s) (ids: ${destructiveItems.map((it) => it.id).join(', ')}). Re-submit with confirmedDestructive=true to proceed.`, + type: 'destructive_confirmation_required', + }); + } + + const connectionType = items[0].databaseType as ConnectionTypesEnum; + assertDialectSupported(connectionType); + + const connection = await this._dbContext.connectionRepository.findAndDecryptConnection( + items[0].connectionId, + masterPassword, + ); + if (!connection) { + throw new NotFoundException(Messages.CONNECTION_NOT_FOUND); + } + + const applied: TableSchemaChangeEntity[] = []; + + for (const item of pending) { + const transitioned = await this._dbContext.tableSchemaChangeRepository.transitionStatusIfMatches( + item.id, + item.status, + SchemaChangeStatusEnum.APPLYING, + ); + if (!transitioned) { + await this.rollbackApplied(applied, connection, connectionType); + throw new ConflictException(`Schema change ${item.id} state changed concurrently; batch aborted.`); + } + + try { + await executeSchemaChange({ + connection, + connectionType, + changeType: item.changeType, + targetTableName: item.targetTableName, + sql: item.userModifiedSql ?? item.forwardSql, + }); + const updated = await this._dbContext.tableSchemaChangeRepository.updateStatus( + item.id, + SchemaChangeStatusEnum.APPLIED, + { appliedAt: new Date(), executionError: null }, + ); + if (updated) { + applied.push(updated); + } + } catch (err) { + const error = err as Error; + this.logger.error( + `Batch ${batchId}: forward execution failed at order ${item.orderInBatch} (${item.id}): ${error.message}`, + ); + + const compensation = await this.runCompensation(item, connection, connectionType); + const composedError = compensation.rollbackError + ? `${error.message} | self auto-rollback failed: ${compensation.rollbackError}` + : error.message; + + await this._dbContext.tableSchemaChangeRepository.updateStatus(item.id, SchemaChangeStatusEnum.FAILED, { + executionError: composedError, + autoRollbackAttempted: !!item.rollbackSql, + autoRollbackSucceeded: compensation.autoRollbackSucceeded, + }); + + const cascadeOutcome = await this.rollbackApplied(applied, connection, connectionType); + const cascadeSummary = cascadeOutcome.successIds.length + ? ` Rolled back ${cascadeOutcome.successIds.length} earlier item(s): ${cascadeOutcome.successIds.join(', ')}.` + : ''; + const cascadeFailureSummary = cascadeOutcome.failures.length + ? ` Cascade rollback failures: ${cascadeOutcome.failures.map((f) => `${f.changeId}=${f.error}`).join('; ')}.` + : ''; + + throw new BadRequestException({ + message: `Batch ${batchId} failed at order ${item.orderInBatch} (changeId=${item.id}): ${error.message}.${cascadeSummary}${cascadeFailureSummary}`, + type: cascadeOutcome.failures.length ? 'batch_partial_failure' : 'batch_failed_with_cascade_rollback', + }); + } + } + + const refreshed = await this._dbContext.tableSchemaChangeRepository.findByBatchId(batchId); + return { + batchId, + changes: refreshed.map(mapSchemaChangeToResponseDto), + }; + } + + private async runCompensation( + item: TableSchemaChangeEntity, + connection: Parameters[0]['connection'], + connectionType: ConnectionTypesEnum, + ): Promise<{ autoRollbackSucceeded: boolean; rollbackError: string | null }> { + if (!item.rollbackSql) { + return { autoRollbackSucceeded: false, rollbackError: null }; + } + try { + await executeSchemaChange({ + connection, + connectionType, + changeType: item.changeType, + targetTableName: item.targetTableName, + sql: item.rollbackSql, + allowAnyOperation: true, + }); + return { autoRollbackSucceeded: true, rollbackError: null }; + } catch (rbErr) { + const message = (rbErr as Error).message; + this.logger.error(`Auto-rollback failed for change ${item.id}: ${message}`); + return { autoRollbackSucceeded: false, rollbackError: message }; + } + } + + private async rollbackApplied( + applied: TableSchemaChangeEntity[], + connection: Parameters[0]['connection'], + connectionType: ConnectionTypesEnum, + ): Promise<{ successIds: string[]; failures: Array<{ changeId: string; error: string }> }> { + const successIds: string[] = []; + const failures: Array<{ changeId: string; error: string }> = []; + for (let i = applied.length - 1; i >= 0; i--) { + const item = applied[i]; + if (!item.rollbackSql) { + failures.push({ changeId: item.id, error: 'no rollback SQL recorded' }); + continue; + } + try { + await executeSchemaChange({ + connection, + connectionType, + changeType: item.changeType, + targetTableName: item.targetTableName, + sql: item.rollbackSql, + allowAnyOperation: true, + }); + await this._dbContext.tableSchemaChangeRepository.updateStatus(item.id, SchemaChangeStatusEnum.ROLLED_BACK, { + rolledBackAt: new Date(), + }); + successIds.push(item.id); + } catch (rbErr) { + const message = (rbErr as Error).message; + this.logger.error(`Cascade rollback failed for change ${item.id}: ${message}`); + failures.push({ changeId: item.id, error: message }); + } + } + return { successIds, failures }; + } +} diff --git a/backend/src/entities/table-schema/use-cases/generate-schema-change.use-case.ts b/backend/src/entities/table-schema/use-cases/generate-schema-change.use-case.ts new file mode 100644 index 000000000..d1b697a5d --- /dev/null +++ b/backend/src/entities/table-schema/use-cases/generate-schema-change.use-case.ts @@ -0,0 +1,218 @@ +import { BadRequestException, Inject, Injectable, Logger, NotFoundException, Scope } from '@nestjs/common'; +import { getDataAccessObject } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/create-data-access-object.js'; +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; +import crypto from 'crypto'; +import { AICoreService, AIProviderType, MessageBuilder } from '../../../ai-core/index.js'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { Messages } from '../../../exceptions/text/messages.js'; +import { runSchemaChangeAiLoop } from '../ai/run-schema-change-ai-loop.js'; +import { buildSchemaChangePrompt } from '../ai/schema-change-prompts.js'; +import { + createDynamoDbSchemaChangeTools, + createElasticsearchSchemaChangeTools, + createMongoSchemaChangeTools, + createSchemaChangeTools, + ProposeSchemaChangeArgs, +} from '../ai/schema-change-tools.js'; +import { GenerateSchemaChangeDs } from '../application/data-structures/generate-schema-change.ds.js'; +import { SchemaChangeBatchResponseDto } from '../application/data-transfer-objects/schema-change-batch-response.dto.js'; +import { TableSchemaChangeEntity } from '../table-schema-change.entity.js'; +import { + isDynamoDbSchemaChangeType, + isElasticsearchSchemaChangeType, + isMongoSchemaChangeType, + SchemaChangeStatusEnum, + SchemaChangeTypeEnum, +} from '../table-schema-change-enums.js'; +import { + assertDialectSupported, + isDynamoDbDialect, + isElasticsearchDialect, + isMongoDialect, +} from '../utils/assert-dialect-supported.js'; +import { validateProposedDynamoDbOp } from '../utils/dynamodb-schema-op.js'; +import { validateProposedElasticsearchOp } from '../utils/elasticsearch-schema-op.js'; +import { mapSchemaChangeToResponseDto } from '../utils/map-schema-change-to-response-dto.js'; +import { validateProposedMongoOp } from '../utils/mongo-schema-op.js'; +import { validateProposedDdl } from '../utils/validate-proposed-ddl.js'; +import { IGenerateSchemaChange } from './table-schema-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class GenerateSchemaChangeUseCase + extends AbstractUseCase + implements IGenerateSchemaChange +{ + private readonly logger = new Logger(GenerateSchemaChangeUseCase.name); + private readonly provider: AIProviderType = AIProviderType.BEDROCK; + + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + private readonly aiCoreService: AICoreService, + ) { + super(); + } + + protected async implementation(inputData: GenerateSchemaChangeDs): Promise { + const { connectionId, userPrompt, userId, masterPassword } = inputData; + + const connection = await this._dbContext.connectionRepository.findAndDecryptConnection( + connectionId, + masterPassword, + ); + if (!connection) { + throw new NotFoundException(Messages.CONNECTION_NOT_FOUND); + } + + const connectionType = connection.type as ConnectionTypesEnum; + assertDialectSupported(connectionType); + + const connectionProperties = + await this._dbContext.connectionPropertiesRepository.findConnectionProperties(connectionId); + if (connectionProperties && !connectionProperties.allow_ai_requests) { + throw new BadRequestException(Messages.AI_REQUESTS_NOT_ALLOWED); + } + + const dao = getDataAccessObject(connection); + const tableList = await dao.getTablesFromDB(); + const tableNames = tableList.map((t) => t.tableName); + + const systemPrompt = buildSchemaChangePrompt(connectionType, tableNames, connection.schema ?? null); + const messages = new MessageBuilder().system(systemPrompt).human(userPrompt).build(); + const tools = isMongoDialect(connectionType) + ? createMongoSchemaChangeTools() + : isDynamoDbDialect(connectionType) + ? createDynamoDbSchemaChangeTools() + : isElasticsearchDialect(connectionType) + ? createElasticsearchSchemaChangeTools() + : createSchemaChangeTools(); + + let proposals: ProposeSchemaChangeArgs[]; + try { + const result = await runSchemaChangeAiLoop({ + aiCoreService: this.aiCoreService, + provider: this.provider, + messages, + tools, + dao, + userEmail: undefined, + logger: this.logger, + }); + proposals = result.proposals; + } catch (err) { + this.logger.error(`AI loop failed: ${(err as Error).message}`); + throw new BadRequestException(`AI generation failed: ${(err as Error).message}`); + } + + for (let i = 0; i < proposals.length; i++) { + this.validateProposal(proposals[i], connectionType, i); + } + + const latestApplied = await this._dbContext.tableSchemaChangeRepository.findLatestAppliedChange(connectionId); + const previousChangeId = latestApplied?.id ?? null; + const aiModelUsed = + this.aiCoreService.getAvailableProviders().find((p) => p.type === this.provider)?.defaultModel ?? null; + + const batchId = crypto.randomUUID(); + const items: Partial[] = proposals.map((proposal, index) => ({ + connectionId, + batchId, + orderInBatch: index, + authorId: userId, + previousChangeId, + forwardSql: proposal.forwardSql, + rollbackSql: proposal.rollbackSql ?? null, + userModifiedSql: null, + status: SchemaChangeStatusEnum.PENDING, + changeType: proposal.changeType, + targetTableName: proposal.targetTableName, + databaseType: connectionType, + isReversible: !!proposal.isReversible, + userPrompt, + aiSummary: proposal.summary ?? null, + aiReasoning: proposal.reasoning ?? null, + aiModelUsed, + })); + + const saved = await this._dbContext.tableSchemaChangeRepository.createPendingBatch(items); + saved.sort((a, b) => a.orderInBatch - b.orderInBatch); + + return { + batchId, + changes: saved.map(mapSchemaChangeToResponseDto), + }; + } + + private validateProposal( + proposal: ProposeSchemaChangeArgs, + connectionType: ConnectionTypesEnum, + index: number, + ): void { + const fieldHint = `proposals[${index}]`; + try { + if (isMongoSchemaChangeType(proposal.changeType)) { + validateProposedMongoOp({ + opJson: proposal.forwardSql, + changeType: proposal.changeType, + targetTableName: proposal.targetTableName, + }); + if (proposal.rollbackSql) { + validateProposedMongoOp({ + opJson: proposal.rollbackSql, + changeType: proposal.changeType, + targetTableName: proposal.targetTableName, + allowAnyOperation: true, + }); + } + } else if (isDynamoDbSchemaChangeType(proposal.changeType)) { + validateProposedDynamoDbOp({ + opJson: proposal.forwardSql, + changeType: proposal.changeType, + targetTableName: proposal.targetTableName, + }); + if (proposal.rollbackSql) { + validateProposedDynamoDbOp({ + opJson: proposal.rollbackSql, + changeType: proposal.changeType, + targetTableName: proposal.targetTableName, + allowAnyOperation: true, + }); + } + } else if (isElasticsearchSchemaChangeType(proposal.changeType)) { + validateProposedElasticsearchOp({ + opJson: proposal.forwardSql, + changeType: proposal.changeType, + targetTableName: proposal.targetTableName, + }); + if (proposal.rollbackSql) { + validateProposedElasticsearchOp({ + opJson: proposal.rollbackSql, + changeType: proposal.changeType, + targetTableName: proposal.targetTableName, + allowAnyOperation: true, + }); + } + } else { + validateProposedDdl({ + sql: proposal.forwardSql, + connectionType, + changeType: proposal.changeType, + targetTableName: proposal.targetTableName, + }); + if (proposal.rollbackSql) { + validateProposedDdl({ + sql: proposal.rollbackSql, + connectionType, + changeType: SchemaChangeTypeEnum.ROLLBACK, + targetTableName: proposal.targetTableName, + }); + } + } + } catch (err) { + const message = (err as Error).message ?? 'validation error'; + throw new BadRequestException(`${fieldHint}: ${message}`); + } + } +} diff --git a/backend/src/entities/table-schema/use-cases/get-batch-schema-changes.use-case.ts b/backend/src/entities/table-schema/use-cases/get-batch-schema-changes.use-case.ts new file mode 100644 index 000000000..caff3ed01 --- /dev/null +++ b/backend/src/entities/table-schema/use-cases/get-batch-schema-changes.use-case.ts @@ -0,0 +1,32 @@ +import { Inject, Injectable, NotFoundException, Scope } from '@nestjs/common'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { GetBatchSchemaChangeDs } from '../application/data-structures/get-batch-schema-change.ds.js'; +import { SchemaChangeBatchResponseDto } from '../application/data-transfer-objects/schema-change-batch-response.dto.js'; +import { mapSchemaChangeToResponseDto } from '../utils/map-schema-change-to-response-dto.js'; +import { IGetBatchSchemaChange } from './table-schema-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class GetBatchSchemaChangesUseCase + extends AbstractUseCase + implements IGetBatchSchemaChange +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + ) { + super(); + } + + protected async implementation(inputData: GetBatchSchemaChangeDs): Promise { + const items = await this._dbContext.tableSchemaChangeRepository.findByBatchId(inputData.batchId); + if (items.length === 0) { + throw new NotFoundException('Schema change batch not found.'); + } + return { + batchId: inputData.batchId, + changes: items.map(mapSchemaChangeToResponseDto), + }; + } +} diff --git a/backend/src/entities/table-schema/use-cases/get-schema-change.use-case.ts b/backend/src/entities/table-schema/use-cases/get-schema-change.use-case.ts new file mode 100644 index 000000000..79e310ad6 --- /dev/null +++ b/backend/src/entities/table-schema/use-cases/get-schema-change.use-case.ts @@ -0,0 +1,30 @@ +import { Inject, Injectable, NotFoundException, Scope } from '@nestjs/common'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { GetSchemaChangeDs } from '../application/data-structures/get-schema-change.ds.js'; +import { SchemaChangeResponseDto } from '../application/data-transfer-objects/schema-change-response.dto.js'; +import { mapSchemaChangeToResponseDto } from '../utils/map-schema-change-to-response-dto.js'; +import { IGetSchemaChange } from './table-schema-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class GetSchemaChangeUseCase + extends AbstractUseCase + implements IGetSchemaChange +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + ) { + super(); + } + + protected async implementation(inputData: GetSchemaChangeDs): Promise { + const { changeId } = inputData; + const change = await this._dbContext.tableSchemaChangeRepository.findByIdWithRelations(changeId); + if (!change) { + throw new NotFoundException('Schema change not found.'); + } + return mapSchemaChangeToResponseDto(change); + } +} diff --git a/backend/src/entities/table-schema/use-cases/list-schema-changes.use-case.ts b/backend/src/entities/table-schema/use-cases/list-schema-changes.use-case.ts new file mode 100644 index 000000000..50c6c7dd6 --- /dev/null +++ b/backend/src/entities/table-schema/use-cases/list-schema-changes.use-case.ts @@ -0,0 +1,39 @@ +import { Inject, Injectable, Scope } from '@nestjs/common'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { ListSchemaChangesDs } from '../application/data-structures/list-schema-changes.ds.js'; +import { SchemaChangeListResponseDto } from '../application/data-transfer-objects/schema-change-list-response.dto.js'; +import { mapSchemaChangeToResponseDto } from '../utils/map-schema-change-to-response-dto.js'; +import { IListSchemaChanges } from './table-schema-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class ListSchemaChangesUseCase + extends AbstractUseCase + implements IListSchemaChanges +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + ) { + super(); + } + + protected async implementation(inputData: ListSchemaChangesDs): Promise { + const { connectionId, limit, offset } = inputData; + + const [items, total] = await this._dbContext.tableSchemaChangeRepository.findByConnectionPaginated(connectionId, { + limit, + offset, + }); + + const perPage = limit > 0 ? limit : 1; + const currentPage = Math.floor(offset / perPage) + 1; + const lastPage = Math.max(1, Math.ceil(total / perPage)); + + return { + data: items.map(mapSchemaChangeToResponseDto), + pagination: { currentPage, lastPage, perPage, total }, + }; + } +} diff --git a/backend/src/entities/table-schema/use-cases/reject-batch-schema-changes.use-case.ts b/backend/src/entities/table-schema/use-cases/reject-batch-schema-changes.use-case.ts new file mode 100644 index 000000000..f22450e15 --- /dev/null +++ b/backend/src/entities/table-schema/use-cases/reject-batch-schema-changes.use-case.ts @@ -0,0 +1,48 @@ +import { ConflictException, Inject, Injectable, NotFoundException, Scope } from '@nestjs/common'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { RejectBatchSchemaChangeDs } from '../application/data-structures/reject-batch-schema-change.ds.js'; +import { SchemaChangeBatchResponseDto } from '../application/data-transfer-objects/schema-change-batch-response.dto.js'; +import { SchemaChangeStatusEnum } from '../table-schema-change-enums.js'; +import { mapSchemaChangeToResponseDto } from '../utils/map-schema-change-to-response-dto.js'; +import { IRejectBatchSchemaChange } from './table-schema-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class RejectBatchSchemaChangesUseCase + extends AbstractUseCase + implements IRejectBatchSchemaChange +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + ) { + super(); + } + + protected async implementation(inputData: RejectBatchSchemaChangeDs): Promise { + const { batchId } = inputData; + + const items = await this._dbContext.tableSchemaChangeRepository.findByBatchId(batchId); + if (items.length === 0) { + throw new NotFoundException('Schema change batch not found.'); + } + + const rejectable = items.filter( + (it) => it.status === SchemaChangeStatusEnum.PENDING || it.status === SchemaChangeStatusEnum.APPROVED, + ); + if (rejectable.length === 0) { + throw new ConflictException('No PENDING or APPROVED items in this batch can be rejected.'); + } + + for (const item of rejectable) { + await this._dbContext.tableSchemaChangeRepository.updateStatus(item.id, SchemaChangeStatusEnum.REJECTED); + } + + const refreshed = await this._dbContext.tableSchemaChangeRepository.findByBatchId(batchId); + return { + batchId, + changes: refreshed.map(mapSchemaChangeToResponseDto), + }; + } +} diff --git a/backend/src/entities/table-schema/use-cases/reject-schema-change.use-case.ts b/backend/src/entities/table-schema/use-cases/reject-schema-change.use-case.ts new file mode 100644 index 000000000..adc0380e1 --- /dev/null +++ b/backend/src/entities/table-schema/use-cases/reject-schema-change.use-case.ts @@ -0,0 +1,41 @@ +import { ConflictException, Inject, Injectable, NotFoundException, Scope } from '@nestjs/common'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { RejectSchemaChangeDs } from '../application/data-structures/reject-schema-change.ds.js'; +import { SchemaChangeResponseDto } from '../application/data-transfer-objects/schema-change-response.dto.js'; +import { SchemaChangeStatusEnum } from '../table-schema-change-enums.js'; +import { mapSchemaChangeToResponseDto } from '../utils/map-schema-change-to-response-dto.js'; +import { IRejectSchemaChange } from './table-schema-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class RejectSchemaChangeUseCase + extends AbstractUseCase + implements IRejectSchemaChange +{ + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + ) { + super(); + } + + protected async implementation(inputData: RejectSchemaChangeDs): Promise { + const { changeId } = inputData; + + const change = await this._dbContext.tableSchemaChangeRepository.findByIdWithRelations(changeId); + if (!change) { + throw new NotFoundException('Schema change not found.'); + } + + if (change.status !== SchemaChangeStatusEnum.PENDING) { + throw new ConflictException(`Schema change is ${change.status}; only PENDING changes can be rejected.`); + } + + const updated = await this._dbContext.tableSchemaChangeRepository.updateStatus( + change.id, + SchemaChangeStatusEnum.REJECTED, + ); + return mapSchemaChangeToResponseDto(updated!); + } +} diff --git a/backend/src/entities/table-schema/use-cases/rollback-batch-schema-changes.use-case.ts b/backend/src/entities/table-schema/use-cases/rollback-batch-schema-changes.use-case.ts new file mode 100644 index 000000000..efe4edc80 --- /dev/null +++ b/backend/src/entities/table-schema/use-cases/rollback-batch-schema-changes.use-case.ts @@ -0,0 +1,124 @@ +import { + BadRequestException, + ConflictException, + Inject, + Injectable, + Logger, + NotFoundException, + Scope, +} from '@nestjs/common'; +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { Messages } from '../../../exceptions/text/messages.js'; +import { RollbackBatchSchemaChangeDs } from '../application/data-structures/rollback-batch-schema-change.ds.js'; +import { SchemaChangeBatchResponseDto } from '../application/data-transfer-objects/schema-change-batch-response.dto.js'; +import { SchemaChangeStatusEnum, SchemaChangeTypeEnum } from '../table-schema-change-enums.js'; +import { assertDialectSupported } from '../utils/assert-dialect-supported.js'; +import { executeSchemaChange } from '../utils/execute-schema-change.js'; +import { mapSchemaChangeToResponseDto } from '../utils/map-schema-change-to-response-dto.js'; +import { IRollbackBatchSchemaChange } from './table-schema-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class RollbackBatchSchemaChangesUseCase + extends AbstractUseCase + implements IRollbackBatchSchemaChange +{ + private readonly logger = new Logger(RollbackBatchSchemaChangesUseCase.name); + + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + ) { + super(); + } + + protected async implementation(inputData: RollbackBatchSchemaChangeDs): Promise { + const { batchId, userId, masterPassword } = inputData; + + const items = await this._dbContext.tableSchemaChangeRepository.findByBatchId(batchId); + if (items.length === 0) { + throw new NotFoundException('Schema change batch not found.'); + } + + const applied = items.filter((it) => it.status === SchemaChangeStatusEnum.APPLIED); + if (applied.length === 0) { + throw new ConflictException('No APPLIED items in this batch can be rolled back.'); + } + + const missingRollback = applied.filter((it) => !it.rollbackSql); + if (missingRollback.length > 0) { + throw new BadRequestException({ + message: `One or more applied changes in this batch have no rollback SQL and cannot be reverted (ids: ${missingRollback.map((it) => it.id).join(', ')}).`, + type: 'batch_rollback_unavailable', + }); + } + + const connectionType = items[0].databaseType as ConnectionTypesEnum; + assertDialectSupported(connectionType); + + const connection = await this._dbContext.connectionRepository.findAndDecryptConnection( + items[0].connectionId, + masterPassword, + ); + if (!connection) { + throw new NotFoundException(Messages.CONNECTION_NOT_FOUND); + } + + const failures: Array<{ changeId: string; error: string }> = []; + + for (let i = applied.length - 1; i >= 0; i--) { + const item = applied[i]; + try { + await executeSchemaChange({ + connection, + connectionType, + changeType: item.changeType, + targetTableName: item.targetTableName, + sql: item.rollbackSql!, + allowAnyOperation: true, + }); + await this._dbContext.tableSchemaChangeRepository.updateStatus(item.id, SchemaChangeStatusEnum.ROLLED_BACK, { + rolledBackAt: new Date(), + }); + await this._dbContext.tableSchemaChangeRepository.createPendingChange({ + connectionId: item.connectionId, + authorId: userId, + previousChangeId: item.id, + forwardSql: item.rollbackSql!, + rollbackSql: item.forwardSql, + userModifiedSql: null, + status: SchemaChangeStatusEnum.APPLIED, + changeType: SchemaChangeTypeEnum.ROLLBACK, + targetTableName: item.targetTableName, + databaseType: item.databaseType, + isReversible: true, + userPrompt: `Manual rollback of batch ${batchId} (change ${item.id})`, + aiSummary: `Rollback of "${item.aiSummary ?? item.id}"`, + aiReasoning: null, + aiModelUsed: null, + appliedAt: new Date(), + }); + } catch (err) { + const message = (err as Error).message; + this.logger.error(`Batch ${batchId}: rollback failed for change ${item.id}: ${message}`); + failures.push({ changeId: item.id, error: message }); + } + } + + if (failures.length > 0) { + const detail = failures.map((f) => `${f.changeId}=${f.error}`).join('; '); + throw new BadRequestException({ + message: `Batch rollback partially failed: ${failures.length} of ${applied.length} items could not be rolled back. ${detail}`, + type: 'batch_rollback_partial_failure', + }); + } + + const refreshed = await this._dbContext.tableSchemaChangeRepository.findByBatchId(batchId); + return { + batchId, + changes: refreshed.map(mapSchemaChangeToResponseDto), + }; + } +} diff --git a/backend/src/entities/table-schema/use-cases/rollback-schema-change.use-case.ts b/backend/src/entities/table-schema/use-cases/rollback-schema-change.use-case.ts new file mode 100644 index 000000000..61fe4cd4a --- /dev/null +++ b/backend/src/entities/table-schema/use-cases/rollback-schema-change.use-case.ts @@ -0,0 +1,106 @@ +import { + BadRequestException, + ConflictException, + Inject, + Injectable, + Logger, + NotFoundException, + Scope, +} from '@nestjs/common'; +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; +import AbstractUseCase from '../../../common/abstract-use.case.js'; +import { IGlobalDatabaseContext } from '../../../common/application/global-database-context.interface.js'; +import { BaseType } from '../../../common/data-injection.tokens.js'; +import { Messages } from '../../../exceptions/text/messages.js'; +import { RollbackSchemaChangeDs } from '../application/data-structures/rollback-schema-change.ds.js'; +import { SchemaChangeResponseDto } from '../application/data-transfer-objects/schema-change-response.dto.js'; +import { SchemaChangeStatusEnum, SchemaChangeTypeEnum } from '../table-schema-change-enums.js'; +import { assertDialectSupported } from '../utils/assert-dialect-supported.js'; +import { executeSchemaChange } from '../utils/execute-schema-change.js'; +import { mapSchemaChangeToResponseDto } from '../utils/map-schema-change-to-response-dto.js'; +import { IRollbackSchemaChange } from './table-schema-use-cases.interface.js'; + +@Injectable({ scope: Scope.REQUEST }) +export class RollbackSchemaChangeUseCase + extends AbstractUseCase + implements IRollbackSchemaChange +{ + private readonly logger = new Logger(RollbackSchemaChangeUseCase.name); + + constructor( + @Inject(BaseType.GLOBAL_DB_CONTEXT) + protected _dbContext: IGlobalDatabaseContext, + ) { + super(); + } + + protected async implementation(inputData: RollbackSchemaChangeDs): Promise { + const { changeId, userId, masterPassword } = inputData; + + const change = await this._dbContext.tableSchemaChangeRepository.findByIdWithRelations(changeId); + if (!change) { + throw new NotFoundException('Schema change not found.'); + } + + if (change.status !== SchemaChangeStatusEnum.APPLIED) { + throw new ConflictException(`Only APPLIED changes can be rolled back; this change is ${change.status}.`); + } + + if (!change.rollbackSql) { + throw new BadRequestException('This change has no rollback SQL and cannot be reverted.'); + } + + const connectionType = change.databaseType as ConnectionTypesEnum; + assertDialectSupported(connectionType); + + const connection = await this._dbContext.connectionRepository.findAndDecryptConnection( + change.connectionId, + masterPassword, + ); + if (!connection) { + throw new NotFoundException(Messages.CONNECTION_NOT_FOUND); + } + + try { + await executeSchemaChange({ + connection, + connectionType, + changeType: change.changeType, + targetTableName: change.targetTableName, + sql: change.rollbackSql, + allowAnyOperation: true, + }); + } catch (err) { + const error = err as Error; + this.logger.error(`Rollback failed for change ${change.id}: ${error.message}`); + throw new BadRequestException(`Rollback execution failed: ${error.message}`); + } + + const updated = await this._dbContext.tableSchemaChangeRepository.updateStatus( + change.id, + SchemaChangeStatusEnum.ROLLED_BACK, + { rolledBackAt: new Date() }, + ); + + await this._dbContext.tableSchemaChangeRepository.createPendingChange({ + connectionId: change.connectionId, + authorId: userId, + previousChangeId: change.id, + forwardSql: change.rollbackSql, + rollbackSql: change.forwardSql, + userModifiedSql: null, + status: SchemaChangeStatusEnum.APPLIED, + changeType: SchemaChangeTypeEnum.ROLLBACK, + targetTableName: change.targetTableName, + databaseType: change.databaseType, + isReversible: true, + userPrompt: `Manual rollback of change ${change.id}`, + aiSummary: `Rollback of "${change.aiSummary ?? change.id}"`, + aiReasoning: null, + aiModelUsed: null, + appliedAt: new Date(), + }); + + return mapSchemaChangeToResponseDto(updated!); + } +} diff --git a/backend/src/entities/table-schema/use-cases/table-schema-use-cases.interface.ts b/backend/src/entities/table-schema/use-cases/table-schema-use-cases.interface.ts new file mode 100644 index 000000000..ac122a889 --- /dev/null +++ b/backend/src/entities/table-schema/use-cases/table-schema-use-cases.interface.ts @@ -0,0 +1,53 @@ +import { ApproveBatchSchemaChangeDs } from '../application/data-structures/approve-batch-schema-change.ds.js'; +import { ApproveSchemaChangeDs } from '../application/data-structures/approve-schema-change.ds.js'; +import { GenerateSchemaChangeDs } from '../application/data-structures/generate-schema-change.ds.js'; +import { GetBatchSchemaChangeDs } from '../application/data-structures/get-batch-schema-change.ds.js'; +import { GetSchemaChangeDs } from '../application/data-structures/get-schema-change.ds.js'; +import { ListSchemaChangesDs } from '../application/data-structures/list-schema-changes.ds.js'; +import { RejectBatchSchemaChangeDs } from '../application/data-structures/reject-batch-schema-change.ds.js'; +import { RejectSchemaChangeDs } from '../application/data-structures/reject-schema-change.ds.js'; +import { RollbackBatchSchemaChangeDs } from '../application/data-structures/rollback-batch-schema-change.ds.js'; +import { RollbackSchemaChangeDs } from '../application/data-structures/rollback-schema-change.ds.js'; +import { SchemaChangeBatchResponseDto } from '../application/data-transfer-objects/schema-change-batch-response.dto.js'; +import { SchemaChangeListResponseDto } from '../application/data-transfer-objects/schema-change-list-response.dto.js'; +import { SchemaChangeResponseDto } from '../application/data-transfer-objects/schema-change-response.dto.js'; + +export interface IGenerateSchemaChange { + execute(inputData: GenerateSchemaChangeDs): Promise; +} + +export interface IApproveSchemaChange { + execute(inputData: ApproveSchemaChangeDs): Promise; +} + +export interface IRejectSchemaChange { + execute(inputData: RejectSchemaChangeDs): Promise; +} + +export interface IRollbackSchemaChange { + execute(inputData: RollbackSchemaChangeDs): Promise; +} + +export interface IListSchemaChanges { + execute(inputData: ListSchemaChangesDs): Promise; +} + +export interface IGetSchemaChange { + execute(inputData: GetSchemaChangeDs): Promise; +} + +export interface IApproveBatchSchemaChange { + execute(inputData: ApproveBatchSchemaChangeDs): Promise; +} + +export interface IRejectBatchSchemaChange { + execute(inputData: RejectBatchSchemaChangeDs): Promise; +} + +export interface IRollbackBatchSchemaChange { + execute(inputData: RollbackBatchSchemaChangeDs): Promise; +} + +export interface IGetBatchSchemaChange { + execute(inputData: GetBatchSchemaChangeDs): Promise; +} diff --git a/backend/src/entities/table-schema/utils/assert-dialect-supported.ts b/backend/src/entities/table-schema/utils/assert-dialect-supported.ts new file mode 100644 index 000000000..11d5a47c7 --- /dev/null +++ b/backend/src/entities/table-schema/utils/assert-dialect-supported.ts @@ -0,0 +1,66 @@ +import { BadRequestException } from '@nestjs/common'; +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; + +const SUPPORTED_DIALECTS: ReadonlySet = new Set([ + ConnectionTypesEnum.postgres, + ConnectionTypesEnum.mysql, + ConnectionTypesEnum.mysql2, + ConnectionTypesEnum.mssql, + ConnectionTypesEnum.oracledb, + ConnectionTypesEnum.ibmdb2, + ConnectionTypesEnum.mongodb, + ConnectionTypesEnum.clickhouse, + ConnectionTypesEnum.agent_clickhouse, + ConnectionTypesEnum.dynamodb, + ConnectionTypesEnum.cassandra, + ConnectionTypesEnum.agent_cassandra, + ConnectionTypesEnum.elasticsearch, +]); + +export function isDialectSupported(connectionType: ConnectionTypesEnum): boolean { + return SUPPORTED_DIALECTS.has(connectionType); +} + +export function assertDialectSupported(connectionType: ConnectionTypesEnum): void { + if (!isDialectSupported(connectionType)) { + throw new BadRequestException( + `Schema changes via AI are not yet supported for "${connectionType}". Supported: PostgreSQL, MySQL, Microsoft SQL Server, Oracle DB, IBM DB2, MongoDB, ClickHouse, DynamoDB, Cassandra, Elasticsearch.`, + ); + } +} + +export function isMongoDialect(connectionType: ConnectionTypesEnum): boolean { + return connectionType === ConnectionTypesEnum.mongodb || connectionType === ConnectionTypesEnum.agent_mongodb; +} + +export function isClickHouseDialect(connectionType: ConnectionTypesEnum): boolean { + return connectionType === ConnectionTypesEnum.clickhouse || connectionType === ConnectionTypesEnum.agent_clickhouse; +} + +export function isDynamoDbDialect(connectionType: ConnectionTypesEnum): boolean { + return connectionType === ConnectionTypesEnum.dynamodb; +} + +export function isCassandraDialect(connectionType: ConnectionTypesEnum): boolean { + return connectionType === ConnectionTypesEnum.cassandra || connectionType === ConnectionTypesEnum.agent_cassandra; +} + +export function isElasticsearchDialect(connectionType: ConnectionTypesEnum): boolean { + return connectionType === ConnectionTypesEnum.elasticsearch; +} + +const SQL_PARSER_DIALECTS: Record = { + [ConnectionTypesEnum.postgres]: 'PostgresQL', + [ConnectionTypesEnum.agent_postgres]: 'PostgresQL', + [ConnectionTypesEnum.mysql]: 'MySQL', + [ConnectionTypesEnum.mysql2]: 'MySQL', + [ConnectionTypesEnum.agent_mysql]: 'MySQL', + [ConnectionTypesEnum.mssql]: 'TransactSQL', + [ConnectionTypesEnum.agent_mssql]: 'TransactSQL', + [ConnectionTypesEnum.ibmdb2]: 'DB2', + [ConnectionTypesEnum.agent_ibmdb2]: 'DB2', +}; + +export function connectionTypeToParserDialect(connectionType: ConnectionTypesEnum): string { + return SQL_PARSER_DIALECTS[connectionType] ?? 'PostgresQL'; +} diff --git a/backend/src/entities/table-schema/utils/cassandra-ddl.ts b/backend/src/entities/table-schema/utils/cassandra-ddl.ts new file mode 100644 index 000000000..e1c8f57f7 --- /dev/null +++ b/backend/src/entities/table-schema/utils/cassandra-ddl.ts @@ -0,0 +1,47 @@ +import * as cassandra from 'cassandra-driver'; + +export interface CassandraExecutionConnection { + host?: string | null; + port?: number | null; + username?: string | null; + password?: string | null; + database?: string | null; + dataCenter?: string | null; + ssl?: boolean | null; + cert?: string | null; +} + +export async function executeCassandraDdl(connection: CassandraExecutionConnection, cql: string): Promise { + const host = connection.host ?? ''; + const port = connection.port ?? 9042; + const authProvider = new cassandra.auth.PlainTextAuthProvider( + connection.username ?? 'cassandra', + connection.password ?? 'cassandra', + ); + + const clientOptions: cassandra.ClientOptions = { + contactPoints: [host], + localDataCenter: connection.dataCenter || undefined, + keyspace: connection.database ?? undefined, + authProvider, + protocolOptions: { port }, + queryOptions: { + consistency: cassandra.types.consistencies.localQuorum, + }, + }; + + if (connection.ssl) { + clientOptions.sslOptions = { rejectUnauthorized: false }; + if (connection.cert) { + clientOptions.sslOptions.ca = [connection.cert]; + } + } + + const client = new cassandra.Client(clientOptions); + try { + await client.connect(); + await client.execute(cql); + } finally { + await client.shutdown().catch(() => undefined); + } +} diff --git a/backend/src/entities/table-schema/utils/clickhouse-ddl.ts b/backend/src/entities/table-schema/utils/clickhouse-ddl.ts new file mode 100644 index 000000000..338ed932e --- /dev/null +++ b/backend/src/entities/table-schema/utils/clickhouse-ddl.ts @@ -0,0 +1,36 @@ +import { createClient } from '@clickhouse/client'; +import { NodeClickHouseClientConfigOptions } from '@clickhouse/client/dist/config.js'; + +export interface ClickHouseExecutionConnection { + host?: string | null; + port?: number | null; + username?: string | null; + password?: string | null; + database?: string | null; + ssl?: boolean | null; + cert?: string | null; +} + +export async function executeClickHouseDdl(connection: ClickHouseExecutionConnection, sql: string): Promise { + const host = connection.host ?? ''; + const port = connection.port ?? 8123; + const protocol = connection.ssl ? 'https' : 'http'; + + const clientConfig: NodeClickHouseClientConfigOptions = { + url: `${protocol}://${host}:${port}`, + username: connection.username ?? 'default', + password: connection.password ?? '', + database: connection.database ?? 'default', + }; + + if (connection.ssl && connection.cert) { + clientConfig.tls = { ca_cert: Buffer.from(connection.cert) }; + } + + const client = createClient(clientConfig); + try { + await client.command({ query: sql }); + } finally { + await client.close(); + } +} diff --git a/backend/src/entities/table-schema/utils/dynamodb-schema-op.ts b/backend/src/entities/table-schema/utils/dynamodb-schema-op.ts new file mode 100644 index 000000000..0930ae30c --- /dev/null +++ b/backend/src/entities/table-schema/utils/dynamodb-schema-op.ts @@ -0,0 +1,609 @@ +import { + AttributeDefinition, + BillingMode, + CreateTableCommand, + DeleteTableCommand, + DescribeTableCommand, + DynamoDB, + GlobalSecondaryIndex, + GlobalSecondaryIndexUpdate, + KeySchemaElement, + KeyType, + LocalSecondaryIndex, + ProjectionType, + ProvisionedThroughput, + ResourceNotFoundException, + ScalarAttributeType, + SSESpecification, + SSEType, + StreamSpecification, + StreamViewType, + UpdateTableCommand, + UpdateTimeToLiveCommand, +} from '@aws-sdk/client-dynamodb'; +import { BadRequestException } from '@nestjs/common'; +import { SchemaChangeTypeEnum } from '../table-schema-change-enums.js'; + +export interface DynamoDbExecutionConnection { + host?: string | null; + username?: string | null; + password?: string | null; +} + +export type DynamoDbOperationKind = 'createTable' | 'deleteTable' | 'updateTable' | 'updateTimeToLive'; + +export interface DynamoDbSchemaOp { + operation: DynamoDbOperationKind; + tableName: string; + attributeDefinitions?: AttributeDefinition[]; + keySchema?: KeySchemaElement[]; + billingMode?: BillingMode; + provisionedThroughput?: ProvisionedThroughput; + globalSecondaryIndexes?: GlobalSecondaryIndex[]; + localSecondaryIndexes?: LocalSecondaryIndex[]; + globalSecondaryIndexUpdates?: GlobalSecondaryIndexUpdate[]; + streamSpecification?: StreamSpecification; + sseSpecification?: SSESpecification; + timeToLiveSpecification?: { enabled: boolean; attributeName: string }; +} + +const ALLOWED_OPERATIONS: ReadonlySet = new Set([ + 'createTable', + 'deleteTable', + 'updateTable', + 'updateTimeToLive', +]); + +const CHANGE_TYPE_TO_OPERATION: Record = { + [SchemaChangeTypeEnum.DYNAMODB_CREATE_TABLE]: 'createTable', + [SchemaChangeTypeEnum.DYNAMODB_DROP_TABLE]: 'deleteTable', + [SchemaChangeTypeEnum.DYNAMODB_UPDATE_TABLE]: 'updateTable', + [SchemaChangeTypeEnum.DYNAMODB_UPDATE_TTL]: 'updateTimeToLive', +}; + +const ALLOWED_ATTRIBUTE_TYPES: ReadonlySet = new Set(['S', 'N', 'B'] as ScalarAttributeType[]); +const ALLOWED_KEY_TYPES: ReadonlySet = new Set(['HASH', 'RANGE'] as KeyType[]); +const ALLOWED_BILLING_MODES: ReadonlySet = new Set(['PROVISIONED', 'PAY_PER_REQUEST'] as BillingMode[]); +const ALLOWED_PROJECTION_TYPES: ReadonlySet = new Set([ + 'ALL', + 'KEYS_ONLY', + 'INCLUDE', +] as ProjectionType[]); +const ALLOWED_STREAM_VIEW_TYPES: ReadonlySet = new Set([ + 'NEW_IMAGE', + 'OLD_IMAGE', + 'NEW_AND_OLD_IMAGES', + 'KEYS_ONLY', +] as StreamViewType[]); +const ALLOWED_SSE_TYPES: ReadonlySet = new Set(['AES256', 'KMS'] as SSEType[]); + +export interface ValidateDynamoDbOpInput { + opJson: string; + changeType: SchemaChangeTypeEnum; + targetTableName: string; + allowAnyOperation?: boolean; +} + +export function validateProposedDynamoDbOp(input: ValidateDynamoDbOpInput): DynamoDbSchemaOp { + const { opJson, changeType, targetTableName, allowAnyOperation } = input; + + if (!opJson || opJson.trim().length === 0) { + throw new BadRequestException('Proposed DynamoDB operation is empty.'); + } + + let parsed: unknown; + try { + parsed = JSON.parse(opJson); + } catch (err) { + throw new BadRequestException(`Proposed DynamoDB operation is not valid JSON: ${(err as Error).message}`); + } + + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new BadRequestException('Proposed DynamoDB operation must be a single JSON object.'); + } + + const op = parsed as Record; + const operation = op.operation; + if (typeof operation !== 'string' || !ALLOWED_OPERATIONS.has(operation as DynamoDbOperationKind)) { + throw new BadRequestException( + `Proposed DynamoDB operation "${String(operation)}" is not one of ${Array.from(ALLOWED_OPERATIONS).join(', ')}.`, + ); + } + + const tableName = op.tableName; + if (typeof tableName !== 'string' || tableName.trim().length === 0) { + throw new BadRequestException('Proposed DynamoDB operation must include a non-empty "tableName".'); + } + if (tableName !== targetTableName) { + throw new BadRequestException( + `Proposed DynamoDB operation targets table "${tableName}" which does not match declared targetTableName "${targetTableName}".`, + ); + } + + if (!allowAnyOperation) { + const expected = CHANGE_TYPE_TO_OPERATION[changeType]; + if (expected && expected !== operation) { + throw new BadRequestException( + `Proposed DynamoDB operation "${operation}" does not match declared changeType "${changeType}" (expected "${expected}").`, + ); + } + } + + const typed: DynamoDbSchemaOp = { operation: operation as DynamoDbOperationKind, tableName }; + + switch (operation) { + case 'createTable': { + typed.attributeDefinitions = parseAttributeDefinitions(op.attributeDefinitions, true); + typed.keySchema = parseKeySchema(op.keySchema, true); + ensureAttributesCoverKeys(typed.attributeDefinitions, typed.keySchema); + typed.billingMode = parseBillingMode(op.billingMode) ?? 'PAY_PER_REQUEST'; + typed.provisionedThroughput = parseProvisionedThroughput(op.provisionedThroughput); + typed.globalSecondaryIndexes = parseGsiList(op.globalSecondaryIndexes); + typed.localSecondaryIndexes = parseLsiList(op.localSecondaryIndexes); + typed.streamSpecification = parseStreamSpecification(op.streamSpecification); + typed.sseSpecification = parseSseSpecification(op.sseSpecification); + if (typed.billingMode === 'PROVISIONED' && !typed.provisionedThroughput) { + throw new BadRequestException( + 'createTable with billingMode=PROVISIONED requires provisionedThroughput { readCapacityUnits, writeCapacityUnits }.', + ); + } + break; + } + case 'deleteTable': { + break; + } + case 'updateTable': { + typed.attributeDefinitions = parseAttributeDefinitions(op.attributeDefinitions, false); + typed.billingMode = parseBillingMode(op.billingMode); + typed.provisionedThroughput = parseProvisionedThroughput(op.provisionedThroughput); + typed.globalSecondaryIndexUpdates = parseGsiUpdates(op.globalSecondaryIndexUpdates); + typed.streamSpecification = parseStreamSpecification(op.streamSpecification); + const hasAnyField = + (typed.attributeDefinitions && typed.attributeDefinitions.length > 0) || + typed.billingMode !== undefined || + typed.provisionedThroughput !== undefined || + (typed.globalSecondaryIndexUpdates && typed.globalSecondaryIndexUpdates.length > 0) || + typed.streamSpecification !== undefined; + if (!hasAnyField) { + throw new BadRequestException( + 'updateTable must change at least one of: attributeDefinitions, billingMode, provisionedThroughput, globalSecondaryIndexUpdates, streamSpecification.', + ); + } + break; + } + case 'updateTimeToLive': { + const ttl = op.timeToLiveSpecification; + if (!ttl || typeof ttl !== 'object' || Array.isArray(ttl)) { + throw new BadRequestException( + 'updateTimeToLive requires timeToLiveSpecification { enabled: boolean, attributeName: string }.', + ); + } + const ttlObj = ttl as Record; + if (typeof ttlObj.enabled !== 'boolean') { + throw new BadRequestException('timeToLiveSpecification.enabled must be a boolean.'); + } + if (typeof ttlObj.attributeName !== 'string' || ttlObj.attributeName.trim().length === 0) { + throw new BadRequestException('timeToLiveSpecification.attributeName must be a non-empty string.'); + } + typed.timeToLiveSpecification = { enabled: ttlObj.enabled, attributeName: ttlObj.attributeName }; + break; + } + } + + return typed; +} + +export async function executeDynamoDbSchemaOp( + connection: DynamoDbExecutionConnection, + op: DynamoDbSchemaOp, +): Promise { + const client = buildDynamoDbClient(connection); + try { + await dispatchDynamoDbOp(client, op); + } finally { + client.destroy(); + } +} + +function buildDynamoDbClient(connection: DynamoDbExecutionConnection): DynamoDB { + const endpoint = connection.host ?? ''; + const regionMatch = endpoint.match(/dynamodb\.(.+?)\.amazonaws\.com/); + const region = regionMatch ? regionMatch[1] : 'us-east-1'; + return new DynamoDB({ + endpoint, + region: process.env.NODE_ENV === 'test' ? 'localhost' : region, + credentials: { + accessKeyId: connection.username ?? '', + secretAccessKey: connection.password ?? '', + }, + }); +} + +async function dispatchDynamoDbOp(client: DynamoDB, op: DynamoDbSchemaOp): Promise { + switch (op.operation) { + case 'createTable': { + await client.send( + new CreateTableCommand({ + TableName: op.tableName, + AttributeDefinitions: op.attributeDefinitions, + KeySchema: op.keySchema, + BillingMode: op.billingMode, + ProvisionedThroughput: op.provisionedThroughput, + GlobalSecondaryIndexes: op.globalSecondaryIndexes, + LocalSecondaryIndexes: op.localSecondaryIndexes, + StreamSpecification: op.streamSpecification, + SSESpecification: op.sseSpecification, + }), + ); + await waitForTableActive(client, op.tableName); + return; + } + case 'deleteTable': { + await waitForTableActive(client, op.tableName); + await client.send(new DeleteTableCommand({ TableName: op.tableName })); + return; + } + case 'updateTable': { + await waitForTableActive(client, op.tableName); + await client.send( + new UpdateTableCommand({ + TableName: op.tableName, + AttributeDefinitions: op.attributeDefinitions, + BillingMode: op.billingMode, + ProvisionedThroughput: op.provisionedThroughput, + GlobalSecondaryIndexUpdates: op.globalSecondaryIndexUpdates, + StreamSpecification: op.streamSpecification, + }), + ); + await waitForTableActive(client, op.tableName); + return; + } + case 'updateTimeToLive': { + if (!op.timeToLiveSpecification) { + throw new BadRequestException('updateTimeToLive is missing timeToLiveSpecification.'); + } + await waitForTableActive(client, op.tableName); + await client.send( + new UpdateTimeToLiveCommand({ + TableName: op.tableName, + TimeToLiveSpecification: { + Enabled: op.timeToLiveSpecification.enabled, + AttributeName: op.timeToLiveSpecification.attributeName, + }, + }), + ); + return; + } + } +} + +// DynamoDB rejects concurrent UpdateTable / DeleteTable calls while the table or any of its +// GSIs is in a transitional state (CREATING / UPDATING / DELETING). Poll DescribeTable until +// everything is ACTIVE so a forward op immediately followed by its rollback does not race. +async function waitForTableActive(client: DynamoDB, tableName: string, timeoutMs: number = 120_000): Promise { + const deadline = Date.now() + timeoutMs; + let delayMs = 100; + while (Date.now() < deadline) { + let table; + try { + const resp = await client.send(new DescribeTableCommand({ TableName: tableName })); + table = resp.Table; + } catch (err) { + if (err instanceof ResourceNotFoundException) return; + throw err; + } + const tableActive = table?.TableStatus === 'ACTIVE'; + const gsisActive = (table?.GlobalSecondaryIndexes ?? []).every((g) => g.IndexStatus === 'ACTIVE'); + if (tableActive && gsisActive) return; + await sleep(delayMs); + delayMs = Math.min(delayMs * 2, 2_000); + } + throw new Error(`Timed out waiting for DynamoDB table "${tableName}" to reach ACTIVE state.`); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function parseAttributeDefinitions(value: unknown, required: boolean): AttributeDefinition[] | undefined { + if (value === undefined) { + if (required) throw new BadRequestException('attributeDefinitions is required.'); + return undefined; + } + if (!Array.isArray(value)) { + throw new BadRequestException('attributeDefinitions must be an array.'); + } + if (required && value.length === 0) { + throw new BadRequestException('attributeDefinitions must not be empty.'); + } + return value.map((entry, idx) => { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + throw new BadRequestException(`attributeDefinitions[${idx}] must be an object.`); + } + const obj = entry as Record; + const name = obj.attributeName; + const type = obj.attributeType; + if (typeof name !== 'string' || name.trim().length === 0) { + throw new BadRequestException(`attributeDefinitions[${idx}].attributeName must be a non-empty string.`); + } + if (typeof type !== 'string' || !ALLOWED_ATTRIBUTE_TYPES.has(type as ScalarAttributeType)) { + throw new BadRequestException( + `attributeDefinitions[${idx}].attributeType must be one of ${Array.from(ALLOWED_ATTRIBUTE_TYPES).join(', ')}.`, + ); + } + return { AttributeName: name, AttributeType: type as ScalarAttributeType }; + }); +} + +function parseKeySchema(value: unknown, required: boolean): KeySchemaElement[] | undefined { + if (value === undefined) { + if (required) throw new BadRequestException('keySchema is required.'); + return undefined; + } + if (!Array.isArray(value) || value.length === 0) { + throw new BadRequestException('keySchema must be a non-empty array.'); + } + if (value.length > 2) { + throw new BadRequestException('keySchema may contain at most 2 elements (HASH and optional RANGE).'); + } + const seenKeyTypes = new Set(); + return value.map((entry, idx) => { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + throw new BadRequestException(`keySchema[${idx}] must be an object.`); + } + const obj = entry as Record; + const name = obj.attributeName; + const keyType = obj.keyType; + if (typeof name !== 'string' || name.trim().length === 0) { + throw new BadRequestException(`keySchema[${idx}].attributeName must be a non-empty string.`); + } + if (typeof keyType !== 'string' || !ALLOWED_KEY_TYPES.has(keyType as KeyType)) { + throw new BadRequestException(`keySchema[${idx}].keyType must be "HASH" or "RANGE".`); + } + if (seenKeyTypes.has(keyType)) { + throw new BadRequestException(`keySchema contains duplicate keyType "${keyType}".`); + } + seenKeyTypes.add(keyType); + return { AttributeName: name, KeyType: keyType as KeyType }; + }); +} + +function ensureAttributesCoverKeys( + attributes: AttributeDefinition[] | undefined, + keys: KeySchemaElement[] | undefined, +): void { + if (!attributes || !keys) return; + const known = new Set(attributes.map((a) => a.AttributeName)); + for (const k of keys) { + if (k.AttributeName && !known.has(k.AttributeName)) { + throw new BadRequestException( + `keySchema references attribute "${k.AttributeName}" which is not declared in attributeDefinitions.`, + ); + } + } +} + +function parseBillingMode(value: unknown): BillingMode | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'string' || !ALLOWED_BILLING_MODES.has(value as BillingMode)) { + throw new BadRequestException(`billingMode must be one of ${Array.from(ALLOWED_BILLING_MODES).join(', ')}.`); + } + return value as BillingMode; +} + +function parseProvisionedThroughput(value: unknown): ProvisionedThroughput | undefined { + if (value === undefined) return undefined; + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new BadRequestException('provisionedThroughput must be an object.'); + } + const obj = value as Record; + const read = obj.readCapacityUnits; + const write = obj.writeCapacityUnits; + if (typeof read !== 'number' || !Number.isInteger(read) || read < 1) { + throw new BadRequestException('provisionedThroughput.readCapacityUnits must be a positive integer.'); + } + if (typeof write !== 'number' || !Number.isInteger(write) || write < 1) { + throw new BadRequestException('provisionedThroughput.writeCapacityUnits must be a positive integer.'); + } + return { ReadCapacityUnits: read, WriteCapacityUnits: write }; +} + +function parseGsiList(value: unknown): GlobalSecondaryIndex[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value)) { + throw new BadRequestException('globalSecondaryIndexes must be an array.'); + } + return value.map((entry, idx) => parseGsi(entry, idx)); +} + +function parseLsiList(value: unknown): LocalSecondaryIndex[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value)) { + throw new BadRequestException('localSecondaryIndexes must be an array.'); + } + return value.map((entry, idx) => { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + throw new BadRequestException(`localSecondaryIndexes[${idx}] must be an object.`); + } + const obj = entry as Record; + const name = obj.indexName; + if (typeof name !== 'string' || name.trim().length === 0) { + throw new BadRequestException(`localSecondaryIndexes[${idx}].indexName must be a non-empty string.`); + } + const keySchema = parseKeySchema(obj.keySchema, true)!; + const projection = parseProjection(obj.projection, idx, 'localSecondaryIndexes'); + return { IndexName: name, KeySchema: keySchema, Projection: projection }; + }); +} + +function parseGsi(entry: unknown, idx: number): GlobalSecondaryIndex { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + throw new BadRequestException(`globalSecondaryIndexes[${idx}] must be an object.`); + } + const obj = entry as Record; + const name = obj.indexName; + if (typeof name !== 'string' || name.trim().length === 0) { + throw new BadRequestException(`globalSecondaryIndexes[${idx}].indexName must be a non-empty string.`); + } + const keySchema = parseKeySchema(obj.keySchema, true)!; + const projection = parseProjection(obj.projection, idx, 'globalSecondaryIndexes'); + const throughput = parseProvisionedThroughput(obj.provisionedThroughput); + return { + IndexName: name, + KeySchema: keySchema, + Projection: projection, + ProvisionedThroughput: throughput, + }; +} + +function parseGsiUpdates(value: unknown): GlobalSecondaryIndexUpdate[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value)) { + throw new BadRequestException('globalSecondaryIndexUpdates must be an array.'); + } + return value.map((entry, idx) => { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + throw new BadRequestException(`globalSecondaryIndexUpdates[${idx}] must be an object.`); + } + const obj = entry as Record; + const out: GlobalSecondaryIndexUpdate = {}; + if (obj.create !== undefined) { + out.Create = parseGsiCreateAction(obj.create, idx); + } + if (obj.update !== undefined) { + out.Update = parseGsiUpdateAction(obj.update, idx); + } + if (obj.delete !== undefined) { + out.Delete = parseGsiDeleteAction(obj.delete, idx); + } + const filled = [out.Create, out.Update, out.Delete].filter((v) => v !== undefined); + if (filled.length !== 1) { + throw new BadRequestException( + `globalSecondaryIndexUpdates[${idx}] must contain exactly one of create, update, or delete.`, + ); + } + return out; + }); +} + +function parseGsiCreateAction(value: unknown, idx: number): GlobalSecondaryIndexUpdate['Create'] { + const gsi = parseGsi(value, idx); + return { + IndexName: gsi.IndexName!, + KeySchema: gsi.KeySchema!, + Projection: gsi.Projection!, + ProvisionedThroughput: gsi.ProvisionedThroughput, + }; +} + +function parseGsiUpdateAction(value: unknown, idx: number): GlobalSecondaryIndexUpdate['Update'] { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new BadRequestException(`globalSecondaryIndexUpdates[${idx}].update must be an object.`); + } + const obj = value as Record; + const name = obj.indexName; + if (typeof name !== 'string' || name.trim().length === 0) { + throw new BadRequestException(`globalSecondaryIndexUpdates[${idx}].update.indexName must be a non-empty string.`); + } + const throughput = parseProvisionedThroughput(obj.provisionedThroughput); + if (!throughput) { + throw new BadRequestException(`globalSecondaryIndexUpdates[${idx}].update.provisionedThroughput is required.`); + } + return { IndexName: name, ProvisionedThroughput: throughput }; +} + +function parseGsiDeleteAction(value: unknown, idx: number): GlobalSecondaryIndexUpdate['Delete'] { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new BadRequestException(`globalSecondaryIndexUpdates[${idx}].delete must be an object.`); + } + const obj = value as Record; + const name = obj.indexName; + if (typeof name !== 'string' || name.trim().length === 0) { + throw new BadRequestException(`globalSecondaryIndexUpdates[${idx}].delete.indexName must be a non-empty string.`); + } + return { IndexName: name }; +} + +function parseProjection( + value: unknown, + idx: number, + parent: 'globalSecondaryIndexes' | 'localSecondaryIndexes', +): { ProjectionType: ProjectionType; NonKeyAttributes?: string[] } { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new BadRequestException(`${parent}[${idx}].projection must be an object.`); + } + const obj = value as Record; + const pt = obj.projectionType; + if (typeof pt !== 'string' || !ALLOWED_PROJECTION_TYPES.has(pt as ProjectionType)) { + throw new BadRequestException( + `${parent}[${idx}].projection.projectionType must be one of ${Array.from(ALLOWED_PROJECTION_TYPES).join(', ')}.`, + ); + } + const result: { ProjectionType: ProjectionType; NonKeyAttributes?: string[] } = { + ProjectionType: pt as ProjectionType, + }; + if (pt === 'INCLUDE') { + const nonKey = obj.nonKeyAttributes; + if (!Array.isArray(nonKey) || nonKey.length === 0 || !nonKey.every((a) => typeof a === 'string' && a.length > 0)) { + throw new BadRequestException( + `${parent}[${idx}].projection.nonKeyAttributes must be a non-empty array of strings when projectionType=INCLUDE.`, + ); + } + result.NonKeyAttributes = nonKey as string[]; + } + return result; +} + +function parseStreamSpecification(value: unknown): StreamSpecification | undefined { + if (value === undefined) return undefined; + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new BadRequestException('streamSpecification must be an object.'); + } + const obj = value as Record; + if (typeof obj.streamEnabled !== 'boolean') { + throw new BadRequestException('streamSpecification.streamEnabled must be a boolean.'); + } + const spec: StreamSpecification = { StreamEnabled: obj.streamEnabled }; + if (obj.streamViewType !== undefined) { + if ( + typeof obj.streamViewType !== 'string' || + !ALLOWED_STREAM_VIEW_TYPES.has(obj.streamViewType as StreamViewType) + ) { + throw new BadRequestException( + `streamSpecification.streamViewType must be one of ${Array.from(ALLOWED_STREAM_VIEW_TYPES).join(', ')}.`, + ); + } + spec.StreamViewType = obj.streamViewType as StreamViewType; + } + return spec; +} + +function parseSseSpecification(value: unknown): SSESpecification | undefined { + if (value === undefined) return undefined; + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new BadRequestException('sseSpecification must be an object.'); + } + const obj = value as Record; + const spec: SSESpecification = {}; + if (obj.enabled !== undefined) { + if (typeof obj.enabled !== 'boolean') { + throw new BadRequestException('sseSpecification.enabled must be a boolean.'); + } + spec.Enabled = obj.enabled; + } + if (obj.sseType !== undefined) { + if (typeof obj.sseType !== 'string' || !ALLOWED_SSE_TYPES.has(obj.sseType as SSEType)) { + throw new BadRequestException( + `sseSpecification.sseType must be one of ${Array.from(ALLOWED_SSE_TYPES).join(', ')}.`, + ); + } + spec.SSEType = obj.sseType as SSEType; + } + if (obj.kmsMasterKeyId !== undefined) { + if (typeof obj.kmsMasterKeyId !== 'string') { + throw new BadRequestException('sseSpecification.kmsMasterKeyId must be a string.'); + } + spec.KMSMasterKeyId = obj.kmsMasterKeyId; + } + return spec; +} diff --git a/backend/src/entities/table-schema/utils/elasticsearch-schema-op.ts b/backend/src/entities/table-schema/utils/elasticsearch-schema-op.ts new file mode 100644 index 000000000..c75495792 --- /dev/null +++ b/backend/src/entities/table-schema/utils/elasticsearch-schema-op.ts @@ -0,0 +1,206 @@ +import { Client } from '@elastic/elasticsearch'; +import { BadRequestException } from '@nestjs/common'; +import { SchemaChangeTypeEnum } from '../table-schema-change-enums.js'; + +export interface ElasticsearchExecutionConnection { + host?: string | null; + port?: number | null; + username?: string | null; + password?: string | null; + ssl?: boolean | null; + cert?: string | null; +} + +export type ElasticsearchOperationKind = 'createIndex' | 'deleteIndex' | 'updateMapping'; + +export interface ElasticsearchSchemaOp { + operation: ElasticsearchOperationKind; + indexName: string; + mappings?: Record; + settings?: Record; + properties?: Record; +} + +const ALLOWED_OPERATIONS: ReadonlySet = new Set([ + 'createIndex', + 'deleteIndex', + 'updateMapping', +]); + +const CHANGE_TYPE_TO_OPERATION: Record = { + [SchemaChangeTypeEnum.ELASTICSEARCH_CREATE_INDEX]: 'createIndex', + [SchemaChangeTypeEnum.ELASTICSEARCH_DELETE_INDEX]: 'deleteIndex', + [SchemaChangeTypeEnum.ELASTICSEARCH_UPDATE_MAPPING]: 'updateMapping', +}; + +// Identifiers Elasticsearch reserves for system indices and component templates we must not touch. +const FORBIDDEN_INDEX_PREFIXES = ['.', '_']; + +export interface ValidateElasticsearchOpInput { + opJson: string; + changeType: SchemaChangeTypeEnum; + targetTableName: string; + allowAnyOperation?: boolean; +} + +export function validateProposedElasticsearchOp(input: ValidateElasticsearchOpInput): ElasticsearchSchemaOp { + const { opJson, changeType, targetTableName, allowAnyOperation } = input; + + if (!opJson || opJson.trim().length === 0) { + throw new BadRequestException('Proposed Elasticsearch operation is empty.'); + } + + let parsed: unknown; + try { + parsed = JSON.parse(opJson); + } catch (err) { + throw new BadRequestException(`Proposed Elasticsearch operation is not valid JSON: ${(err as Error).message}`); + } + + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new BadRequestException('Proposed Elasticsearch operation must be a single JSON object.'); + } + + const op = parsed as Record; + const operation = op.operation; + if (typeof operation !== 'string' || !ALLOWED_OPERATIONS.has(operation as ElasticsearchOperationKind)) { + throw new BadRequestException( + `Proposed Elasticsearch operation "${String(operation)}" is not one of ${Array.from(ALLOWED_OPERATIONS).join(', ')}.`, + ); + } + + const indexName = op.indexName; + if (typeof indexName !== 'string' || indexName.trim().length === 0) { + throw new BadRequestException('Proposed Elasticsearch operation must include a non-empty "indexName".'); + } + if (indexName !== targetTableName) { + throw new BadRequestException( + `Proposed Elasticsearch operation targets index "${indexName}" which does not match declared targetTableName "${targetTableName}".`, + ); + } + for (const prefix of FORBIDDEN_INDEX_PREFIXES) { + if (indexName.startsWith(prefix)) { + throw new BadRequestException( + `Proposed Elasticsearch indexName "${indexName}" starts with reserved prefix "${prefix}"; system indices cannot be modified.`, + ); + } + } + + if (!allowAnyOperation) { + const expected = CHANGE_TYPE_TO_OPERATION[changeType]; + if (expected && expected !== operation) { + throw new BadRequestException( + `Proposed Elasticsearch operation "${operation}" does not match declared changeType "${changeType}" (expected "${expected}").`, + ); + } + } + + const typed: ElasticsearchSchemaOp = { + operation: operation as ElasticsearchOperationKind, + indexName, + }; + + switch (operation) { + case 'createIndex': { + if (op.mappings !== undefined) { + if (!isPlainObject(op.mappings)) { + throw new BadRequestException('createIndex.mappings must be an object.'); + } + typed.mappings = op.mappings as Record; + } + if (op.settings !== undefined) { + if (!isPlainObject(op.settings)) { + throw new BadRequestException('createIndex.settings must be an object.'); + } + typed.settings = op.settings as Record; + } + break; + } + case 'deleteIndex': { + break; + } + case 'updateMapping': { + if (!isPlainObject(op.properties) || Object.keys(op.properties as Record).length === 0) { + throw new BadRequestException('updateMapping.properties must be a non-empty object of new field definitions.'); + } + for (const [fieldName, def] of Object.entries(op.properties as Record)) { + if (!isPlainObject(def)) { + throw new BadRequestException(`updateMapping.properties["${fieldName}"] must be an object.`); + } + } + typed.properties = op.properties as Record; + break; + } + } + + return typed; +} + +export async function executeElasticsearchSchemaOp( + connection: ElasticsearchExecutionConnection, + op: ElasticsearchSchemaOp, +): Promise { + const client = buildElasticsearchClient(connection); + try { + await dispatchElasticsearchOp(client, op); + } finally { + await client.close().catch(() => undefined); + } +} + +function buildElasticsearchClient(connection: ElasticsearchExecutionConnection): Client { + const host = connection.host ?? ''; + const port = connection.port ?? 9200; + const protocol = connection.ssl ? 'https' : 'http'; + const node = `${protocol}://${host}:${port}`; + const options: Record = { + node, + auth: { + username: connection.username ?? '', + password: connection.password ?? '', + }, + }; + if (connection.ssl) { + const tls: Record = { + rejectUnauthorized: !connection.cert, + }; + if (connection.cert) { + tls.ca = connection.cert; + } + options.tls = tls; + } + return new Client(options as ConstructorParameters[0]); +} + +async function dispatchElasticsearchOp(client: Client, op: ElasticsearchSchemaOp): Promise { + switch (op.operation) { + case 'createIndex': { + await client.indices.create({ + index: op.indexName, + body: { + ...(op.mappings ? { mappings: op.mappings } : {}), + ...(op.settings ? { settings: op.settings } : {}), + }, + } as Parameters[0]); + return; + } + case 'deleteIndex': { + await client.indices.delete({ index: op.indexName }); + return; + } + case 'updateMapping': { + if (!op.properties) { + throw new BadRequestException('updateMapping is missing properties.'); + } + await client.indices.putMapping({ + index: op.indexName, + body: { properties: op.properties }, + } as Parameters[0]); + return; + } + } +} + +function isPlainObject(value: unknown): value is Record { + return !!value && typeof value === 'object' && !Array.isArray(value); +} diff --git a/backend/src/entities/table-schema/utils/execute-schema-change.ts b/backend/src/entities/table-schema/utils/execute-schema-change.ts new file mode 100644 index 000000000..3b09df054 --- /dev/null +++ b/backend/src/entities/table-schema/utils/execute-schema-change.ts @@ -0,0 +1,79 @@ +import { getDataAccessObject } from '@rocketadmin/shared-code/dist/src/data-access-layer/shared/create-data-access-object.js'; +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; +import { ConnectionEntity } from '../../connection/connection.entity.js'; +import { + isDynamoDbSchemaChangeType, + isElasticsearchSchemaChangeType, + isMongoSchemaChangeType, + SchemaChangeTypeEnum, +} from '../table-schema-change-enums.js'; +import { + isCassandraDialect, + isClickHouseDialect, + isDynamoDbDialect, + isElasticsearchDialect, +} from './assert-dialect-supported.js'; +import { executeCassandraDdl } from './cassandra-ddl.js'; +import { executeClickHouseDdl } from './clickhouse-ddl.js'; +import { executeDynamoDbSchemaOp, validateProposedDynamoDbOp } from './dynamodb-schema-op.js'; +import { executeElasticsearchSchemaOp, validateProposedElasticsearchOp } from './elasticsearch-schema-op.js'; +import { executeMongoSchemaOp, validateProposedMongoOp } from './mongo-schema-op.js'; + +export interface ExecuteSchemaChangeOptions { + connection: ConnectionEntity; + connectionType: ConnectionTypesEnum; + changeType: SchemaChangeTypeEnum; + targetTableName: string; + sql: string; + allowAnyOperation?: boolean; +} + +export async function executeSchemaChange(opts: ExecuteSchemaChangeOptions): Promise { + const { connection, connectionType, changeType, targetTableName, sql, allowAnyOperation } = opts; + + if (isMongoSchemaChangeType(changeType)) { + const op = validateProposedMongoOp({ + opJson: sql, + changeType, + targetTableName, + allowAnyOperation, + }); + await executeMongoSchemaOp(connection, op); + return; + } + + if (isDynamoDbSchemaChangeType(changeType) || isDynamoDbDialect(connectionType)) { + const op = validateProposedDynamoDbOp({ + opJson: sql, + changeType, + targetTableName, + allowAnyOperation, + }); + await executeDynamoDbSchemaOp(connection, op); + return; + } + + if (isElasticsearchSchemaChangeType(changeType) || isElasticsearchDialect(connectionType)) { + const op = validateProposedElasticsearchOp({ + opJson: sql, + changeType, + targetTableName, + allowAnyOperation, + }); + await executeElasticsearchSchemaOp(connection, op); + return; + } + + if (isClickHouseDialect(connectionType)) { + await executeClickHouseDdl(connection, sql); + return; + } + + if (isCassandraDialect(connectionType)) { + await executeCassandraDdl(connection, sql); + return; + } + + const dao = getDataAccessObject(connection); + await dao.executeRawQuery(sql, targetTableName, null); +} diff --git a/backend/src/entities/table-schema/utils/map-schema-change-to-response-dto.ts b/backend/src/entities/table-schema/utils/map-schema-change-to-response-dto.ts new file mode 100644 index 000000000..eecdf0596 --- /dev/null +++ b/backend/src/entities/table-schema/utils/map-schema-change-to-response-dto.ts @@ -0,0 +1,30 @@ +import { SchemaChangeResponseDto } from '../application/data-transfer-objects/schema-change-response.dto.js'; +import { TableSchemaChangeEntity } from '../table-schema-change.entity.js'; + +export function mapSchemaChangeToResponseDto(entity: TableSchemaChangeEntity): SchemaChangeResponseDto { + return { + id: entity.id, + connectionId: entity.connectionId, + batchId: entity.batchId, + orderInBatch: entity.orderInBatch, + authorId: entity.authorId, + previousChangeId: entity.previousChangeId, + forwardSql: entity.forwardSql, + rollbackSql: entity.rollbackSql, + userModifiedSql: entity.userModifiedSql, + status: entity.status, + changeType: entity.changeType, + targetTableName: entity.targetTableName, + databaseType: entity.databaseType, + executionError: entity.executionError, + isReversible: entity.isReversible, + autoRollbackAttempted: entity.autoRollbackAttempted, + autoRollbackSucceeded: entity.autoRollbackSucceeded, + userPrompt: entity.userPrompt, + aiSummary: entity.aiSummary, + aiReasoning: entity.aiReasoning, + createdAt: entity.createdAt, + appliedAt: entity.appliedAt, + rolledBackAt: entity.rolledBackAt, + }; +} diff --git a/backend/src/entities/table-schema/utils/mongo-schema-op.ts b/backend/src/entities/table-schema/utils/mongo-schema-op.ts new file mode 100644 index 000000000..62572f9da --- /dev/null +++ b/backend/src/entities/table-schema/utils/mongo-schema-op.ts @@ -0,0 +1,261 @@ +import { BadRequestException } from '@nestjs/common'; +import { Db, Document, IndexDirection, MongoClient, MongoClientOptions } from 'mongodb'; +import { SchemaChangeTypeEnum } from '../table-schema-change-enums.js'; + +export interface MongoExecutionConnection { + host?: string | null; + port?: number | null; + username?: string | null; + password?: string | null; + database?: string | null; + ssl?: boolean | null; + cert?: string | null; + authSource?: string | null; +} + +export type MongoOperationKind = 'createCollection' | 'dropCollection' | 'setValidator' | 'createIndex' | 'dropIndex'; + +export interface MongoSchemaOp { + operation: MongoOperationKind; + collectionName: string; + validatorSchema?: Document | null; + validationLevel?: 'off' | 'strict' | 'moderate'; + validationAction?: 'warn' | 'error'; + indexName?: string; + indexSpec?: Record; + indexOptions?: Document; +} + +const ALLOWED_OPERATIONS: ReadonlySet = new Set([ + 'createCollection', + 'dropCollection', + 'setValidator', + 'createIndex', + 'dropIndex', +]); + +const CHANGE_TYPE_TO_OPERATION: Record = { + [SchemaChangeTypeEnum.MONGO_CREATE_COLLECTION]: 'createCollection', + [SchemaChangeTypeEnum.MONGO_DROP_COLLECTION]: 'dropCollection', + [SchemaChangeTypeEnum.MONGO_SET_VALIDATOR]: 'setValidator', + [SchemaChangeTypeEnum.MONGO_CREATE_INDEX]: 'createIndex', + [SchemaChangeTypeEnum.MONGO_DROP_INDEX]: 'dropIndex', +}; + +// JavaScript-executing operators that would let the AI smuggle arbitrary code into the server. +const FORBIDDEN_VALIDATOR_OPERATORS = ['$where', '$function', '$accumulator']; + +export interface ValidateMongoOpInput { + opJson: string; + changeType: SchemaChangeTypeEnum; + targetTableName: string; + // When true (rollback path), we do not enforce changeType↔operation matching — rollback + // for a createCollection is a dropCollection, etc. We still enforce structural safety. + allowAnyOperation?: boolean; +} + +export function validateProposedMongoOp(input: ValidateMongoOpInput): MongoSchemaOp { + const { opJson, changeType, targetTableName, allowAnyOperation } = input; + + if (!opJson || opJson.trim().length === 0) { + throw new BadRequestException('Proposed Mongo operation is empty.'); + } + + let parsed: unknown; + try { + parsed = JSON.parse(opJson); + } catch (err) { + throw new BadRequestException(`Proposed Mongo operation is not valid JSON: ${(err as Error).message}`); + } + + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new BadRequestException('Proposed Mongo operation must be a single JSON object.'); + } + + const op = parsed as Record; + const operation = op.operation; + if (typeof operation !== 'string' || !ALLOWED_OPERATIONS.has(operation as MongoOperationKind)) { + throw new BadRequestException( + `Proposed Mongo operation "${String(operation)}" is not one of ${Array.from(ALLOWED_OPERATIONS).join(', ')}.`, + ); + } + + const collectionName = op.collectionName; + if (typeof collectionName !== 'string' || collectionName.trim().length === 0) { + throw new BadRequestException('Proposed Mongo operation must include a non-empty "collectionName".'); + } + if (collectionName !== targetTableName) { + throw new BadRequestException( + `Proposed Mongo operation targets collection "${collectionName}" which does not match declared targetTableName "${targetTableName}".`, + ); + } + + if (!allowAnyOperation) { + const expected = CHANGE_TYPE_TO_OPERATION[changeType]; + if (expected && expected !== operation) { + throw new BadRequestException( + `Proposed Mongo operation "${operation}" does not match declared changeType "${changeType}" (expected "${expected}").`, + ); + } + } + + const typedOp: MongoSchemaOp = { operation: operation as MongoOperationKind, collectionName }; + + switch (operation) { + case 'setValidator': { + const schemaValue = op.validatorSchema; + if (schemaValue !== null && schemaValue !== undefined && typeof schemaValue !== 'object') { + throw new BadRequestException('setValidator.validatorSchema must be an object or null.'); + } + assertNoForbiddenValidatorOperators(schemaValue); + typedOp.validatorSchema = (schemaValue ?? null) as Document | null; + if (op.validationLevel !== undefined) { + if (op.validationLevel !== 'off' && op.validationLevel !== 'strict' && op.validationLevel !== 'moderate') { + throw new BadRequestException('setValidator.validationLevel must be "off", "strict", or "moderate".'); + } + typedOp.validationLevel = op.validationLevel; + } + if (op.validationAction !== undefined) { + if (op.validationAction !== 'warn' && op.validationAction !== 'error') { + throw new BadRequestException('setValidator.validationAction must be "warn" or "error".'); + } + typedOp.validationAction = op.validationAction; + } + break; + } + case 'createIndex': { + const spec = op.indexSpec; + if (!spec || typeof spec !== 'object' || Array.isArray(spec) || Object.keys(spec).length === 0) { + throw new BadRequestException('createIndex.indexSpec must be a non-empty object.'); + } + typedOp.indexSpec = spec as Record; + const topLevelName = typeof op.indexName === 'string' ? op.indexName : undefined; + const optionsValue = op.indexOptions; + if ( + optionsValue !== undefined && + (typeof optionsValue !== 'object' || optionsValue === null || Array.isArray(optionsValue)) + ) { + throw new BadRequestException('createIndex.indexOptions must be an object when provided.'); + } + const options = (optionsValue as Document | undefined) ?? {}; + const optionsName = typeof options.name === 'string' ? (options.name as string) : undefined; + const resolvedName = topLevelName ?? optionsName; + if (!resolvedName || resolvedName.trim().length === 0) { + throw new BadRequestException( + 'createIndex requires an explicit indexName (so rollback dropIndex is unambiguous).', + ); + } + options.name = resolvedName; + typedOp.indexName = resolvedName; + typedOp.indexOptions = options; + break; + } + case 'dropIndex': { + const name = op.indexName; + if (typeof name !== 'string' || name.trim().length === 0) { + throw new BadRequestException('dropIndex requires a non-empty "indexName".'); + } + typedOp.indexName = name; + break; + } + case 'createCollection': + case 'dropCollection': + break; + } + + return typedOp; +} + +export async function executeMongoSchemaOp(connection: MongoExecutionConnection, op: MongoSchemaOp): Promise { + const { client, db } = await openMongoConnection(connection); + try { + await dispatchMongoOp(db, op); + } finally { + await client.close().catch(() => undefined); + } +} + +function assertNoForbiddenValidatorOperators(value: unknown): void { + if (!value || typeof value !== 'object') return; + const stack: unknown[] = [value]; + while (stack.length > 0) { + const current = stack.pop(); + if (!current || typeof current !== 'object') continue; + if (Array.isArray(current)) { + for (const item of current) stack.push(item); + continue; + } + for (const [k, v] of Object.entries(current as Record)) { + if (FORBIDDEN_VALIDATOR_OPERATORS.includes(k)) { + throw new BadRequestException( + `Validator schema contains forbidden operator "${k}" (JavaScript execution in validators is blocked).`, + ); + } + if (v && typeof v === 'object') stack.push(v); + } + } +} + +async function openMongoConnection(connection: MongoExecutionConnection): Promise<{ client: MongoClient; db: Db }> { + const host = connection.host ?? ''; + const username = connection.username ?? ''; + const password = connection.password ?? ''; + const database = connection.database ?? ''; + const port = connection.port ?? 27017; + + let mongoConnectionString = ''; + if (host.includes('mongodb+srv')) { + const hostNameParts = host.split('//'); + mongoConnectionString = `${hostNameParts[0]}//${encodeURIComponent(username)}:${encodeURIComponent(password)}@${hostNameParts[1]}/${database}`; + } else { + mongoConnectionString = `mongodb://${encodeURIComponent(username)}:${encodeURIComponent(password)}@${host}:${port}/${database}`; + } + + let options: MongoClientOptions = {}; + if (connection.ssl) { + options = { + ssl: true, + ca: connection.cert ? [connection.cert] : undefined, + } as MongoClientOptions; + } + + if (connection.authSource) { + mongoConnectionString += mongoConnectionString.includes('?') ? '&' : '?'; + mongoConnectionString += `authSource=${connection.authSource}`; + } + + const client = new MongoClient(mongoConnectionString, options); + const connectedClient = await client.connect(); + return { client, db: connectedClient.db(database) }; +} + +async function dispatchMongoOp(db: Db, op: MongoSchemaOp): Promise { + switch (op.operation) { + case 'createCollection': { + await db.createCollection(op.collectionName); + return; + } + case 'dropCollection': { + await db.collection(op.collectionName).drop(); + return; + } + case 'setValidator': { + const command: Document = { collMod: op.collectionName }; + command.validator = op.validatorSchema ?? {}; + command.validationLevel = op.validationLevel ?? 'strict'; + command.validationAction = op.validationAction ?? 'error'; + await db.command(command); + return; + } + case 'createIndex': { + if (!op.indexSpec) throw new BadRequestException('createIndex is missing indexSpec.'); + await db.collection(op.collectionName).createIndex(op.indexSpec, op.indexOptions ?? {}); + return; + } + case 'dropIndex': { + if (!op.indexName) throw new BadRequestException('dropIndex is missing indexName.'); + await db.collection(op.collectionName).dropIndex(op.indexName); + return; + } + } +} diff --git a/backend/src/entities/table-schema/utils/validate-proposed-ddl.ts b/backend/src/entities/table-schema/utils/validate-proposed-ddl.ts new file mode 100644 index 000000000..e50e0be73 --- /dev/null +++ b/backend/src/entities/table-schema/utils/validate-proposed-ddl.ts @@ -0,0 +1,162 @@ +import { BadRequestException } from '@nestjs/common'; +import { ConnectionTypesEnum } from '@rocketadmin/shared-code/dist/src/shared/enums/connection-types-enum.js'; +import sqlParser from 'node-sql-parser'; + +const { Parser } = sqlParser; + +import { isMongoSchemaChangeType, SchemaChangeTypeEnum } from '../table-schema-change-enums.js'; +import { connectionTypeToParserDialect } from './assert-dialect-supported.js'; + +const FORBIDDEN_PATTERNS: ReadonlyArray = [ + /\bDROP\s+DATABASE\b/i, + /\bDROP\s+SCHEMA\b/i, + /\bDROP\s+USER\b/i, + /\bDROP\s+ROLE\b/i, + /\bCREATE\s+DATABASE\b/i, + /\bCREATE\s+SCHEMA\b/i, + /\bCREATE\s+USER\b/i, + /\bCREATE\s+ROLE\b/i, + /\bALTER\s+DATABASE\b/i, + /\bALTER\s+SCHEMA\b/i, + /\bALTER\s+USER\b/i, + /\bALTER\s+ROLE\b/i, + /\bGRANT\b/i, + /\bREVOKE\b/i, + /\bTRUNCATE\b/i, + /\bDELETE\s+FROM\b/i, + /\bINSERT\s+INTO\b/i, + /\bUPDATE\s+[^\s]+\s+SET\b/i, + /\bCOPY\b/i, + /\\\w/, +]; + +interface ExpectedShape { + type: string; + keyword?: string; +} + +const EXPECTED_SHAPES: Record = { + [SchemaChangeTypeEnum.CREATE_TABLE]: { type: 'create', keyword: 'table' }, + [SchemaChangeTypeEnum.DROP_TABLE]: { type: 'drop', keyword: 'table' }, + [SchemaChangeTypeEnum.ADD_COLUMN]: { type: 'alter', keyword: 'table' }, + [SchemaChangeTypeEnum.DROP_COLUMN]: { type: 'alter', keyword: 'table' }, + [SchemaChangeTypeEnum.ALTER_COLUMN]: { type: 'alter', keyword: 'table' }, + [SchemaChangeTypeEnum.ADD_INDEX]: { type: 'create', keyword: 'index' }, + [SchemaChangeTypeEnum.DROP_INDEX]: { type: 'drop', keyword: 'index' }, + [SchemaChangeTypeEnum.ADD_FOREIGN_KEY]: { type: 'alter', keyword: 'table' }, + [SchemaChangeTypeEnum.DROP_FOREIGN_KEY]: { type: 'alter', keyword: 'table' }, + [SchemaChangeTypeEnum.ADD_PRIMARY_KEY]: { type: 'alter', keyword: 'table' }, + [SchemaChangeTypeEnum.DROP_PRIMARY_KEY]: { type: 'alter', keyword: 'table' }, + [SchemaChangeTypeEnum.MONGO_CREATE_COLLECTION]: null, + [SchemaChangeTypeEnum.MONGO_DROP_COLLECTION]: null, + [SchemaChangeTypeEnum.MONGO_SET_VALIDATOR]: null, + [SchemaChangeTypeEnum.MONGO_CREATE_INDEX]: null, + [SchemaChangeTypeEnum.MONGO_DROP_INDEX]: null, + [SchemaChangeTypeEnum.DYNAMODB_CREATE_TABLE]: null, + [SchemaChangeTypeEnum.DYNAMODB_DROP_TABLE]: null, + [SchemaChangeTypeEnum.DYNAMODB_UPDATE_TABLE]: null, + [SchemaChangeTypeEnum.DYNAMODB_UPDATE_TTL]: null, + [SchemaChangeTypeEnum.ELASTICSEARCH_CREATE_INDEX]: null, + [SchemaChangeTypeEnum.ELASTICSEARCH_DELETE_INDEX]: null, + [SchemaChangeTypeEnum.ELASTICSEARCH_UPDATE_MAPPING]: null, + [SchemaChangeTypeEnum.ROLLBACK]: null, + [SchemaChangeTypeEnum.OTHER]: null, +}; + +export interface ValidateProposedDdlOptions { + sql: string; + connectionType: ConnectionTypesEnum; + changeType: SchemaChangeTypeEnum; + targetTableName: string; +} + +export function validateProposedDdl(opts: ValidateProposedDdlOptions): void { + const { sql, connectionType, changeType, targetTableName } = opts; + const trimmed = sql?.trim() ?? ''; + if (!trimmed) { + throw new BadRequestException('Proposed SQL is empty.'); + } + + for (const pattern of FORBIDDEN_PATTERNS) { + if (pattern.test(trimmed)) { + throw new BadRequestException(`Proposed SQL contains a forbidden construct (matched ${pattern}).`); + } + } + + const stripped = trimmed.replace(/;+\s*$/, ''); + if (/;/.test(stripped)) { + throw new BadRequestException('Proposed SQL must be a single statement; multi-statement scripts are rejected.'); + } + + // MongoDB schema changes are emitted as structured JSON, not SQL. Parsing them with + // node-sql-parser is meaningless (and will always fall through to the warn path), + // and the AST/target-table checks below don't apply. Richer shape validation runs + // in validateProposedMongoOp, which the use-cases call for Mongo dispatch. + if (isMongoSchemaChangeType(changeType)) { + return; + } + + const parser = new Parser(); + let ast: unknown; + try { + ast = parser.astify(trimmed, { database: connectionTypeToParserDialect(connectionType) }); + } catch (err) { + // AST shape verification is defense-in-depth, not the only gate: + // the forbidden-construct regex and single-statement check above have already run, + // and the DB itself rejects invalid DDL with auto-rollback. node-sql-parser's grammar + // doesn't cover every dialect feature (GENERATED AS IDENTITY, EXCLUDE, PARTITION BY, + // INHERITS, TABLESPACE, vendor-specific types), so we log and fall through rather than + // block legitimate statements. + // eslint-disable-next-line no-console + console.warn( + `[validate-proposed-ddl] parser could not analyze SQL (${(err as Error).message}); falling back to regex checks. sql=${trimmed.slice(0, 200)}`, + ); + return; + } + + const statements = Array.isArray(ast) ? ast : [ast]; + if (statements.length !== 1) { + throw new BadRequestException('Proposed SQL must contain exactly one statement.'); + } + + const expected = EXPECTED_SHAPES[changeType]; + if (expected) { + const stmt = statements[0] as { type?: string; keyword?: string }; + if (stmt?.type !== expected.type) { + throw new BadRequestException( + `Proposed SQL does not match declared changeType ${changeType}: expected AST type "${expected.type}", got "${stmt?.type ?? 'unknown'}".`, + ); + } + if (expected.keyword && stmt.keyword && stmt.keyword !== expected.keyword) { + throw new BadRequestException( + `Proposed SQL does not match declared changeType ${changeType}: expected keyword "${expected.keyword}", got "${stmt.keyword}".`, + ); + } + } + + const extractedName = extractTableName(statements[0]); + if (extractedName && normalizeIdentifier(extractedName) !== normalizeIdentifier(targetTableName)) { + throw new BadRequestException( + `Proposed SQL targets table "${extractedName}" which does not match declared targetTableName "${targetTableName}".`, + ); + } +} + +function extractTableName(ast: unknown): string | null { + if (!ast || typeof ast !== 'object') return null; + const node = ast as Record; + + const tableArr = node.table ?? node.name; + if (Array.isArray(tableArr) && tableArr.length > 0) { + const first = tableArr[0] as Record; + if (first && typeof first === 'object') { + const name = first.table ?? first.name; + if (typeof name === 'string') return name; + } + } + return null; +} + +function normalizeIdentifier(name: string): string { + return name.replace(/^[`"[]|[`"\]]$/g, '').toLowerCase(); +} diff --git a/backend/src/guards/index.ts b/backend/src/guards/index.ts index cded724f3..f83450d0a 100644 --- a/backend/src/guards/index.ts +++ b/backend/src/guards/index.ts @@ -5,6 +5,8 @@ export { DashboardEditGuard } from './dashboard-edit.guard.js'; export { DashboardReadGuard } from './dashboard-read.guard.js'; export { GroupEditGuard } from './group-edit.guard.js'; export { GroupReadGuard } from './group-read.guard.js'; +export { SchemaChangeBatchOwnershipGuard } from './schema-change-batch-ownership.guard.js'; +export { SchemaChangeOwnershipGuard } from './schema-change-ownership.guard.js'; export { TableAddGuard } from './table-add.guard.js'; export { TableDeleteGuard } from './table-delete.guard.js'; export { TableEditGuard } from './table-edit.guard.js'; diff --git a/backend/src/guards/schema-change-batch-ownership.guard.ts b/backend/src/guards/schema-change-batch-ownership.guard.ts new file mode 100644 index 000000000..967a51353 --- /dev/null +++ b/backend/src/guards/schema-change-batch-ownership.guard.ts @@ -0,0 +1,53 @@ +import { + BadRequestException, + CanActivate, + ExecutionContext, + ForbiddenException, + Injectable, + NotFoundException, +} from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { IRequestWithCognitoInfo } from '../authorization/index.js'; +import { CedarAction } from '../entities/cedar-authorization/cedar-action-map.js'; +import { CedarAuthorizationService } from '../entities/cedar-authorization/cedar-authorization.service.js'; +import { TableSchemaChangeEntity } from '../entities/table-schema/table-schema-change.entity.js'; +import { Messages } from '../exceptions/text/messages.js'; +import { ValidationHelper } from '../helpers/validators/validation-helper.js'; + +@Injectable() +export class SchemaChangeBatchOwnershipGuard implements CanActivate { + constructor( + @InjectRepository(TableSchemaChangeEntity) + private readonly tableSchemaChangeRepository: Repository, + private readonly cedarAuthService: CedarAuthorizationService, + ) {} + + async canActivate(context: ExecutionContext): Promise { + const request: IRequestWithCognitoInfo = context.switchToHttp().getRequest(); + const userId = request.decoded.sub; + const batchId: string = request.params?.batchId; + + if (!batchId || !ValidationHelper.isValidUUID(batchId)) { + throw new BadRequestException('Invalid or missing batchId.'); + } + + const member = await this.tableSchemaChangeRepository.findOne({ + where: { batchId }, + select: ['id', 'connectionId'], + }); + if (!member) { + throw new NotFoundException('Schema change batch not found.'); + } + + const allowed = await this.cedarAuthService.validate({ + userId, + action: CedarAction.ConnectionEdit, + connectionId: member.connectionId, + }); + if (!allowed) { + throw new ForbiddenException(Messages.DONT_HAVE_PERMISSIONS); + } + return true; + } +} diff --git a/backend/src/guards/schema-change-ownership.guard.ts b/backend/src/guards/schema-change-ownership.guard.ts new file mode 100644 index 000000000..8a22f8b06 --- /dev/null +++ b/backend/src/guards/schema-change-ownership.guard.ts @@ -0,0 +1,53 @@ +import { + BadRequestException, + CanActivate, + ExecutionContext, + ForbiddenException, + Injectable, + NotFoundException, +} from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { Repository } from 'typeorm'; +import { IRequestWithCognitoInfo } from '../authorization/index.js'; +import { CedarAction } from '../entities/cedar-authorization/cedar-action-map.js'; +import { CedarAuthorizationService } from '../entities/cedar-authorization/cedar-authorization.service.js'; +import { TableSchemaChangeEntity } from '../entities/table-schema/table-schema-change.entity.js'; +import { Messages } from '../exceptions/text/messages.js'; +import { ValidationHelper } from '../helpers/validators/validation-helper.js'; + +@Injectable() +export class SchemaChangeOwnershipGuard implements CanActivate { + constructor( + @InjectRepository(TableSchemaChangeEntity) + private readonly tableSchemaChangeRepository: Repository, + private readonly cedarAuthService: CedarAuthorizationService, + ) {} + + async canActivate(context: ExecutionContext): Promise { + const request: IRequestWithCognitoInfo = context.switchToHttp().getRequest(); + const userId = request.decoded.sub; + const changeId: string = request.params?.changeId; + + if (!changeId || !ValidationHelper.isValidUUID(changeId)) { + throw new BadRequestException('Invalid or missing changeId.'); + } + + const change = await this.tableSchemaChangeRepository.findOne({ + where: { id: changeId }, + select: ['id', 'connectionId'], + }); + if (!change) { + throw new NotFoundException('Schema change not found.'); + } + + const allowed = await this.cedarAuthService.validate({ + userId, + action: CedarAction.ConnectionEdit, + connectionId: change.connectionId, + }); + if (!allowed) { + throw new ForbiddenException(Messages.DONT_HAVE_PERMISSIONS); + } + return true; + } +} diff --git a/backend/src/migrations/1776857063726-AddTableSchemaChangeEntity.ts b/backend/src/migrations/1776857063726-AddTableSchemaChangeEntity.ts new file mode 100644 index 000000000..8008e0b9e --- /dev/null +++ b/backend/src/migrations/1776857063726-AddTableSchemaChangeEntity.ts @@ -0,0 +1,89 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddTableSchemaChangeEntity1776857063726 implements MigrationInterface { + name = 'AddTableSchemaChangeEntity1776857063726'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "panel" DROP CONSTRAINT "FK_panel_connection_id"`); + await queryRunner.query(`ALTER TABLE "panel_position" DROP CONSTRAINT "FK_panel_position_dashboard_id"`); + await queryRunner.query(`ALTER TABLE "panel_position" DROP CONSTRAINT "FK_panel_position_query_id"`); + await queryRunner.query( + `CREATE TYPE "public"."table_schema_change_status_enum" AS ENUM('PENDING', 'APPROVED', 'APPLYING', 'APPLIED', 'FAILED', 'REJECTED', 'ROLLED_BACK')`, + ); + await queryRunner.query( + `CREATE TYPE "public"."table_schema_change_changetype_enum" AS ENUM('CREATE_TABLE', 'DROP_TABLE', 'ADD_COLUMN', 'DROP_COLUMN', 'ALTER_COLUMN', 'ADD_INDEX', 'DROP_INDEX', 'ADD_FOREIGN_KEY', 'DROP_FOREIGN_KEY', 'ADD_PRIMARY_KEY', 'DROP_PRIMARY_KEY', 'ROLLBACK', 'OTHER')`, + ); + await queryRunner.query( + `CREATE TYPE "public"."table_schema_change_databasetype_enum" AS ENUM('postgres', 'mysql', 'mysql2', 'oracledb', 'mssql', 'ibmdb2', 'mongodb', 'dynamodb', 'elasticsearch', 'cassandra', 'redis', 'clickhouse', 'agent_postgres', 'agent_mysql', 'agent_oracledb', 'agent_mssql', 'agent_ibmdb2', 'agent_mongodb', 'agent_cassandra', 'agent_redis', 'agent_clickhouse')`, + ); + await queryRunner.query( + `CREATE TABLE "table_schema_change" ("id" uuid NOT NULL DEFAULT uuid_generate_v4(), "connectionId" character varying(38) NOT NULL, "authorId" uuid, "previousChangeId" uuid, "forwardSql" text NOT NULL, "rollbackSql" text, "userModifiedSql" text, "status" "public"."table_schema_change_status_enum" NOT NULL DEFAULT 'PENDING', "changeType" "public"."table_schema_change_changetype_enum" NOT NULL, "targetTableName" character varying(255) NOT NULL, "databaseType" "public"."table_schema_change_databasetype_enum" NOT NULL, "executionError" text, "isReversible" boolean NOT NULL DEFAULT false, "autoRollbackAttempted" boolean NOT NULL DEFAULT false, "autoRollbackSucceeded" boolean NOT NULL DEFAULT false, "userPrompt" text NOT NULL, "aiSummary" text, "aiReasoning" text, "aiModelUsed" character varying(128), "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "appliedAt" TIMESTAMP, "rolledBackAt" TIMESTAMP, CONSTRAINT "PK_ee65b71505c45d00372ff208cd2" PRIMARY KEY ("id"))`, + ); + await queryRunner.query(`CREATE INDEX "IDX_tsc_previous_change" ON "table_schema_change" ("previousChangeId") `); + await queryRunner.query( + `CREATE INDEX "IDX_tsc_connection_created" ON "table_schema_change" ("connectionId", "createdAt") `, + ); + await queryRunner.query( + `ALTER TYPE "public"."table_filters_dynamic_filter_comparator_enum" RENAME TO "table_filters_dynamic_filter_comparator_enum_old"`, + ); + await queryRunner.query( + `CREATE TYPE "public"."table_filters_dynamic_filter_comparator_enum" AS ENUM('startswith', 'endswith', 'gt', 'lt', 'lte', 'gte', 'contains', 'icontains', 'eq', 'empty', 'in', 'between')`, + ); + await queryRunner.query( + `ALTER TABLE "table_filters" ALTER COLUMN "dynamic_filter_comparator" TYPE "public"."table_filters_dynamic_filter_comparator_enum" USING "dynamic_filter_comparator"::"text"::"public"."table_filters_dynamic_filter_comparator_enum"`, + ); + await queryRunner.query(`DROP TYPE "public"."table_filters_dynamic_filter_comparator_enum_old"`); + await queryRunner.query( + `ALTER TABLE "panel" ADD CONSTRAINT "FK_3599e7a2eea197b002732551452" FOREIGN KEY ("connection_id") REFERENCES "connection"("id") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "panel_position" ADD CONSTRAINT "FK_9c17df1164acacbd35ba53ceb0e" FOREIGN KEY ("dashboard_id") REFERENCES "dashboard"("id") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "panel_position" ADD CONSTRAINT "FK_13678f419020d212019508d4d68" FOREIGN KEY ("query_id") REFERENCES "panel"("id") ON DELETE SET NULL ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "table_schema_change" ADD CONSTRAINT "FK_ab6fb01554213543f0a39f7d98e" FOREIGN KEY ("connectionId") REFERENCES "connection"("id") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "table_schema_change" ADD CONSTRAINT "FK_d4a735643602c7e2337770d368b" FOREIGN KEY ("authorId") REFERENCES "user"("id") ON DELETE SET NULL ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "table_schema_change" ADD CONSTRAINT "FK_f15e652a55b856bf9c64c012d00" FOREIGN KEY ("previousChangeId") REFERENCES "table_schema_change"("id") ON DELETE SET NULL ON UPDATE NO ACTION`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "table_schema_change" DROP CONSTRAINT "FK_f15e652a55b856bf9c64c012d00"`); + await queryRunner.query(`ALTER TABLE "table_schema_change" DROP CONSTRAINT "FK_d4a735643602c7e2337770d368b"`); + await queryRunner.query(`ALTER TABLE "table_schema_change" DROP CONSTRAINT "FK_ab6fb01554213543f0a39f7d98e"`); + await queryRunner.query(`ALTER TABLE "panel_position" DROP CONSTRAINT "FK_13678f419020d212019508d4d68"`); + await queryRunner.query(`ALTER TABLE "panel_position" DROP CONSTRAINT "FK_9c17df1164acacbd35ba53ceb0e"`); + await queryRunner.query(`ALTER TABLE "panel" DROP CONSTRAINT "FK_3599e7a2eea197b002732551452"`); + await queryRunner.query( + `CREATE TYPE "public"."table_filters_dynamic_filter_comparator_enum_old" AS ENUM('startswith', 'endswith', 'gt', 'lt', 'lte', 'gte', 'contains', 'icontains', 'eq', 'empty')`, + ); + await queryRunner.query( + `ALTER TABLE "table_filters" ALTER COLUMN "dynamic_filter_comparator" TYPE "public"."table_filters_dynamic_filter_comparator_enum_old" USING "dynamic_filter_comparator"::"text"::"public"."table_filters_dynamic_filter_comparator_enum_old"`, + ); + await queryRunner.query(`DROP TYPE "public"."table_filters_dynamic_filter_comparator_enum"`); + await queryRunner.query( + `ALTER TYPE "public"."table_filters_dynamic_filter_comparator_enum_old" RENAME TO "table_filters_dynamic_filter_comparator_enum"`, + ); + await queryRunner.query(`DROP INDEX "public"."IDX_tsc_connection_created"`); + await queryRunner.query(`DROP INDEX "public"."IDX_tsc_previous_change"`); + await queryRunner.query(`DROP TABLE "table_schema_change"`); + await queryRunner.query(`DROP TYPE "public"."table_schema_change_databasetype_enum"`); + await queryRunner.query(`DROP TYPE "public"."table_schema_change_changetype_enum"`); + await queryRunner.query(`DROP TYPE "public"."table_schema_change_status_enum"`); + await queryRunner.query( + `ALTER TABLE "panel_position" ADD CONSTRAINT "FK_panel_position_query_id" FOREIGN KEY ("query_id") REFERENCES "panel"("id") ON DELETE SET NULL ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "panel_position" ADD CONSTRAINT "FK_panel_position_dashboard_id" FOREIGN KEY ("dashboard_id") REFERENCES "dashboard"("id") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + await queryRunner.query( + `ALTER TABLE "panel" ADD CONSTRAINT "FK_panel_connection_id" FOREIGN KEY ("connection_id") REFERENCES "connection"("id") ON DELETE CASCADE ON UPDATE NO ACTION`, + ); + } +} diff --git a/backend/src/migrations/1776953988756-AddMongoSchemaChangeTypes.ts b/backend/src/migrations/1776953988756-AddMongoSchemaChangeTypes.ts new file mode 100644 index 000000000..37744e4ea --- /dev/null +++ b/backend/src/migrations/1776953988756-AddMongoSchemaChangeTypes.ts @@ -0,0 +1,31 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddMongoSchemaChangeTypes1776953988756 implements MigrationInterface { + name = 'AddMongoSchemaChangeTypes1776953988756'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TYPE "public"."table_schema_change_changetype_enum" RENAME TO "table_schema_change_changetype_enum_old"`, + ); + await queryRunner.query( + `CREATE TYPE "public"."table_schema_change_changetype_enum" AS ENUM('CREATE_TABLE', 'DROP_TABLE', 'ADD_COLUMN', 'DROP_COLUMN', 'ALTER_COLUMN', 'ADD_INDEX', 'DROP_INDEX', 'ADD_FOREIGN_KEY', 'DROP_FOREIGN_KEY', 'ADD_PRIMARY_KEY', 'DROP_PRIMARY_KEY', 'MONGO_CREATE_COLLECTION', 'MONGO_DROP_COLLECTION', 'MONGO_SET_VALIDATOR', 'MONGO_CREATE_INDEX', 'MONGO_DROP_INDEX', 'ROLLBACK', 'OTHER')`, + ); + await queryRunner.query( + `ALTER TABLE "table_schema_change" ALTER COLUMN "changeType" TYPE "public"."table_schema_change_changetype_enum" USING "changeType"::"text"::"public"."table_schema_change_changetype_enum"`, + ); + await queryRunner.query(`DROP TYPE "public"."table_schema_change_changetype_enum_old"`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TYPE "public"."table_schema_change_changetype_enum_old" AS ENUM('CREATE_TABLE', 'DROP_TABLE', 'ADD_COLUMN', 'DROP_COLUMN', 'ALTER_COLUMN', 'ADD_INDEX', 'DROP_INDEX', 'ADD_FOREIGN_KEY', 'DROP_FOREIGN_KEY', 'ADD_PRIMARY_KEY', 'DROP_PRIMARY_KEY', 'ROLLBACK', 'OTHER')`, + ); + await queryRunner.query( + `ALTER TABLE "table_schema_change" ALTER COLUMN "changeType" TYPE "public"."table_schema_change_changetype_enum_old" USING "changeType"::"text"::"public"."table_schema_change_changetype_enum_old"`, + ); + await queryRunner.query(`DROP TYPE "public"."table_schema_change_changetype_enum"`); + await queryRunner.query( + `ALTER TYPE "public"."table_schema_change_changetype_enum_old" RENAME TO "table_schema_change_changetype_enum"`, + ); + } +} diff --git a/backend/src/migrations/1777039182865-AddDynamoDbSchemaChangeTypes.ts b/backend/src/migrations/1777039182865-AddDynamoDbSchemaChangeTypes.ts new file mode 100644 index 000000000..03e4e77af --- /dev/null +++ b/backend/src/migrations/1777039182865-AddDynamoDbSchemaChangeTypes.ts @@ -0,0 +1,31 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddDynamoDbSchemaChangeTypes1777039182865 implements MigrationInterface { + name = 'AddDynamoDbSchemaChangeTypes1777039182865'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TYPE "public"."table_schema_change_changetype_enum" RENAME TO "table_schema_change_changetype_enum_old"`, + ); + await queryRunner.query( + `CREATE TYPE "public"."table_schema_change_changetype_enum" AS ENUM('CREATE_TABLE', 'DROP_TABLE', 'ADD_COLUMN', 'DROP_COLUMN', 'ALTER_COLUMN', 'ADD_INDEX', 'DROP_INDEX', 'ADD_FOREIGN_KEY', 'DROP_FOREIGN_KEY', 'ADD_PRIMARY_KEY', 'DROP_PRIMARY_KEY', 'MONGO_CREATE_COLLECTION', 'MONGO_DROP_COLLECTION', 'MONGO_SET_VALIDATOR', 'MONGO_CREATE_INDEX', 'MONGO_DROP_INDEX', 'DYNAMODB_CREATE_TABLE', 'DYNAMODB_DROP_TABLE', 'DYNAMODB_UPDATE_TABLE', 'DYNAMODB_UPDATE_TTL', 'ROLLBACK', 'OTHER')`, + ); + await queryRunner.query( + `ALTER TABLE "table_schema_change" ALTER COLUMN "changeType" TYPE "public"."table_schema_change_changetype_enum" USING "changeType"::"text"::"public"."table_schema_change_changetype_enum"`, + ); + await queryRunner.query(`DROP TYPE "public"."table_schema_change_changetype_enum_old"`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TYPE "public"."table_schema_change_changetype_enum_old" AS ENUM('ADD_COLUMN', 'ADD_FOREIGN_KEY', 'ADD_INDEX', 'ADD_PRIMARY_KEY', 'ALTER_COLUMN', 'CREATE_TABLE', 'DROP_COLUMN', 'DROP_FOREIGN_KEY', 'DROP_INDEX', 'DROP_PRIMARY_KEY', 'DROP_TABLE', 'MONGO_CREATE_COLLECTION', 'MONGO_CREATE_INDEX', 'MONGO_DROP_COLLECTION', 'MONGO_DROP_INDEX', 'MONGO_SET_VALIDATOR', 'OTHER', 'ROLLBACK')`, + ); + await queryRunner.query( + `ALTER TABLE "table_schema_change" ALTER COLUMN "changeType" TYPE "public"."table_schema_change_changetype_enum_old" USING "changeType"::"text"::"public"."table_schema_change_changetype_enum_old"`, + ); + await queryRunner.query(`DROP TYPE "public"."table_schema_change_changetype_enum"`); + await queryRunner.query( + `ALTER TYPE "public"."table_schema_change_changetype_enum_old" RENAME TO "table_schema_change_changetype_enum"`, + ); + } +} diff --git a/backend/src/migrations/1777272567941-AddElasticsearchSchemaChangeTypes.ts b/backend/src/migrations/1777272567941-AddElasticsearchSchemaChangeTypes.ts new file mode 100644 index 000000000..7ec8c4125 --- /dev/null +++ b/backend/src/migrations/1777272567941-AddElasticsearchSchemaChangeTypes.ts @@ -0,0 +1,31 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddElasticsearchSchemaChangeTypes1777272567941 implements MigrationInterface { + name = 'AddElasticsearchSchemaChangeTypes1777272567941'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TYPE "public"."table_schema_change_changetype_enum" RENAME TO "table_schema_change_changetype_enum_old"`, + ); + await queryRunner.query( + `CREATE TYPE "public"."table_schema_change_changetype_enum" AS ENUM('CREATE_TABLE', 'DROP_TABLE', 'ADD_COLUMN', 'DROP_COLUMN', 'ALTER_COLUMN', 'ADD_INDEX', 'DROP_INDEX', 'ADD_FOREIGN_KEY', 'DROP_FOREIGN_KEY', 'ADD_PRIMARY_KEY', 'DROP_PRIMARY_KEY', 'MONGO_CREATE_COLLECTION', 'MONGO_DROP_COLLECTION', 'MONGO_SET_VALIDATOR', 'MONGO_CREATE_INDEX', 'MONGO_DROP_INDEX', 'DYNAMODB_CREATE_TABLE', 'DYNAMODB_DROP_TABLE', 'DYNAMODB_UPDATE_TABLE', 'DYNAMODB_UPDATE_TTL', 'ELASTICSEARCH_CREATE_INDEX', 'ELASTICSEARCH_DELETE_INDEX', 'ELASTICSEARCH_UPDATE_MAPPING', 'ROLLBACK', 'OTHER')`, + ); + await queryRunner.query( + `ALTER TABLE "table_schema_change" ALTER COLUMN "changeType" TYPE "public"."table_schema_change_changetype_enum" USING "changeType"::"text"::"public"."table_schema_change_changetype_enum"`, + ); + await queryRunner.query(`DROP TYPE "public"."table_schema_change_changetype_enum_old"`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TYPE "public"."table_schema_change_changetype_enum_old" AS ENUM('ADD_COLUMN', 'ADD_FOREIGN_KEY', 'ADD_INDEX', 'ADD_PRIMARY_KEY', 'ALTER_COLUMN', 'CREATE_TABLE', 'DROP_COLUMN', 'DROP_FOREIGN_KEY', 'DROP_INDEX', 'DROP_PRIMARY_KEY', 'DROP_TABLE', 'DYNAMODB_CREATE_TABLE', 'DYNAMODB_DROP_TABLE', 'DYNAMODB_UPDATE_TABLE', 'DYNAMODB_UPDATE_TTL', 'MONGO_CREATE_COLLECTION', 'MONGO_CREATE_INDEX', 'MONGO_DROP_COLLECTION', 'MONGO_DROP_INDEX', 'MONGO_SET_VALIDATOR', 'OTHER', 'ROLLBACK')`, + ); + await queryRunner.query( + `ALTER TABLE "table_schema_change" ALTER COLUMN "changeType" TYPE "public"."table_schema_change_changetype_enum_old" USING "changeType"::"text"::"public"."table_schema_change_changetype_enum_old"`, + ); + await queryRunner.query(`DROP TYPE "public"."table_schema_change_changetype_enum"`); + await queryRunner.query( + `ALTER TYPE "public"."table_schema_change_changetype_enum_old" RENAME TO "table_schema_change_changetype_enum"`, + ); + } +} diff --git a/backend/src/migrations/1777295364595-AddBatchColumnsToTableSchemaChange.ts b/backend/src/migrations/1777295364595-AddBatchColumnsToTableSchemaChange.ts new file mode 100644 index 000000000..060ed84e7 --- /dev/null +++ b/backend/src/migrations/1777295364595-AddBatchColumnsToTableSchemaChange.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddBatchColumnsToTableSchemaChange1777295364595 implements MigrationInterface { + name = 'AddBatchColumnsToTableSchemaChange1777295364595'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "table_schema_change" ADD "batchId" uuid`); + await queryRunner.query(`ALTER TABLE "table_schema_change" ADD "orderInBatch" integer NOT NULL DEFAULT '0'`); + await queryRunner.query(`CREATE INDEX "IDX_tsc_batch_order" ON "table_schema_change" ("batchId", "orderInBatch") `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "public"."IDX_tsc_batch_order"`); + await queryRunner.query(`ALTER TABLE "table_schema_change" DROP COLUMN "orderInBatch"`); + await queryRunner.query(`ALTER TABLE "table_schema_change" DROP COLUMN "batchId"`); + } +} diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-cassandra-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-cassandra-e2e.test.ts new file mode 100644 index 000000000..4cda393ba --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-cassandra-e2e.test.ts @@ -0,0 +1,472 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { faker } from '@faker-js/faker'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import * as cassandra from 'cassandra-driver'; +import { ValidationError } from 'class-validator'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { AICoreService } from '../../../src/ai-core/index.js'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { WinstonLogger } from '../../../src/entities/logging/winston-logger.js'; +import { + SchemaChangeStatusEnum, + SchemaChangeTypeEnum, +} from '../../../src/entities/table-schema/table-schema-change-enums.js'; +import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { getTestData } from '../../utils/get-test-data.js'; +import { + createInitialTestUser, + registerUserAndReturnUserInfo, +} from '../../utils/register-user-and-return-user-info.js'; +import { setSaasEnvVariable } from '../../utils/set-saas-env-variable.js'; +import { TestUtils } from '../../utils/test.utils.js'; + +interface ProposedChange { + forwardSql: string; + rollbackSql: string; + changeType: SchemaChangeTypeEnum; + targetTableName: string; + isReversible: boolean; + summary: string; + reasoning: string; +} + +let nextProposal: ProposedChange | null = null; + +function createProposalStream(proposal: ProposedChange) { + return { + *[Symbol.asyncIterator]() { + yield { + type: 'tool_call', + toolCall: { + id: faker.string.uuid(), + name: 'proposeSchemaChange', + arguments: { proposals: [proposal] }, + }, + responseId: faker.string.uuid(), + }; + }, + }; +} + +const mockAICoreService = { + streamChatWithToolsAndProvider: async () => { + if (!nextProposal) throw new Error('Test must set nextProposal.'); + return createProposalStream(nextProposal); + }, + complete: async () => 'mocked', + chat: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChat: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + chatWithTools: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithTools: async () => createProposalStream(nextProposal!), + chatWithToolsAndProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithProvider: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + completeWithProvider: async () => 'mocked', + chatWithProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueAfterToolCall: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueStreamingAfterToolCall: async () => createProposalStream(nextProposal!), + getDefaultProvider: () => 'openai', + setDefaultProvider: () => {}, + getAvailableProviders: () => [], +}; + +const mockFactory = new MockFactory(); +let app: INestApplication; +let _testUtils: TestUtils; +const createdTables: string[] = []; + +function cassandraConnectionParams() { + return getTestData(mockFactory).cassandraTestConnection; +} + +function buildCassandraClient(useKeyspace: boolean): cassandra.Client { + const params = cassandraConnectionParams(); + return new cassandra.Client({ + contactPoints: [params.host], + localDataCenter: params.dataCenter, + keyspace: useKeyspace ? params.database : undefined, + authProvider: new cassandra.auth.PlainTextAuthProvider(params.username, params.password), + protocolOptions: { port: params.port }, + }); +} + +async function withCassandra(fn: (client: cassandra.Client) => Promise, useKeyspace = true): Promise { + const client = buildCassandraClient(useKeyspace); + try { + await client.connect(); + return await fn(client); + } finally { + await client.shutdown().catch(() => undefined); + } +} + +async function ensureKeyspace(): Promise { + const params = cassandraConnectionParams(); + await withCassandra(async (client) => { + await client.execute( + `CREATE KEYSPACE IF NOT EXISTS ${params.database} WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1}`, + ); + }, false); +} + +async function tableExists(tableName: string): Promise { + const params = cassandraConnectionParams(); + return withCassandra(async (client) => { + const result = await client.execute( + 'SELECT table_name FROM system_schema.tables WHERE keyspace_name = ? AND table_name = ?', + [params.database, tableName.toLowerCase()], + { prepare: true }, + ); + return result.rows.length > 0; + }, false); +} + +async function columnExists(tableName: string, columnName: string): Promise { + const params = cassandraConnectionParams(); + return withCassandra(async (client) => { + const result = await client.execute( + 'SELECT column_name FROM system_schema.columns WHERE keyspace_name = ? AND table_name = ? AND column_name = ?', + [params.database, tableName.toLowerCase(), columnName.toLowerCase()], + { prepare: true }, + ); + return result.rows.length > 0; + }, false); +} + +async function dropTableIfExists(tableName: string): Promise { + await withCassandra(async (client) => { + await client.execute(`DROP TABLE IF EXISTS ${tableName}`); + }); +} + +async function seedTable(tableName: string, schema: string): Promise { + await withCassandra(async (client) => { + await client.execute(`CREATE TABLE IF NOT EXISTS ${tableName} (${schema})`); + }); +} + +test.before(async () => { + setSaasEnvVariable(); + await ensureKeyspace(); + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }) + .overrideProvider(AICoreService) + .useValue(mockAICoreService) + .compile(); + + _testUtils = moduleFixture.get(TestUtils); + app = moduleFixture.createNestApplication(); + app.use(cookieParser()); + app.useGlobalFilters(new AllExceptionsFilter(app.get(WinstonLogger))); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + await createInitialTestUser(app); + app.getHttpServer().listen(0); +}); + +test.after(async () => { + try { + for (const name of createdTables) { + await dropTableIfExists(name); + } + await Cacher.clearAllCache(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +async function createConnection(token: string): Promise { + const connectionToTestDB = cassandraConnectionParams(); + const resp = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + if (resp.status !== 201) { + throw new Error(`Failed to create connection: ${resp.status} ${resp.text}`); + } + return JSON.parse(resp.text).id; +} + +function randomTableName(prefix: string): string { + return `${prefix}_${faker.string.alphanumeric(6).toLowerCase()}`; +} + +test.serial('Cassandra: generate → approve creates a table', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ca_t'); + createdTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE ${tableName} (id UUID PRIMARY KEY, name TEXT)`, + rollbackSql: `DROP TABLE ${tableName}`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: `Create ${tableName}`, + reasoning: 'basic create', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: `create ${tableName}` }); + t.is(generateResp.status, 201); + const change = JSON.parse(generateResp.text).changes[0]; + t.is(change.status, SchemaChangeStatusEnum.PENDING); + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${change.id}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).status, SchemaChangeStatusEnum.APPLIED); + t.true(await tableExists(tableName)); +}); + +test.serial('Cassandra: generate → approve → rollback drops the table', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ca_rb'); + createdTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE ${tableName} (id UUID PRIMARY KEY)`, + rollbackSql: `DROP TABLE ${tableName}`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'rollback test', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create then rollback' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + await request(app.getHttpServer()).post(`/table-schema/change/${changeId}/approve`).set('Cookie', token).send({}); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + t.is(JSON.parse(rollbackResp.text).status, SchemaChangeStatusEnum.ROLLED_BACK); + t.false(await tableExists(tableName)); + + const listResp = await request(app.getHttpServer()).get(`/table-schema/${connectionId}/changes`).set('Cookie', token); + const list = JSON.parse(listResp.text); + const auditRow = list.data.find( + (c: any) => c.changeType === SchemaChangeTypeEnum.ROLLBACK && c.previousChangeId === changeId, + ); + t.truthy(auditRow, 'expected linked rollback audit row'); +}); + +test.serial('Cassandra: ADD column → approve adds the column; rollback removes it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ca_addcol'); + createdTables.push(tableName); + + await seedTable(tableName, 'id UUID PRIMARY KEY, name TEXT'); + + nextProposal = { + forwardSql: `ALTER TABLE ${tableName} ADD phone TEXT`, + rollbackSql: `ALTER TABLE ${tableName} DROP phone`, + changeType: SchemaChangeTypeEnum.ADD_COLUMN, + targetTableName: tableName, + isReversible: true, + summary: 'add phone', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add phone column' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.true(await columnExists(tableName, 'phone')); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + t.false(await columnExists(tableName, 'phone')); +}); + +test.serial('Cassandra: DROP TABLE requires confirmedDestructive', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ca_drop'); + createdTables.push(tableName); + + await seedTable(tableName, 'id UUID PRIMARY KEY'); + + nextProposal = { + forwardSql: `DROP TABLE ${tableName}`, + rollbackSql: `CREATE TABLE ${tableName} (id UUID PRIMARY KEY)`, + changeType: SchemaChangeTypeEnum.DROP_TABLE, + targetTableName: tableName, + isReversible: false, + summary: 'destructive', + reasoning: 'user wants to drop', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'drop it' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const firstApprove = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(firstApprove.status, 400); + t.is(firstApprove.body?.type, 'destructive_confirmation_required'); + t.true(await tableExists(tableName), 'table must still exist'); + + const secondApprove = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ confirmedDestructive: true }); + t.is(secondApprove.status, 200); + t.is(JSON.parse(secondApprove.text).status, SchemaChangeStatusEnum.APPLIED); + t.false(await tableExists(tableName)); +}); + +test.serial('Cassandra: invalid CQL marks FAILED and attempts auto-rollback', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ca_bad'); + createdTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE ${tableName} (id NOTATYPE PRIMARY KEY)`, + rollbackSql: `DROP TABLE IF EXISTS ${tableName}`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'bad', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'bad sql' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 400); + + const getResp = await request(app.getHttpServer()).get(`/table-schema/change/${changeId}`).set('Cookie', token); + const record = JSON.parse(getResp.text); + t.is(record.status, SchemaChangeStatusEnum.FAILED); + t.true(record.autoRollbackAttempted); + t.truthy(record.executionError); + t.false(await tableExists(tableName)); +}); + +test.serial('Cassandra: userModifiedSql is validated and applied in place of AI SQL', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ca_um'); + createdTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE ${tableName} (id UUID PRIMARY KEY, foo TEXT)`, + rollbackSql: `DROP TABLE ${tableName}`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'original', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const editedSql = `CREATE TABLE ${tableName} (id UUID PRIMARY KEY, bar TEXT)`; + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: editedSql }); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).userModifiedSql, editedSql); + t.true(await columnExists(tableName, 'bar')); + t.false(await columnExists(tableName, 'foo')); +}); + +test.serial('Cassandra: userModifiedSql with a forbidden construct is rejected', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ca_umbad'); + + nextProposal = { + forwardSql: `CREATE TABLE ${tableName} (id UUID PRIMARY KEY)`, + rollbackSql: `DROP TABLE ${tableName}`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'valid original', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: `GRANT ALL ON ${tableName} TO cassandra` }); + t.is(approveResp.status, 400); + + const getResp = await request(app.getHttpServer()).get(`/table-schema/change/${changeId}`).set('Cookie', token); + t.is(JSON.parse(getResp.text).status, SchemaChangeStatusEnum.PENDING); + t.false(await tableExists(tableName)); +}); diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-clickhouse-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-clickhouse-e2e.test.ts new file mode 100644 index 000000000..0a4223cac --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-clickhouse-e2e.test.ts @@ -0,0 +1,537 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { ClickHouseClient, createClient } from '@clickhouse/client'; +import { faker } from '@faker-js/faker'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import { ValidationError } from 'class-validator'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { AICoreService } from '../../../src/ai-core/index.js'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { WinstonLogger } from '../../../src/entities/logging/winston-logger.js'; +import { + SchemaChangeStatusEnum, + SchemaChangeTypeEnum, +} from '../../../src/entities/table-schema/table-schema-change-enums.js'; +import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { getTestData } from '../../utils/get-test-data.js'; +import { + createInitialTestUser, + registerUserAndReturnUserInfo, +} from '../../utils/register-user-and-return-user-info.js'; +import { setSaasEnvVariable } from '../../utils/set-saas-env-variable.js'; +import { TestUtils } from '../../utils/test.utils.js'; + +interface ProposedChange { + forwardSql: string; + rollbackSql: string; + changeType: SchemaChangeTypeEnum; + targetTableName: string; + isReversible: boolean; + summary: string; + reasoning: string; +} + +let nextProposal: ProposedChange | null = null; + +function createProposalStream(proposal: ProposedChange) { + return { + *[Symbol.asyncIterator]() { + yield { + type: 'tool_call', + toolCall: { + id: faker.string.uuid(), + name: 'proposeSchemaChange', + arguments: { proposals: [proposal] }, + }, + responseId: faker.string.uuid(), + }; + }, + }; +} + +const mockAICoreService = { + streamChatWithToolsAndProvider: async () => { + if (!nextProposal) throw new Error('Test must set nextProposal.'); + return createProposalStream(nextProposal); + }, + complete: async () => 'mocked', + chat: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChat: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + chatWithTools: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithTools: async () => createProposalStream(nextProposal!), + chatWithToolsAndProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithProvider: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + completeWithProvider: async () => 'mocked', + chatWithProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueAfterToolCall: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueStreamingAfterToolCall: async () => createProposalStream(nextProposal!), + getDefaultProvider: () => 'openai', + setDefaultProvider: () => {}, + getAvailableProviders: () => [], +}; + +const mockFactory = new MockFactory(); +let app: INestApplication; +let _testUtils: TestUtils; +const createdTables: string[] = []; + +function clickHouseConnectionParams() { + return getTestData(mockFactory).clickhouseTestConnection; +} + +function buildClickHouseClient(): ClickHouseClient { + const params = clickHouseConnectionParams(); + return createClient({ + url: `http://${params.host}:${params.port}`, + username: params.username ?? 'default', + password: params.password ?? '', + database: params.database ?? 'default', + }); +} + +async function withClickHouse(fn: (client: ClickHouseClient) => Promise): Promise { + const client = buildClickHouseClient(); + try { + return await fn(client); + } finally { + await client.close(); + } +} + +async function tableExists(tableName: string): Promise { + return withClickHouse(async (client) => { + const params = clickHouseConnectionParams(); + const result = await client.query({ + query: `SELECT count() AS c FROM system.tables WHERE database = {db:String} AND name = {t:String}`, + query_params: { db: params.database ?? 'default', t: tableName }, + format: 'JSONEachRow', + }); + const rows = (await result.json()) as Array<{ c: number | string }>; + return Number(rows[0]?.c ?? 0) > 0; + }); +} + +async function columnExists(tableName: string, columnName: string): Promise { + return withClickHouse(async (client) => { + const params = clickHouseConnectionParams(); + const result = await client.query({ + query: `SELECT count() AS c FROM system.columns WHERE database = {db:String} AND table = {t:String} AND name = {c:String}`, + query_params: { db: params.database ?? 'default', t: tableName, c: columnName }, + format: 'JSONEachRow', + }); + const rows = (await result.json()) as Array<{ c: number | string }>; + return Number(rows[0]?.c ?? 0) > 0; + }); +} + +async function getColumnType(tableName: string, columnName: string): Promise { + return withClickHouse(async (client) => { + const params = clickHouseConnectionParams(); + const result = await client.query({ + query: `SELECT type FROM system.columns WHERE database = {db:String} AND table = {t:String} AND name = {c:String}`, + query_params: { db: params.database ?? 'default', t: tableName, c: columnName }, + format: 'JSONEachRow', + }); + const rows = (await result.json()) as Array<{ type: string }>; + return rows[0]?.type ?? null; + }); +} + +async function dropTableIfExists(tableName: string): Promise { + await withClickHouse(async (client) => { + await client.command({ query: `DROP TABLE IF EXISTS ${tableName}` }); + }); +} + +async function seedTable(tableName: string, orderByColumn: string, schema: string): Promise { + await withClickHouse(async (client) => { + await client.command({ + query: `CREATE TABLE IF NOT EXISTS ${tableName} (${schema}) ENGINE = MergeTree() ORDER BY ${orderByColumn}`, + }); + }); +} + +test.before(async () => { + setSaasEnvVariable(); + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }) + .overrideProvider(AICoreService) + .useValue(mockAICoreService) + .compile(); + + _testUtils = moduleFixture.get(TestUtils); + app = moduleFixture.createNestApplication(); + app.use(cookieParser()); + app.useGlobalFilters(new AllExceptionsFilter(app.get(WinstonLogger))); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + await createInitialTestUser(app); + app.getHttpServer().listen(0); +}); + +test.after(async () => { + try { + for (const name of createdTables) { + await dropTableIfExists(name); + } + await Cacher.clearAllCache(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +async function createConnection(token: string): Promise { + const connectionToTestDB = clickHouseConnectionParams(); + const resp = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + if (resp.status !== 201) { + throw new Error(`Failed to create connection: ${resp.status} ${resp.text}`); + } + return JSON.parse(resp.text).id; +} + +function randomTableName(prefix: string): string { + return `${prefix}_${faker.string.alphanumeric(6).toLowerCase()}`; +} + +test.serial('ClickHouse: generate → approve creates a MergeTree table', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ch_t'); + createdTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE ${tableName} (id UInt32, name String) ENGINE = MergeTree() ORDER BY id`, + rollbackSql: `DROP TABLE ${tableName}`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: `Create ${tableName}`, + reasoning: 'basic create', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: `create ${tableName}` }); + t.is(generateResp.status, 201); + const change = JSON.parse(generateResp.text).changes[0]; + t.is(change.status, SchemaChangeStatusEnum.PENDING); + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${change.id}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).status, SchemaChangeStatusEnum.APPLIED); + t.true(await tableExists(tableName)); +}); + +test.serial('ClickHouse: generate → approve → rollback drops the table', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ch_rb'); + createdTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE ${tableName} (id UInt32) ENGINE = MergeTree() ORDER BY id`, + rollbackSql: `DROP TABLE ${tableName}`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'rollback test', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create then rollback' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + await request(app.getHttpServer()).post(`/table-schema/change/${changeId}/approve`).set('Cookie', token).send({}); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + t.is(JSON.parse(rollbackResp.text).status, SchemaChangeStatusEnum.ROLLED_BACK); + t.false(await tableExists(tableName)); + + const listResp = await request(app.getHttpServer()).get(`/table-schema/${connectionId}/changes`).set('Cookie', token); + const list = JSON.parse(listResp.text); + const auditRow = list.data.find( + (c: any) => c.changeType === SchemaChangeTypeEnum.ROLLBACK && c.previousChangeId === changeId, + ); + t.truthy(auditRow, 'expected linked rollback audit row'); +}); + +test.serial('ClickHouse: ADD COLUMN → approve adds the column; rollback removes it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ch_addcol'); + createdTables.push(tableName); + + await seedTable(tableName, 'id', 'id UInt32, name String'); + + nextProposal = { + forwardSql: `ALTER TABLE ${tableName} ADD COLUMN phone String`, + rollbackSql: `ALTER TABLE ${tableName} DROP COLUMN phone`, + changeType: SchemaChangeTypeEnum.ADD_COLUMN, + targetTableName: tableName, + isReversible: true, + summary: 'add phone', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add phone column' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.true(await columnExists(tableName, 'phone')); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + t.false(await columnExists(tableName, 'phone')); +}); + +test.serial('ClickHouse: DROP TABLE requires confirmedDestructive', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ch_drop'); + createdTables.push(tableName); + + await seedTable(tableName, 'id', 'id UInt32'); + + nextProposal = { + forwardSql: `DROP TABLE ${tableName}`, + rollbackSql: `CREATE TABLE ${tableName} (id UInt32) ENGINE = MergeTree() ORDER BY id`, + changeType: SchemaChangeTypeEnum.DROP_TABLE, + targetTableName: tableName, + isReversible: false, + summary: 'destructive', + reasoning: 'user wants to drop', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'drop it' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const firstApprove = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(firstApprove.status, 400); + t.is(firstApprove.body?.type, 'destructive_confirmation_required'); + t.true(await tableExists(tableName), 'table must still exist'); + + const secondApprove = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ confirmedDestructive: true }); + t.is(secondApprove.status, 200); + t.is(JSON.parse(secondApprove.text).status, SchemaChangeStatusEnum.APPLIED); + t.false(await tableExists(tableName)); +}); + +test.serial('ClickHouse: ALTER COLUMN MODIFY preserves row data when widening', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ch_alt'); + createdTables.push(tableName); + + // Seed a FixedString(16) column and populate; then widen to a variable-length String. + await withClickHouse(async (client) => { + await client.command({ + query: `CREATE TABLE ${tableName} (id UInt32, name FixedString(16)) ENGINE = MergeTree() ORDER BY id`, + }); + await client.insert({ + table: tableName, + values: [ + { id: 1, name: 'alice' }, + { id: 2, name: 'bob' }, + { id: 3, name: 'carol' }, + ], + format: 'JSONEachRow', + }); + }); + + nextProposal = { + forwardSql: `ALTER TABLE ${tableName} MODIFY COLUMN name String`, + rollbackSql: `ALTER TABLE ${tableName} MODIFY COLUMN name FixedString(16)`, + changeType: SchemaChangeTypeEnum.ALTER_COLUMN, + targetTableName: tableName, + isReversible: true, + summary: 'widen name', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'widen name' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.is(await getColumnType(tableName, 'name'), 'String'); + + const rows = await withClickHouse(async (client) => { + const result = await client.query({ + query: `SELECT id, name FROM ${tableName} ORDER BY id`, + format: 'JSONEachRow', + }); + return (await result.json()) as Array<{ id: number; name: string }>; + }); + // FixedString pads with NUL bytes on insert; trim them when asserting content. + t.deepEqual( + rows.map((r) => r.name.replace(/\0+$/, '')), + ['alice', 'bob', 'carol'], + ); +}); + +test.serial('ClickHouse: invalid SQL marks FAILED and attempts auto-rollback', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ch_bad'); + createdTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE ${tableName} (id NOTATYPE) ENGINE = MergeTree() ORDER BY id`, + rollbackSql: `DROP TABLE IF EXISTS ${tableName}`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'bad', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'bad sql' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 400); + + const getResp = await request(app.getHttpServer()).get(`/table-schema/change/${changeId}`).set('Cookie', token); + const record = JSON.parse(getResp.text); + t.is(record.status, SchemaChangeStatusEnum.FAILED); + t.true(record.autoRollbackAttempted); + t.truthy(record.executionError); + t.false(await tableExists(tableName)); +}); + +test.serial('ClickHouse: userModifiedSql is validated and applied in place of AI SQL', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ch_um'); + createdTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE ${tableName} (id UInt32, foo String) ENGINE = MergeTree() ORDER BY id`, + rollbackSql: `DROP TABLE ${tableName}`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'original', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const editedSql = `CREATE TABLE ${tableName} (id UInt32, bar String) ENGINE = MergeTree() ORDER BY id`; + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: editedSql }); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).userModifiedSql, editedSql); + t.true(await columnExists(tableName, 'bar')); + t.false(await columnExists(tableName, 'foo')); +}); + +test.serial('ClickHouse: userModifiedSql with a forbidden construct is rejected', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ch_umbad'); + + nextProposal = { + forwardSql: `CREATE TABLE ${tableName} (id UInt32) ENGINE = MergeTree() ORDER BY id`, + rollbackSql: `DROP TABLE ${tableName}`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'valid original', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: `GRANT ALL ON ${tableName} TO default` }); + t.is(approveResp.status, 400); + + const getResp = await request(app.getHttpServer()).get(`/table-schema/change/${changeId}`).set('Cookie', token); + t.is(JSON.parse(getResp.text).status, SchemaChangeStatusEnum.PENDING); + t.false(await tableExists(tableName)); +}); diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-dynamodb-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-dynamodb-e2e.test.ts new file mode 100644 index 000000000..c4b3662ab --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-dynamodb-e2e.test.ts @@ -0,0 +1,691 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { + AttributeDefinition, + CreateTableCommand, + DeleteTableCommand, + DescribeTableCommand, + DescribeTimeToLiveCommand, + DynamoDB, + KeySchemaElement, +} from '@aws-sdk/client-dynamodb'; +import { faker } from '@faker-js/faker'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import { ValidationError } from 'class-validator'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { AICoreService } from '../../../src/ai-core/index.js'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { WinstonLogger } from '../../../src/entities/logging/winston-logger.js'; +import { + SchemaChangeStatusEnum, + SchemaChangeTypeEnum, +} from '../../../src/entities/table-schema/table-schema-change-enums.js'; +import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { getTestData } from '../../utils/get-test-data.js'; +import { + createInitialTestUser, + registerUserAndReturnUserInfo, +} from '../../utils/register-user-and-return-user-info.js'; +import { setSaasEnvVariable } from '../../utils/set-saas-env-variable.js'; +import { TestUtils } from '../../utils/test.utils.js'; + +interface DynamoProposedChange { + forwardOp: string; + rollbackOp: string; + changeType: SchemaChangeTypeEnum; + targetTableName: string; + isReversible: boolean; + summary: string; + reasoning: string; +} + +let nextProposal: DynamoProposedChange | null = null; + +function createDynamoProposalStream(proposal: DynamoProposedChange) { + return { + *[Symbol.asyncIterator]() { + yield { + type: 'tool_call', + toolCall: { + id: faker.string.uuid(), + name: 'proposeDynamoDbSchemaChange', + arguments: { proposals: [proposal] }, + }, + responseId: faker.string.uuid(), + }; + }, + }; +} + +const mockAICoreService = { + streamChatWithToolsAndProvider: async () => { + if (!nextProposal) throw new Error('Test must set nextProposal.'); + return createDynamoProposalStream(nextProposal); + }, + complete: async () => 'mocked', + chat: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChat: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + chatWithTools: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithTools: async () => createDynamoProposalStream(nextProposal!), + chatWithToolsAndProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithProvider: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + completeWithProvider: async () => 'mocked', + chatWithProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueAfterToolCall: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueStreamingAfterToolCall: async () => createDynamoProposalStream(nextProposal!), + getDefaultProvider: () => 'openai', + setDefaultProvider: () => {}, + getAvailableProviders: () => [], +}; + +const mockFactory = new MockFactory(); +let app: INestApplication; +let _testUtils: TestUtils; +const createdTables: string[] = []; + +function dynamoConnectionParams() { + return getTestData(mockFactory).dynamoDBConnection; +} + +function buildDynamoDbClient(): DynamoDB { + const params = dynamoConnectionParams(); + return new DynamoDB({ + endpoint: params.host, + region: 'localhost', + credentials: { + accessKeyId: params.username!, + secretAccessKey: params.password!, + }, + }); +} + +async function tableExists(tableName: string): Promise { + const client = buildDynamoDbClient(); + try { + await client.send(new DescribeTableCommand({ TableName: tableName })); + return true; + } catch (err) { + if ((err as Error).name === 'ResourceNotFoundException') return false; + throw err; + } finally { + client.destroy(); + } +} + +async function describeTable(tableName: string) { + const client = buildDynamoDbClient(); + try { + const resp = await client.send(new DescribeTableCommand({ TableName: tableName })); + return resp.Table; + } finally { + client.destroy(); + } +} + +async function describeTtl(tableName: string) { + const client = buildDynamoDbClient(); + try { + const resp = await client.send(new DescribeTimeToLiveCommand({ TableName: tableName })); + return resp.TimeToLiveDescription; + } finally { + client.destroy(); + } +} + +async function seedTable( + tableName: string, + keySchema: KeySchemaElement[], + attributeDefinitions: AttributeDefinition[], +): Promise { + const client = buildDynamoDbClient(); + try { + await client.send( + new CreateTableCommand({ + TableName: tableName, + KeySchema: keySchema, + AttributeDefinitions: attributeDefinitions, + BillingMode: 'PAY_PER_REQUEST', + }), + ); + } finally { + client.destroy(); + } +} + +async function dropTableIfExists(tableName: string): Promise { + const client = buildDynamoDbClient(); + try { + await client.send(new DeleteTableCommand({ TableName: tableName })); + } catch (err) { + if ((err as Error).name !== 'ResourceNotFoundException') { + console.warn(`Could not drop dynamodb table ${tableName}: ${(err as Error).message}`); + } + } finally { + client.destroy(); + } +} + +test.before(async () => { + setSaasEnvVariable(); + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }) + .overrideProvider(AICoreService) + .useValue(mockAICoreService) + .compile(); + + _testUtils = moduleFixture.get(TestUtils); + app = moduleFixture.createNestApplication(); + app.use(cookieParser()); + app.useGlobalFilters(new AllExceptionsFilter(app.get(WinstonLogger))); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + await createInitialTestUser(app); + app.getHttpServer().listen(0); +}); + +test.after(async () => { + try { + for (const name of createdTables) { + await dropTableIfExists(name); + } + await Cacher.clearAllCache(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +async function createConnection(token: string): Promise { + const connectionToTestDB = dynamoConnectionParams(); + const resp = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + if (resp.status !== 201) { + throw new Error(`Failed to create connection: ${resp.status} ${resp.text}`); + } + return JSON.parse(resp.text).id; +} + +function randomTableName(prefix: string): string { + return `${prefix}_${faker.string.alphanumeric(6).toLowerCase()}`; +} + +test.serial('DynamoDB: generate → approve creates a table', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ddb_c'); + createdTables.push(tableName); + + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'createTable', + tableName, + attributeDefinitions: [{ attributeName: 'id', attributeType: 'S' }], + keySchema: [{ attributeName: 'id', keyType: 'HASH' }], + billingMode: 'PAY_PER_REQUEST', + }), + rollbackOp: JSON.stringify({ operation: 'deleteTable', tableName }), + changeType: SchemaChangeTypeEnum.DYNAMODB_CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: `Create ${tableName}`, + reasoning: 'basic', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: `create table ${tableName}` }); + t.is(generateResp.status, 201); + const change = JSON.parse(generateResp.text).changes[0]; + t.is(change.status, SchemaChangeStatusEnum.PENDING); + t.is(change.changeType, SchemaChangeTypeEnum.DYNAMODB_CREATE_TABLE); + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${change.id}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).status, SchemaChangeStatusEnum.APPLIED); + t.true(await tableExists(tableName)); +}); + +test.serial('DynamoDB: createTable → rollback drops the table', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ddb_rb'); + createdTables.push(tableName); + + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'createTable', + tableName, + attributeDefinitions: [{ attributeName: 'id', attributeType: 'N' }], + keySchema: [{ attributeName: 'id', keyType: 'HASH' }], + }), + rollbackOp: JSON.stringify({ operation: 'deleteTable', tableName }), + changeType: SchemaChangeTypeEnum.DYNAMODB_CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'rollback test', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create then rollback' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + await request(app.getHttpServer()).post(`/table-schema/change/${changeId}/approve`).set('Cookie', token).send({}); + t.true(await tableExists(tableName)); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + t.is(JSON.parse(rollbackResp.text).status, SchemaChangeStatusEnum.ROLLED_BACK); + t.false(await tableExists(tableName)); + + const listResp = await request(app.getHttpServer()).get(`/table-schema/${connectionId}/changes`).set('Cookie', token); + const list = JSON.parse(listResp.text); + const auditRow = list.data.find( + (c: any) => c.changeType === SchemaChangeTypeEnum.ROLLBACK && c.previousChangeId === changeId, + ); + t.truthy(auditRow, 'expected linked rollback audit row'); +}); + +test.serial('DynamoDB: deleteTable requires confirmedDestructive', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ddb_drop'); + createdTables.push(tableName); + + await seedTable(tableName, [{ AttributeName: 'id', KeyType: 'HASH' }], [{ AttributeName: 'id', AttributeType: 'S' }]); + + nextProposal = { + forwardOp: JSON.stringify({ operation: 'deleteTable', tableName }), + rollbackOp: JSON.stringify({ + operation: 'createTable', + tableName, + attributeDefinitions: [{ attributeName: 'id', attributeType: 'S' }], + keySchema: [{ attributeName: 'id', keyType: 'HASH' }], + billingMode: 'PAY_PER_REQUEST', + }), + changeType: SchemaChangeTypeEnum.DYNAMODB_DROP_TABLE, + targetTableName: tableName, + isReversible: false, + summary: 'drop', + reasoning: 'destructive', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'drop table' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const firstApprove = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(firstApprove.status, 400); + t.is(firstApprove.body?.type, 'destructive_confirmation_required'); + t.true(await tableExists(tableName)); + + const secondApprove = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ confirmedDestructive: true }); + t.is(secondApprove.status, 200); + t.is(JSON.parse(secondApprove.text).status, SchemaChangeStatusEnum.APPLIED); + t.false(await tableExists(tableName)); +}); + +test.serial('DynamoDB: updateTable adds GSI; rollback removes it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ddb_gsi'); + createdTables.push(tableName); + + await seedTable(tableName, [{ AttributeName: 'id', KeyType: 'HASH' }], [{ AttributeName: 'id', AttributeType: 'S' }]); + + const indexName = 'gsi_by_email'; + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'updateTable', + tableName, + attributeDefinitions: [ + { attributeName: 'id', attributeType: 'S' }, + { attributeName: 'email', attributeType: 'S' }, + ], + globalSecondaryIndexUpdates: [ + { + create: { + indexName, + keySchema: [{ attributeName: 'email', keyType: 'HASH' }], + projection: { projectionType: 'ALL' }, + }, + }, + ], + }), + rollbackOp: JSON.stringify({ + operation: 'updateTable', + tableName, + globalSecondaryIndexUpdates: [{ delete: { indexName } }], + }), + changeType: SchemaChangeTypeEnum.DYNAMODB_UPDATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'add email GSI', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add email GSI' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + + let described = await describeTable(tableName); + t.truthy(described?.GlobalSecondaryIndexes?.find((g) => g.IndexName === indexName)); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + + described = await describeTable(tableName); + t.falsy(described?.GlobalSecondaryIndexes?.find((g) => g.IndexName === indexName)); +}); + +test.serial('DynamoDB: updateTimeToLive enables and rolls back', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ddb_ttl'); + createdTables.push(tableName); + + await seedTable(tableName, [{ AttributeName: 'id', KeyType: 'HASH' }], [{ AttributeName: 'id', AttributeType: 'S' }]); + + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'updateTimeToLive', + tableName, + timeToLiveSpecification: { enabled: true, attributeName: 'expiresAt' }, + }), + rollbackOp: JSON.stringify({ + operation: 'updateTimeToLive', + tableName, + timeToLiveSpecification: { enabled: false, attributeName: 'expiresAt' }, + }), + changeType: SchemaChangeTypeEnum.DYNAMODB_UPDATE_TTL, + targetTableName: tableName, + isReversible: true, + summary: 'enable TTL on expiresAt', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'enable ttl' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + + let ttl = await describeTtl(tableName); + t.is(ttl?.AttributeName, 'expiresAt'); + t.is(ttl?.TimeToLiveStatus, 'ENABLED'); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + + ttl = await describeTtl(tableName); + t.not(ttl?.TimeToLiveStatus, 'ENABLED'); +}); + +test.serial('DynamoDB: userModifiedSql JSON op is validated and applied', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ddb_um'); + createdTables.push(tableName); + + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'createTable', + tableName, + attributeDefinitions: [{ attributeName: 'id', attributeType: 'S' }], + keySchema: [{ attributeName: 'id', keyType: 'HASH' }], + }), + rollbackOp: JSON.stringify({ operation: 'deleteTable', tableName }), + changeType: SchemaChangeTypeEnum.DYNAMODB_CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'original', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create table' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const editedOp = JSON.stringify({ + operation: 'createTable', + tableName, + attributeDefinitions: [ + { attributeName: 'id', attributeType: 'S' }, + { attributeName: 'sort', attributeType: 'N' }, + ], + keySchema: [ + { attributeName: 'id', keyType: 'HASH' }, + { attributeName: 'sort', keyType: 'RANGE' }, + ], + billingMode: 'PAY_PER_REQUEST', + }); + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: editedOp }); + t.is(approveResp.status, 200); + + const described = await describeTable(tableName); + const hashKey = described?.KeySchema?.find((k) => k.KeyType === 'HASH'); + const rangeKey = described?.KeySchema?.find((k) => k.KeyType === 'RANGE'); + t.is(hashKey?.AttributeName, 'id'); + t.is(rangeKey?.AttributeName, 'sort'); +}); + +test.serial('DynamoDB: userModifiedSql with mismatched tableName is rejected', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ddb_umbad'); + + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'createTable', + tableName, + attributeDefinitions: [{ attributeName: 'id', attributeType: 'S' }], + keySchema: [{ attributeName: 'id', keyType: 'HASH' }], + }), + rollbackOp: JSON.stringify({ operation: 'deleteTable', tableName }), + changeType: SchemaChangeTypeEnum.DYNAMODB_CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'ok', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const editedOp = JSON.stringify({ + operation: 'createTable', + tableName: `${tableName}_hijack`, + attributeDefinitions: [{ attributeName: 'id', attributeType: 'S' }], + keySchema: [{ attributeName: 'id', keyType: 'HASH' }], + }); + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: editedOp }); + t.is(approveResp.status, 400); +}); + +test.serial('DynamoDB: tool/changeType mismatch is rejected at generate', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ddb_mism'); + + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'createTable', + tableName, + attributeDefinitions: [{ attributeName: 'id', attributeType: 'S' }], + keySchema: [{ attributeName: 'id', keyType: 'HASH' }], + }), + rollbackOp: JSON.stringify({ operation: 'deleteTable', tableName }), + changeType: SchemaChangeTypeEnum.CREATE_TABLE, // deliberate mismatch + targetTableName: tableName, + isReversible: true, + summary: 'mismatch', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'mismatch' }); + t.is(generateResp.status, 400); +}); + +test.serial('DynamoDB: createTable with missing attribute fails at generate-time validation', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ddb_bad'); + + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'createTable', + tableName, + attributeDefinitions: [{ attributeName: 'id', attributeType: 'S' }], + // keySchema references an attribute not declared: + keySchema: [{ attributeName: 'missing_attr', keyType: 'HASH' }], + }), + rollbackOp: JSON.stringify({ operation: 'deleteTable', tableName }), + changeType: SchemaChangeTypeEnum.DYNAMODB_CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'bad', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'broken create' }); + t.is(generateResp.status, 400); +}); + +test.serial('DynamoDB: runtime failure marks FAILED and attempts auto-rollback', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('ra_ddb_fail'); + createdTables.push(tableName); + + await seedTable(tableName, [{ AttributeName: 'id', KeyType: 'HASH' }], [{ AttributeName: 'id', AttributeType: 'S' }]); + + // Try to delete a GSI that doesn't exist — DynamoDB rejects at UpdateTable time. + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'updateTable', + tableName, + globalSecondaryIndexUpdates: [{ delete: { indexName: 'nonexistent_gsi' } }], + }), + rollbackOp: JSON.stringify({ + operation: 'updateTable', + tableName, + attributeDefinitions: [ + { attributeName: 'id', attributeType: 'S' }, + { attributeName: 'email', attributeType: 'S' }, + ], + globalSecondaryIndexUpdates: [ + { + create: { + indexName: 'nonexistent_gsi', + keySchema: [{ attributeName: 'email', keyType: 'HASH' }], + projection: { projectionType: 'ALL' }, + }, + }, + ], + }), + changeType: SchemaChangeTypeEnum.DYNAMODB_UPDATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'delete missing GSI', + reasoning: 'will fail', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'delete missing GSI' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 400); + + const getResp = await request(app.getHttpServer()).get(`/table-schema/change/${changeId}`).set('Cookie', token); + const record = JSON.parse(getResp.text); + t.is(record.status, SchemaChangeStatusEnum.FAILED); + t.true(record.autoRollbackAttempted); + t.truthy(record.executionError); +}); diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-elasticsearch-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-elasticsearch-e2e.test.ts new file mode 100644 index 000000000..8f4b9f7ed --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-elasticsearch-e2e.test.ts @@ -0,0 +1,479 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { Client } from '@elastic/elasticsearch'; +import { faker } from '@faker-js/faker'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import { ValidationError } from 'class-validator'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { AICoreService } from '../../../src/ai-core/index.js'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { WinstonLogger } from '../../../src/entities/logging/winston-logger.js'; +import { + SchemaChangeStatusEnum, + SchemaChangeTypeEnum, +} from '../../../src/entities/table-schema/table-schema-change-enums.js'; +import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { getTestData } from '../../utils/get-test-data.js'; +import { + createInitialTestUser, + registerUserAndReturnUserInfo, +} from '../../utils/register-user-and-return-user-info.js'; +import { setSaasEnvVariable } from '../../utils/set-saas-env-variable.js'; +import { TestUtils } from '../../utils/test.utils.js'; + +interface ElasticProposedChange { + forwardOp: string; + rollbackOp: string; + changeType: SchemaChangeTypeEnum; + targetTableName: string; + isReversible: boolean; + summary: string; + reasoning: string; +} + +let nextProposal: ElasticProposedChange | null = null; + +function createElasticProposalStream(proposal: ElasticProposedChange) { + return { + *[Symbol.asyncIterator]() { + yield { + type: 'tool_call', + toolCall: { + id: faker.string.uuid(), + name: 'proposeElasticsearchSchemaChange', + arguments: { proposals: [proposal] }, + }, + responseId: faker.string.uuid(), + }; + }, + }; +} + +const mockAICoreService = { + streamChatWithToolsAndProvider: async () => { + if (!nextProposal) throw new Error('Test must set nextProposal.'); + return createElasticProposalStream(nextProposal); + }, + complete: async () => 'mocked', + chat: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChat: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + chatWithTools: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithTools: async () => createElasticProposalStream(nextProposal!), + chatWithToolsAndProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithProvider: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + completeWithProvider: async () => 'mocked', + chatWithProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueAfterToolCall: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueStreamingAfterToolCall: async () => createElasticProposalStream(nextProposal!), + getDefaultProvider: () => 'openai', + setDefaultProvider: () => {}, + getAvailableProviders: () => [], +}; + +const mockFactory = new MockFactory(); +let app: INestApplication; +let _testUtils: TestUtils; +const createdIndices: string[] = []; + +function elasticConnectionParams() { + return getTestData(mockFactory).elasticsearchTestConnection; +} + +function buildElasticClient(): Client { + const params = elasticConnectionParams(); + return new Client({ + node: `http://${params.host}:${params.port}`, + auth: { username: params.username, password: params.password }, + }); +} + +async function withElastic(fn: (client: Client) => Promise): Promise { + const client = buildElasticClient(); + try { + return await fn(client); + } finally { + await client.close().catch(() => undefined); + } +} + +async function indexExists(indexName: string): Promise { + return withElastic(async (client) => { + return client.indices.exists({ index: indexName }); + }); +} + +async function getMapping(indexName: string): Promise | null> { + return withElastic(async (client) => { + try { + const resp = await client.indices.getMapping({ index: indexName }); + const entry = (resp as Record }>)[indexName]; + return entry?.mappings ?? null; + } catch { + return null; + } + }); +} + +async function dropIndexIfExists(indexName: string): Promise { + await withElastic(async (client) => { + try { + await client.indices.delete({ index: indexName }); + } catch { + /* ignore missing index */ + } + }); +} + +async function createIndexWithMapping(indexName: string, mappings?: Record): Promise { + await withElastic(async (client) => { + await client.indices.create({ + index: indexName, + body: mappings ? { mappings } : undefined, + } as Parameters[0]); + }); +} + +test.before(async () => { + setSaasEnvVariable(); + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }) + .overrideProvider(AICoreService) + .useValue(mockAICoreService) + .compile(); + + _testUtils = moduleFixture.get(TestUtils); + app = moduleFixture.createNestApplication(); + app.use(cookieParser()); + app.useGlobalFilters(new AllExceptionsFilter(app.get(WinstonLogger))); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + await createInitialTestUser(app); + app.getHttpServer().listen(0); +}); + +test.after(async () => { + try { + for (const name of createdIndices) { + await dropIndexIfExists(name); + } + await Cacher.clearAllCache(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +async function createConnection(token: string): Promise { + const connectionToTestDB = elasticConnectionParams(); + const resp = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + if (resp.status !== 201) { + throw new Error(`Failed to create connection: ${resp.status} ${resp.text}`); + } + return JSON.parse(resp.text).id; +} + +function randomIndexName(prefix: string): string { + return `${prefix}_${faker.string.alphanumeric(6).toLowerCase()}`; +} + +test.serial('Elasticsearch: createIndex → approve creates the index; rollback drops it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const indexName = randomIndexName('ra_es_t'); + createdIndices.push(indexName); + + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'createIndex', + indexName, + mappings: { properties: { name: { type: 'keyword' } } }, + }), + rollbackOp: JSON.stringify({ operation: 'deleteIndex', indexName }), + changeType: SchemaChangeTypeEnum.ELASTICSEARCH_CREATE_INDEX, + targetTableName: indexName, + isReversible: true, + summary: `Create ${indexName}`, + reasoning: 'basic create', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: `create index ${indexName}` }); + t.is(generateResp.status, 201); + const change = JSON.parse(generateResp.text).changes[0]; + t.is(change.status, SchemaChangeStatusEnum.PENDING); + t.is(change.changeType, SchemaChangeTypeEnum.ELASTICSEARCH_CREATE_INDEX); + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${change.id}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).status, SchemaChangeStatusEnum.APPLIED); + t.true(await indexExists(indexName)); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${change.id}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + t.is(JSON.parse(rollbackResp.text).status, SchemaChangeStatusEnum.ROLLED_BACK); + t.false(await indexExists(indexName)); + + const listResp = await request(app.getHttpServer()).get(`/table-schema/${connectionId}/changes`).set('Cookie', token); + const list = JSON.parse(listResp.text); + const auditRow = list.data.find( + (c: any) => c.changeType === SchemaChangeTypeEnum.ROLLBACK && c.previousChangeId === change.id, + ); + t.truthy(auditRow, 'expected linked rollback audit row'); +}); + +test.serial('Elasticsearch: deleteIndex requires confirmedDestructive', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const indexName = randomIndexName('ra_es_d'); + createdIndices.push(indexName); + + await createIndexWithMapping(indexName, { properties: { name: { type: 'keyword' } } }); + + nextProposal = { + forwardOp: JSON.stringify({ operation: 'deleteIndex', indexName }), + rollbackOp: JSON.stringify({ operation: 'createIndex', indexName }), + changeType: SchemaChangeTypeEnum.ELASTICSEARCH_DELETE_INDEX, + targetTableName: indexName, + isReversible: false, + summary: 'destructive', + reasoning: 'user wants to drop', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'drop the index' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const firstApprove = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(firstApprove.status, 400); + t.is(firstApprove.body?.type, 'destructive_confirmation_required'); + t.true(await indexExists(indexName), 'index must still exist'); + + const secondApprove = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ confirmedDestructive: true }); + t.is(secondApprove.status, 200); + t.is(JSON.parse(secondApprove.text).status, SchemaChangeStatusEnum.APPLIED); + t.false(await indexExists(indexName)); +}); + +test.serial('Elasticsearch: updateMapping adds a new field; rollback is recorded', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const indexName = randomIndexName('ra_es_um'); + createdIndices.push(indexName); + + await createIndexWithMapping(indexName, { properties: { name: { type: 'keyword' } } }); + + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'updateMapping', + indexName, + properties: { phone: { type: 'keyword' } }, + }), + // No first-class inverse for putMapping: echo the same updateMapping (idempotent) as a + // best-effort no-op rollback. Approve path then records ROLLED_BACK without any field removal. + rollbackOp: JSON.stringify({ + operation: 'updateMapping', + indexName, + properties: { phone: { type: 'keyword' } }, + }), + changeType: SchemaChangeTypeEnum.ELASTICSEARCH_UPDATE_MAPPING, + targetTableName: indexName, + isReversible: false, + summary: 'add phone field', + reasoning: 'putMapping is forward-only', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add phone field' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + // Update is destructive in the sense that the rollback can't restore the prior + // mapping shape; the use-case requires confirmedDestructive when isReversible=false. + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ confirmedDestructive: true }); + t.is(approveResp.status, 200); + + const mapping = await getMapping(indexName); + const properties = (mapping?.properties as Record) ?? {}; + t.is(properties.phone?.type, 'keyword'); +}); + +test.serial('Elasticsearch: invalid op marks FAILED and attempts auto-rollback', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const indexName = randomIndexName('ra_es_fail'); + createdIndices.push(indexName); + + // updateMapping on a non-existent index — Elasticsearch will reject with an + // index_not_found_exception, exercising the FAILED + auto-rollback path. + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'updateMapping', + indexName, + properties: { phone: { type: 'keyword' } }, + }), + rollbackOp: JSON.stringify({ operation: 'deleteIndex', indexName }), + changeType: SchemaChangeTypeEnum.ELASTICSEARCH_UPDATE_MAPPING, + targetTableName: indexName, + isReversible: true, + summary: 'will fail', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'update missing index' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 400); + + const getResp = await request(app.getHttpServer()).get(`/table-schema/change/${changeId}`).set('Cookie', token); + const record = JSON.parse(getResp.text); + t.is(record.status, SchemaChangeStatusEnum.FAILED); + t.true(record.autoRollbackAttempted); + t.truthy(record.executionError); +}); + +test.serial('Elasticsearch: userModifiedSql JSON op is validated and applied', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const indexName = randomIndexName('ra_es_um2'); + createdIndices.push(indexName); + + nextProposal = { + forwardOp: JSON.stringify({ operation: 'createIndex', indexName }), + rollbackOp: JSON.stringify({ operation: 'deleteIndex', indexName }), + changeType: SchemaChangeTypeEnum.ELASTICSEARCH_CREATE_INDEX, + targetTableName: indexName, + isReversible: true, + summary: 'original', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const editedOp = JSON.stringify({ + operation: 'createIndex', + indexName, + mappings: { properties: { email: { type: 'keyword' } } }, + }); + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: editedOp }); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).userModifiedSql, editedOp); + + const mapping = await getMapping(indexName); + const properties = (mapping?.properties as Record) ?? {}; + t.is(properties.email?.type, 'keyword'); +}); + +test.serial('Elasticsearch: userModifiedSql targeting a system index is rejected', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const indexName = randomIndexName('ra_es_sys'); + + nextProposal = { + forwardOp: JSON.stringify({ operation: 'createIndex', indexName }), + rollbackOp: JSON.stringify({ operation: 'deleteIndex', indexName }), + changeType: SchemaChangeTypeEnum.ELASTICSEARCH_CREATE_INDEX, + targetTableName: indexName, + isReversible: true, + summary: 'valid original', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + // User edits to point at a reserved index, but targetTableName mismatch is checked first; + // even if names matched the leading "." would block the system index. + const editedOp = JSON.stringify({ operation: 'createIndex', indexName: '.security' }); + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: editedOp }); + t.is(approveResp.status, 400); +}); + +test.serial('Elasticsearch: tool/changeType mismatch is rejected at generate', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const indexName = randomIndexName('ra_es_mism'); + + nextProposal = { + forwardOp: JSON.stringify({ operation: 'createIndex', indexName }), + rollbackOp: JSON.stringify({ operation: 'deleteIndex', indexName }), + changeType: SchemaChangeTypeEnum.CREATE_TABLE, // deliberate mismatch + targetTableName: indexName, + isReversible: true, + summary: 'mismatch', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'mismatch' }); + t.is(generateResp.status, 400); +}); diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-ibmdb2-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-ibmdb2-e2e.test.ts new file mode 100644 index 000000000..903ac08e1 --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-ibmdb2-e2e.test.ts @@ -0,0 +1,327 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { faker } from '@faker-js/faker'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import { ValidationError } from 'class-validator'; +import cookieParser from 'cookie-parser'; +import ibmdb from 'ibm_db'; +import request from 'supertest'; +import { AICoreService } from '../../../src/ai-core/index.js'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { WinstonLogger } from '../../../src/entities/logging/winston-logger.js'; +import { + SchemaChangeStatusEnum, + SchemaChangeTypeEnum, +} from '../../../src/entities/table-schema/table-schema-change-enums.js'; +import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { getTestData } from '../../utils/get-test-data.js'; +import { + createInitialTestUser, + registerUserAndReturnUserInfo, +} from '../../utils/register-user-and-return-user-info.js'; +import { setSaasEnvVariable } from '../../utils/set-saas-env-variable.js'; +import { TestUtils } from '../../utils/test.utils.js'; + +interface ProposedChange { + forwardSql: string; + rollbackSql: string; + changeType: SchemaChangeTypeEnum; + targetTableName: string; + isReversible: boolean; + summary: string; + reasoning: string; +} + +let nextProposal: ProposedChange | null = null; + +function createProposalStream(proposal: ProposedChange) { + return { + *[Symbol.asyncIterator]() { + yield { + type: 'tool_call', + toolCall: { + id: faker.string.uuid(), + name: 'proposeSchemaChange', + arguments: { proposals: [proposal] }, + }, + responseId: faker.string.uuid(), + }; + }, + }; +} + +const mockAICoreService = { + streamChatWithToolsAndProvider: async () => { + if (!nextProposal) throw new Error('Test must set nextProposal.'); + return createProposalStream(nextProposal); + }, + complete: async () => 'mocked', + chat: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChat: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + chatWithTools: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithTools: async () => createProposalStream(nextProposal!), + chatWithToolsAndProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithProvider: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + completeWithProvider: async () => 'mocked', + chatWithProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueAfterToolCall: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueStreamingAfterToolCall: async () => createProposalStream(nextProposal!), + getDefaultProvider: () => 'openai', + setDefaultProvider: () => {}, + getAvailableProviders: () => [], +}; + +const mockFactory = new MockFactory(); +let app: INestApplication; +let _testUtils: TestUtils; + +function buildConnStr(params: any): string { + return `DATABASE=${params.database};HOSTNAME=${params.host};UID=${params.username};PWD=${params.password};PORT=${params.port};PROTOCOL=TCPIP`; +} + +async function ensureSchema(params: any): Promise { + const db = ibmdb(); + await db.open(buildConnStr(params)); + try { + const exists = (await db.query( + `SELECT COUNT(*) AS C FROM SYSCAT.SCHEMATA WHERE SCHEMANAME = '${params.schema}'`, + )) as Array<{ C: number }>; + if (!exists[0] || !exists[0].C) { + await db.query(`CREATE SCHEMA ${params.schema}`); + } + } finally { + await db.close(); + } +} + +async function queryDb2(params: any, sql: string): Promise { + const db = ibmdb(); + await db.open(buildConnStr(params)); + try { + return (await db.query(sql)) as T[]; + } finally { + await db.close(); + } +} + +test.before(async () => { + setSaasEnvVariable(); + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }) + .overrideProvider(AICoreService) + .useValue(mockAICoreService) + .compile(); + + _testUtils = moduleFixture.get(TestUtils); + app = moduleFixture.createNestApplication(); + app.use(cookieParser()); + app.useGlobalFilters(new AllExceptionsFilter(app.get(WinstonLogger))); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + await createInitialTestUser(app); + app.getHttpServer().listen(0); + + await ensureSchema(getTestData(mockFactory).connectionToIbmDb2); +}); + +test.after(async () => { + try { + await Cacher.clearAllCache(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +async function createConnection(token: string): Promise { + const connectionToTestDB = getTestData(mockFactory).connectionToIbmDb2; + const resp = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + if (resp.status !== 201) { + throw new Error(`Failed to create connection: ${resp.status} ${resp.text}`); + } + return JSON.parse(resp.text).id; +} + +function randomTableName(prefix: string): string { + return `${prefix}_${faker.string.alphanumeric(6).toUpperCase()}`; +} + +function qualifyName(tableName: string): string { + const schema = getTestData(mockFactory).connectionToIbmDb2.schema; + return `${schema}.${tableName}`; +} + +async function tableExists(tableName: string): Promise { + const params = getTestData(mockFactory).connectionToIbmDb2; + const rows = await queryDb2<{ C: number }>( + params, + `SELECT COUNT(*) AS C FROM SYSCAT.TABLES WHERE TABSCHEMA = '${params.schema}' AND TABNAME = '${tableName}'`, + ); + return Number(rows[0]?.C ?? 0) > 0; +} + +async function columnExists(tableName: string, columnName: string): Promise { + const params = getTestData(mockFactory).connectionToIbmDb2; + const rows = await queryDb2<{ C: number }>( + params, + `SELECT COUNT(*) AS C FROM SYSCAT.COLUMNS WHERE TABSCHEMA = '${params.schema}' AND TABNAME = '${tableName}' AND COLNAME = '${columnName}'`, + ); + return Number(rows[0]?.C ?? 0) > 0; +} + +async function columnLength(tableName: string, columnName: string): Promise { + const params = getTestData(mockFactory).connectionToIbmDb2; + const rows = await queryDb2<{ LENGTH: number }>( + params, + `SELECT LENGTH FROM SYSCAT.COLUMNS WHERE TABSCHEMA = '${params.schema}' AND TABNAME = '${tableName}' AND COLNAME = '${columnName}'`, + ); + return rows[0]?.LENGTH ?? null; +} + +test.serial('DB2: generate → approve creates the table', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('RA_T'); + + nextProposal = { + forwardSql: `CREATE TABLE ${qualifyName(tableName)} (ID BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, NAME VARCHAR(255))`, + rollbackSql: `DROP TABLE ${qualifyName(tableName)}`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'db2 create', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create table' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).status, SchemaChangeStatusEnum.APPLIED); + + t.true(await tableExists(tableName)); +}); + +test.serial('DB2: ADD COLUMN approve adds the column; rollback removes it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('RA_ADDCOL'); + + const params = getTestData(mockFactory).connectionToIbmDb2; + await queryDb2( + params, + `CREATE TABLE ${qualifyName(tableName)} (ID BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY)`, + ); + + nextProposal = { + forwardSql: `ALTER TABLE ${qualifyName(tableName)} ADD COLUMN PHONE VARCHAR(255)`, + rollbackSql: `ALTER TABLE ${qualifyName(tableName)} DROP COLUMN PHONE`, + changeType: SchemaChangeTypeEnum.ADD_COLUMN, + targetTableName: tableName, + isReversible: true, + summary: 'add phone', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add phone column' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.true(await columnExists(tableName, 'PHONE')); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + t.false(await columnExists(tableName, 'PHONE')); +}); + +test.serial('DB2: ALTER COLUMN SET DATA TYPE widens VARCHAR(32) → VARCHAR(255) preserving rows', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomTableName('RA_ALT'); + + const params = getTestData(mockFactory).connectionToIbmDb2; + await queryDb2(params, `CREATE TABLE ${qualifyName(tableName)} (ID INTEGER PRIMARY KEY NOT NULL, NAME VARCHAR(32))`); + await queryDb2(params, `INSERT INTO ${qualifyName(tableName)} (ID, NAME) VALUES (1, 'alice')`); + await queryDb2(params, `INSERT INTO ${qualifyName(tableName)} (ID, NAME) VALUES (2, 'bob')`); + await queryDb2(params, `INSERT INTO ${qualifyName(tableName)} (ID, NAME) VALUES (3, 'carol')`); + + nextProposal = { + forwardSql: `ALTER TABLE ${qualifyName(tableName)} ALTER COLUMN NAME SET DATA TYPE VARCHAR(255)`, + rollbackSql: `ALTER TABLE ${qualifyName(tableName)} ALTER COLUMN NAME SET DATA TYPE VARCHAR(32)`, + changeType: SchemaChangeTypeEnum.ALTER_COLUMN, + targetTableName: tableName, + isReversible: true, + summary: 'widen name', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'widen name' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + + const length = await columnLength(tableName, 'NAME'); + t.is(length, 255); + + const rows = await queryDb2<{ NAME: string; ID: number }>( + params, + `SELECT ID, NAME FROM ${qualifyName(tableName)} ORDER BY ID`, + ); + t.deepEqual( + rows.map((r) => r.NAME), + ['alice', 'bob', 'carol'], + ); +}); diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-mongodb-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-mongodb-e2e.test.ts new file mode 100644 index 000000000..fb1d98da0 --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-mongodb-e2e.test.ts @@ -0,0 +1,580 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { faker } from '@faker-js/faker'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import { ValidationError } from 'class-validator'; +import cookieParser from 'cookie-parser'; +import { MongoClient } from 'mongodb'; +import request from 'supertest'; +import { AICoreService } from '../../../src/ai-core/index.js'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { WinstonLogger } from '../../../src/entities/logging/winston-logger.js'; +import { + SchemaChangeStatusEnum, + SchemaChangeTypeEnum, +} from '../../../src/entities/table-schema/table-schema-change-enums.js'; +import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { getTestData } from '../../utils/get-test-data.js'; +import { + createInitialTestUser, + registerUserAndReturnUserInfo, +} from '../../utils/register-user-and-return-user-info.js'; +import { setSaasEnvVariable } from '../../utils/set-saas-env-variable.js'; +import { TestUtils } from '../../utils/test.utils.js'; + +interface MongoProposedChange { + forwardOp: string; + rollbackOp: string; + changeType: SchemaChangeTypeEnum; + targetTableName: string; + isReversible: boolean; + summary: string; + reasoning: string; +} + +let nextProposal: MongoProposedChange | null = null; + +function createMongoProposalStream(proposal: MongoProposedChange) { + return { + *[Symbol.asyncIterator]() { + yield { + type: 'tool_call', + toolCall: { + id: faker.string.uuid(), + name: 'proposeMongoSchemaChange', + arguments: { proposals: [proposal] }, + }, + responseId: faker.string.uuid(), + }; + }, + }; +} + +const mockAICoreService = { + streamChatWithToolsAndProvider: async () => { + if (!nextProposal) throw new Error('Test must set nextProposal.'); + return createMongoProposalStream(nextProposal); + }, + complete: async () => 'mocked', + chat: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChat: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + chatWithTools: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithTools: async () => createMongoProposalStream(nextProposal!), + chatWithToolsAndProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithProvider: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + completeWithProvider: async () => 'mocked', + chatWithProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueAfterToolCall: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueStreamingAfterToolCall: async () => createMongoProposalStream(nextProposal!), + getDefaultProvider: () => 'openai', + setDefaultProvider: () => {}, + getAvailableProviders: () => [], +}; + +const mockFactory = new MockFactory(); +let app: INestApplication; +let _testUtils: TestUtils; +const createdCollections: string[] = []; + +function buildMongoUri(params: any): string { + let uri = + `mongodb://${encodeURIComponent(params.username)}:${encodeURIComponent(params.password)}` + + `@${params.host}:${params.port}/${params.database}`; + if (params.authSource) { + uri += `?authSource=${encodeURIComponent(params.authSource)}`; + } + return uri; +} + +async function withMongoClient(params: any, fn: (db: import('mongodb').Db) => Promise): Promise { + const client = new MongoClient(buildMongoUri(params)); + await client.connect(); + try { + return await fn(client.db(params.database)); + } finally { + await client.close(); + } +} + +async function collectionExists(params: any, collectionName: string): Promise { + return withMongoClient(params, async (db) => { + const list = await db.listCollections({ name: collectionName }).toArray(); + return list.length > 0; + }); +} + +async function getIndexes(params: any, collectionName: string): Promise>> { + return withMongoClient(params, async (db) => { + return db.collection(collectionName).listIndexes().toArray(); + }); +} + +async function getValidator(params: any, collectionName: string): Promise | null> { + return withMongoClient(params, async (db) => { + const list = (await db.listCollections({ name: collectionName }).toArray()) as Array<{ + options?: { validator?: Record }; + }>; + if (!list.length) return null; + return list[0].options?.validator ?? null; + }); +} + +async function seedCollection( + params: any, + collectionName: string, + docs: Array>, +): Promise { + await withMongoClient(params, async (db) => { + try { + await db.createCollection(collectionName); + } catch (err) { + if ( + !String((err as Error).message) + .toLowerCase() + .includes('already exists') + ) { + throw err; + } + } + if (docs.length > 0) { + await db.collection(collectionName).insertMany(docs); + } + }); +} + +async function dropCollectionIfExists(params: any, collectionName: string): Promise { + await withMongoClient(params, async (db) => { + try { + await db.collection(collectionName).drop(); + } catch { + /* ignore missing collection */ + } + }); +} + +test.before(async () => { + setSaasEnvVariable(); + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }) + .overrideProvider(AICoreService) + .useValue(mockAICoreService) + .compile(); + + _testUtils = moduleFixture.get(TestUtils); + app = moduleFixture.createNestApplication(); + app.use(cookieParser()); + app.useGlobalFilters(new AllExceptionsFilter(app.get(WinstonLogger))); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + await createInitialTestUser(app); + app.getHttpServer().listen(0); +}); + +test.after(async () => { + try { + const mongoParams = getTestData(mockFactory).mongoDbConnection; + for (const name of createdCollections) { + await dropCollectionIfExists(mongoParams, name); + } + await Cacher.clearAllCache(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +async function createConnection(token: string, overrides: Partial> = {}): Promise { + const connectionToTestDB = { ...getTestData(mockFactory).mongoDbConnection, ...overrides }; + const resp = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + if (resp.status !== 201) { + throw new Error(`Failed to create connection: ${resp.status} ${resp.text}`); + } + return JSON.parse(resp.text).id; +} + +function randomCollectionName(prefix: string): string { + return `${prefix}_${faker.string.alphanumeric(6).toLowerCase()}`; +} + +test.serial('MongoDB: createCollection → drop via rollback', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const collectionName = randomCollectionName('ra_mc'); + createdCollections.push(collectionName); + + nextProposal = { + forwardOp: JSON.stringify({ operation: 'createCollection', collectionName }), + rollbackOp: JSON.stringify({ operation: 'dropCollection', collectionName }), + changeType: SchemaChangeTypeEnum.MONGO_CREATE_COLLECTION, + targetTableName: collectionName, + isReversible: true, + summary: `Create ${collectionName}`, + reasoning: 'test', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: `create collection ${collectionName}` }); + t.is(generateResp.status, 201); + const change = JSON.parse(generateResp.text).changes[0]; + t.is(change.status, SchemaChangeStatusEnum.PENDING); + t.is(change.changeType, SchemaChangeTypeEnum.MONGO_CREATE_COLLECTION); + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${change.id}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).status, SchemaChangeStatusEnum.APPLIED); + t.true(await collectionExists(getTestData(mockFactory).mongoDbConnection, collectionName)); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${change.id}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + t.is(JSON.parse(rollbackResp.text).status, SchemaChangeStatusEnum.ROLLED_BACK); + t.false(await collectionExists(getTestData(mockFactory).mongoDbConnection, collectionName)); +}); + +test.serial('MongoDB: dropCollection requires confirmedDestructive', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const collectionName = randomCollectionName('ra_md'); + createdCollections.push(collectionName); + + const mongoParams = getTestData(mockFactory).mongoDbConnection; + await seedCollection(mongoParams, collectionName, [{ x: 1 }, { x: 2 }]); + + nextProposal = { + forwardOp: JSON.stringify({ operation: 'dropCollection', collectionName }), + rollbackOp: JSON.stringify({ operation: 'createCollection', collectionName }), + changeType: SchemaChangeTypeEnum.MONGO_DROP_COLLECTION, + targetTableName: collectionName, + isReversible: false, + summary: 'drop', + reasoning: 'destructive', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'drop collection' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const firstApprove = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(firstApprove.status, 400); + t.is(firstApprove.body?.type, 'destructive_confirmation_required'); + t.true(await collectionExists(mongoParams, collectionName)); + + const secondApprove = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ confirmedDestructive: true }); + t.is(secondApprove.status, 200); + t.is(JSON.parse(secondApprove.text).status, SchemaChangeStatusEnum.APPLIED); + t.false(await collectionExists(mongoParams, collectionName)); +}); + +test.serial('MongoDB: createIndex → rollback drops the index', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const collectionName = randomCollectionName('ra_mi'); + createdCollections.push(collectionName); + + const mongoParams = getTestData(mockFactory).mongoDbConnection; + await seedCollection(mongoParams, collectionName, [{ email: 'a@a.test' }, { email: 'b@b.test' }]); + + const indexName = 'idx_email_asc'; + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'createIndex', + collectionName, + indexName, + indexSpec: { email: 1 }, + indexOptions: { unique: true, name: indexName }, + }), + rollbackOp: JSON.stringify({ operation: 'dropIndex', collectionName, indexName }), + changeType: SchemaChangeTypeEnum.MONGO_CREATE_INDEX, + targetTableName: collectionName, + isReversible: true, + summary: 'add unique email index', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add unique email index' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + + let indexes = await getIndexes(mongoParams, collectionName); + t.truthy(indexes.find((idx) => idx.name === indexName)); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + + indexes = await getIndexes(mongoParams, collectionName); + t.falsy(indexes.find((idx) => idx.name === indexName)); +}); + +test.serial('MongoDB: setValidator applies a JSON Schema', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + // MongoDB refuses validators on the admin/config databases, so this test uses + // an isolated user-space database. authSource stays on admin where the root + // user is defined, per standard MongoDB practice. + const isolatedDb = `ra_test_${faker.string.alphanumeric(6).toLowerCase()}`; + const connectionId = await createConnection(token, { database: isolatedDb, authSource: 'admin' }); + const collectionName = randomCollectionName('ra_mv'); + + const mongoParams = { ...getTestData(mockFactory).mongoDbConnection, database: isolatedDb, authSource: 'admin' }; + await seedCollection(mongoParams, collectionName, []); + + const validatorSchema = { + $jsonSchema: { + bsonType: 'object', + required: ['email'], + properties: { + email: { bsonType: 'string' }, + }, + }, + }; + + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'setValidator', + collectionName, + validatorSchema, + validationLevel: 'strict', + validationAction: 'error', + }), + rollbackOp: JSON.stringify({ + operation: 'setValidator', + collectionName, + validatorSchema: null, + }), + changeType: SchemaChangeTypeEnum.MONGO_SET_VALIDATOR, + targetTableName: collectionName, + isReversible: true, + summary: 'require email', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'require email' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + if (approveResp.status !== 200) { + console.error('setValidator approve failed:', approveResp.status, approveResp.text); + } + t.is(approveResp.status, 200); + + const validator = await getValidator(mongoParams, collectionName); + t.truthy(validator?.$jsonSchema); +}); + +test.serial('MongoDB: validator with $where is rejected', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const collectionName = randomCollectionName('ra_mvbad'); + + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'setValidator', + collectionName, + validatorSchema: { $where: 'this.x === 1' }, + }), + rollbackOp: JSON.stringify({ operation: 'setValidator', collectionName, validatorSchema: null }), + changeType: SchemaChangeTypeEnum.MONGO_SET_VALIDATOR, + targetTableName: collectionName, + isReversible: true, + summary: 'bad validator', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'bad validator' }); + t.is(generateResp.status, 400); +}); + +test.serial('MongoDB: userModifiedSql JSON op is validated and applied', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const collectionName = randomCollectionName('ra_mum'); + createdCollections.push(collectionName); + + nextProposal = { + forwardOp: JSON.stringify({ operation: 'createCollection', collectionName }), + rollbackOp: JSON.stringify({ operation: 'dropCollection', collectionName }), + changeType: SchemaChangeTypeEnum.MONGO_CREATE_COLLECTION, + targetTableName: collectionName, + isReversible: true, + summary: 'original', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create collection' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const editedOp = JSON.stringify({ operation: 'createCollection', collectionName }); + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: editedOp }); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).userModifiedSql, editedOp); + t.true(await collectionExists(getTestData(mockFactory).mongoDbConnection, collectionName)); +}); + +test.serial('MongoDB: userModifiedSql with forbidden $where is rejected', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const collectionName = randomCollectionName('ra_mumbad'); + + nextProposal = { + forwardOp: JSON.stringify({ + operation: 'setValidator', + collectionName, + validatorSchema: { $jsonSchema: { bsonType: 'object' } }, + }), + rollbackOp: JSON.stringify({ operation: 'setValidator', collectionName, validatorSchema: null }), + changeType: SchemaChangeTypeEnum.MONGO_SET_VALIDATOR, + targetTableName: collectionName, + isReversible: true, + summary: 'ok', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'require shape' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const editedOp = JSON.stringify({ + operation: 'setValidator', + collectionName, + validatorSchema: { $where: 'this.x === 1' }, + }); + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: editedOp }); + t.is(approveResp.status, 400); +}); + +test.serial('MongoDB: invalid op marks FAILED and attempts auto-rollback', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const collectionName = randomCollectionName('ra_mfail'); + createdCollections.push(collectionName); + + // Ask the server to dropIndex "nonexistent" on a fresh collection — the initial + // createCollection-is-not-done-yet path is fine; we need a structurally valid op + // that the DB itself will reject. listIndexes on a missing collection throws. + nextProposal = { + forwardOp: JSON.stringify({ operation: 'dropIndex', collectionName, indexName: 'nonexistent_idx' }), + rollbackOp: JSON.stringify({ + operation: 'createIndex', + collectionName, + indexName: 'nonexistent_idx', + indexSpec: { x: 1 }, + indexOptions: { name: 'nonexistent_idx' }, + }), + changeType: SchemaChangeTypeEnum.MONGO_DROP_INDEX, + targetTableName: collectionName, + isReversible: true, + summary: 'drop missing index', + reasoning: 'will fail at runtime', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'drop nonexistent index' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 400); + + const getResp = await request(app.getHttpServer()).get(`/table-schema/change/${changeId}`).set('Cookie', token); + const record = JSON.parse(getResp.text); + t.is(record.status, SchemaChangeStatusEnum.FAILED); + t.true(record.autoRollbackAttempted); + t.truthy(record.executionError); +}); + +test.serial('MongoDB: tool/changeType mismatch is rejected', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const collectionName = randomCollectionName('ra_mmism'); + + nextProposal = { + forwardOp: JSON.stringify({ operation: 'createCollection', collectionName }), + rollbackOp: JSON.stringify({ operation: 'dropCollection', collectionName }), + changeType: SchemaChangeTypeEnum.CREATE_TABLE, // deliberate mismatch + targetTableName: collectionName, + isReversible: true, + summary: 'mismatch', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'mismatch' }); + t.is(generateResp.status, 400); +}); diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-mssql-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-mssql-e2e.test.ts new file mode 100644 index 000000000..a5e39d5d6 --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-mssql-e2e.test.ts @@ -0,0 +1,616 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { faker } from '@faker-js/faker'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import { ValidationError } from 'class-validator'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { AICoreService } from '../../../src/ai-core/index.js'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { WinstonLogger } from '../../../src/entities/logging/winston-logger.js'; +import { + SchemaChangeStatusEnum, + SchemaChangeTypeEnum, +} from '../../../src/entities/table-schema/table-schema-change-enums.js'; +import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { dropTestTables } from '../../utils/drop-test-tables.js'; +import { getTestData } from '../../utils/get-test-data.js'; +import { getTestKnex } from '../../utils/get-test-knex.js'; +import { + createInitialTestUser, + registerUserAndReturnUserInfo, +} from '../../utils/register-user-and-return-user-info.js'; +import { setSaasEnvVariable } from '../../utils/set-saas-env-variable.js'; +import { TestUtils } from '../../utils/test.utils.js'; + +interface ProposedChange { + forwardSql: string; + rollbackSql: string; + changeType: SchemaChangeTypeEnum; + targetTableName: string; + isReversible: boolean; + summary: string; + reasoning: string; +} + +let nextProposal: ProposedChange | null = null; + +function createProposalStream(proposal: ProposedChange) { + return { + *[Symbol.asyncIterator]() { + yield { + type: 'tool_call', + toolCall: { + id: faker.string.uuid(), + name: 'proposeSchemaChange', + arguments: { proposals: [proposal] }, + }, + responseId: faker.string.uuid(), + }; + }, + }; +} + +const mockAICoreService = { + streamChatWithToolsAndProvider: async () => { + if (!nextProposal) throw new Error('Test must set nextProposal.'); + return createProposalStream(nextProposal); + }, + complete: async () => 'mocked', + chat: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChat: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + chatWithTools: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithTools: async () => createProposalStream(nextProposal!), + chatWithToolsAndProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithProvider: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + completeWithProvider: async () => 'mocked', + chatWithProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueAfterToolCall: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueStreamingAfterToolCall: async () => createProposalStream(nextProposal!), + getDefaultProvider: () => 'openai', + setDefaultProvider: () => {}, + getAvailableProviders: () => [], +}; + +const mockFactory = new MockFactory(); +let app: INestApplication; +let _testUtils: TestUtils; +const testTables: Array = []; + +test.before(async () => { + setSaasEnvVariable(); + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }) + .overrideProvider(AICoreService) + .useValue(mockAICoreService) + .compile(); + + _testUtils = moduleFixture.get(TestUtils); + app = moduleFixture.createNestApplication(); + app.use(cookieParser()); + app.useGlobalFilters(new AllExceptionsFilter(app.get(WinstonLogger))); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + await createInitialTestUser(app); + app.getHttpServer().listen(0); +}); + +test.after(async () => { + try { + const connectionToTestDB = getTestData(mockFactory).connectionToTestMSSQL; + await dropTestTables(testTables, connectionToTestDB); + await Cacher.clearAllCache(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +async function createConnection(token: string): Promise { + const connectionToTestDB = getTestData(mockFactory).connectionToTestMSSQL; + const resp = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + if (resp.status !== 201) { + throw new Error(`Failed to create connection: ${resp.status} ${resp.text}`); + } + return JSON.parse(resp.text).id; +} + +function mssqlKnex() { + return getTestKnex(getTestData(mockFactory).connectionToTestMSSQL); +} + +test.serial('MSSQL: generate → approve creates the table', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_t_${faker.string.alphanumeric(8).toLowerCase()}`; + testTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE [${tableName}] (id INT IDENTITY(1,1) PRIMARY KEY, name NVARCHAR(255))`, + rollbackSql: `DROP TABLE [${tableName}]`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'mssql create', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create table' }); + t.is(generateResp.status, 201); + const change = JSON.parse(generateResp.text).changes[0]; + t.is(change.status, SchemaChangeStatusEnum.PENDING); + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${change.id}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).status, SchemaChangeStatusEnum.APPLIED); + + t.true(await mssqlKnex().schema.hasTable(tableName)); +}); + +test.serial('MSSQL: generate → approve → rollback removes table and links audit row', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_t_${faker.string.alphanumeric(8).toLowerCase()}`; + testTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE [${tableName}] (id INT IDENTITY(1,1) PRIMARY KEY)`, + rollbackSql: `DROP TABLE [${tableName}]`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'rollback test', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + await request(app.getHttpServer()).post(`/table-schema/change/${changeId}/approve`).set('Cookie', token).send({}); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + t.is(JSON.parse(rollbackResp.text).status, SchemaChangeStatusEnum.ROLLED_BACK); + t.false(await mssqlKnex().schema.hasTable(tableName)); + + const listResp = await request(app.getHttpServer()).get(`/table-schema/${connectionId}/changes`).set('Cookie', token); + const list = JSON.parse(listResp.text); + const rollbackAuditRow = list.data.find( + (c: any) => c.changeType === SchemaChangeTypeEnum.ROLLBACK && c.previousChangeId === changeId, + ); + t.truthy(rollbackAuditRow, 'Expected a linked rollback audit row'); +}); + +test.serial('MSSQL: invalid SQL marks FAILED and attempts auto-rollback', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_bad_${faker.string.alphanumeric(6).toLowerCase()}`; + + nextProposal = { + forwardSql: `CREATE TABLE [${tableName}] (id INVALIDTYPE)`, + rollbackSql: `IF OBJECT_ID('${tableName}', 'U') IS NOT NULL DROP TABLE [${tableName}]`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'invalid', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'bad sql' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 400); + + const getResp = await request(app.getHttpServer()).get(`/table-schema/change/${changeId}`).set('Cookie', token); + const record = JSON.parse(getResp.text); + t.is(record.status, SchemaChangeStatusEnum.FAILED); + t.true(record.autoRollbackAttempted); + t.truthy(record.executionError); + t.false(await mssqlKnex().schema.hasTable(tableName)); +}); + +test.serial('MSSQL: destructive DROP TABLE requires confirmedDestructive=true', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const seedTableName = `ra_seed_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(seedTableName); + + const knex = mssqlKnex(); + await knex.schema.createTable(seedTableName, (table) => { + table.increments(); + }); + + nextProposal = { + forwardSql: `DROP TABLE [${seedTableName}]`, + rollbackSql: `CREATE TABLE [${seedTableName}] (id INT IDENTITY(1,1) PRIMARY KEY)`, + changeType: SchemaChangeTypeEnum.DROP_TABLE, + targetTableName: seedTableName, + isReversible: false, + summary: 'drop existing', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'drop it' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const firstApproveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(firstApproveResp.status, 400); + t.is(firstApproveResp.body?.type, 'destructive_confirmation_required'); + t.true(await knex.schema.hasTable(seedTableName)); + + const secondApproveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ confirmedDestructive: true }); + t.is(secondApproveResp.status, 200); + t.is(JSON.parse(secondApproveResp.text).status, SchemaChangeStatusEnum.APPLIED); + t.false(await knex.schema.hasTable(seedTableName)); +}); + +test.serial('MSSQL: ADD COLUMN approve adds the column; rollback removes it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_addcol_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(tableName); + + const knex = mssqlKnex(); + await knex.schema.createTable(tableName, (table) => { + table.increments(); + }); + + nextProposal = { + forwardSql: `ALTER TABLE [${tableName}] ADD [phone] NVARCHAR(255)`, + rollbackSql: `ALTER TABLE [${tableName}] DROP COLUMN [phone]`, + changeType: SchemaChangeTypeEnum.ADD_COLUMN, + targetTableName: tableName, + isReversible: true, + summary: 'add phone', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add phone column' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).status, SchemaChangeStatusEnum.APPLIED); + t.true(await knex.schema.hasColumn(tableName, 'phone')); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + t.false(await knex.schema.hasColumn(tableName, 'phone')); +}); + +test.serial('MSSQL: DROP COLUMN blocked without confirmedDestructive, succeeds with it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_dropcol_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(tableName); + + const knex = mssqlKnex(); + await knex.schema.createTable(tableName, (table) => { + table.increments(); + table.string('phone', 255); + }); + + nextProposal = { + forwardSql: `ALTER TABLE [${tableName}] DROP COLUMN [phone]`, + rollbackSql: `ALTER TABLE [${tableName}] ADD [phone] NVARCHAR(255)`, + changeType: SchemaChangeTypeEnum.DROP_COLUMN, + targetTableName: tableName, + isReversible: false, + summary: 'drop phone', + reasoning: 'destructive', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'drop phone column' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const firstApproveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(firstApproveResp.status, 400); + t.is(firstApproveResp.body?.type, 'destructive_confirmation_required'); + t.true(await knex.schema.hasColumn(tableName, 'phone')); + + const secondApproveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ confirmedDestructive: true }); + t.is(secondApproveResp.status, 200); + t.is(JSON.parse(secondApproveResp.text).status, SchemaChangeStatusEnum.APPLIED); + t.false(await knex.schema.hasColumn(tableName, 'phone')); +}); + +test.serial('MSSQL: ADD INDEX approve creates the index; rollback drops it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_idx_${faker.string.alphanumeric(6).toLowerCase()}`; + const indexName = `ix_${tableName}_email`; + testTables.push(tableName); + + const knex = mssqlKnex(); + await knex.schema.createTable(tableName, (table) => { + table.increments(); + table.string('email', 255); + }); + + nextProposal = { + forwardSql: `CREATE INDEX [${indexName}] ON [${tableName}] ([email])`, + rollbackSql: `DROP INDEX [${indexName}] ON [${tableName}]`, + changeType: SchemaChangeTypeEnum.ADD_INDEX, + targetTableName: tableName, + isReversible: true, + summary: 'add email index', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add index' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + + const afterAdd = await knex.raw(`SELECT COUNT(*) AS c FROM sys.indexes WHERE name = ? AND object_id = OBJECT_ID(?)`, [ + indexName, + tableName, + ]); + t.is(Number((afterAdd as any)[0].c), 1); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + + const afterRollback = await knex.raw( + `SELECT COUNT(*) AS c FROM sys.indexes WHERE name = ? AND object_id = OBJECT_ID(?)`, + [indexName, tableName], + ); + t.is(Number((afterRollback as any)[0].c), 0); +}); + +test.serial('MSSQL: ADD FOREIGN KEY with cross-table reference; rollback drops it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const parentName = `ra_par_${faker.string.alphanumeric(6).toLowerCase()}`; + const childName = `ra_ch_${faker.string.alphanumeric(6).toLowerCase()}`; + const fkName = `fk_${childName}_parent`; + testTables.push(parentName, childName); + + const knex = mssqlKnex(); + await knex.schema.createTable(parentName, (table) => { + table.increments(); + }); + await knex.schema.createTable(childName, (table) => { + table.increments(); + table.integer('parent_id').notNullable(); + }); + + nextProposal = { + forwardSql: `ALTER TABLE [${childName}] ADD CONSTRAINT [${fkName}] FOREIGN KEY ([parent_id]) REFERENCES [${parentName}] ([id])`, + rollbackSql: `ALTER TABLE [${childName}] DROP CONSTRAINT [${fkName}]`, + changeType: SchemaChangeTypeEnum.ADD_FOREIGN_KEY, + targetTableName: childName, + isReversible: true, + summary: 'add fk', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add fk' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + + const afterAdd = await knex.raw(`SELECT COUNT(*) AS c FROM sys.foreign_keys WHERE name = ?`, [fkName]); + t.is(Number((afterAdd as any)[0].c), 1); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + + const afterRollback = await knex.raw(`SELECT COUNT(*) AS c FROM sys.foreign_keys WHERE name = ?`, [fkName]); + t.is(Number((afterRollback as any)[0].c), 0); +}); + +test.serial('MSSQL: userModifiedSql is validated and applied in place of AI SQL', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_um_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE [${tableName}] (id INT IDENTITY(1,1) PRIMARY KEY, foo NVARCHAR(255))`, + rollbackSql: `DROP TABLE [${tableName}]`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'ai original', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create table' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const editedSql = `CREATE TABLE [${tableName}] (id INT IDENTITY(1,1) PRIMARY KEY, bar NVARCHAR(255))`; + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: editedSql }); + t.is(approveResp.status, 200); + const applied = JSON.parse(approveResp.text); + t.is(applied.status, SchemaChangeStatusEnum.APPLIED); + t.is(applied.userModifiedSql, editedSql); + + const knex = mssqlKnex(); + t.true(await knex.schema.hasColumn(tableName, 'bar')); + t.false(await knex.schema.hasColumn(tableName, 'foo')); +}); + +test.serial('MSSQL: userModifiedSql with a forbidden construct is rejected', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_umbad_${faker.string.alphanumeric(6).toLowerCase()}`; + + nextProposal = { + forwardSql: `CREATE TABLE [${tableName}] (id INT IDENTITY(1,1) PRIMARY KEY)`, + rollbackSql: `DROP TABLE [${tableName}]`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'valid ai proposal', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create table' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: `TRUNCATE TABLE [${tableName}]` }); + t.is(approveResp.status, 400); + + const getResp = await request(app.getHttpServer()).get(`/table-schema/change/${changeId}`).set('Cookie', token); + t.is(JSON.parse(getResp.text).status, SchemaChangeStatusEnum.PENDING); + + const knex = mssqlKnex(); + t.false(await knex.schema.hasTable(tableName)); +}); + +test.serial('MSSQL: ALTER COLUMN widens NVARCHAR(32) → NVARCHAR(MAX) while preserving row data', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_alt_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(tableName); + + const knex = mssqlKnex(); + await knex.schema.createTable(tableName, (table) => { + table.increments(); + table.string('name', 32); + }); + await knex(tableName).insert([{ name: 'alice' }, { name: 'bob' }, { name: 'carol' }]); + + nextProposal = { + forwardSql: `ALTER TABLE [${tableName}] ALTER COLUMN [name] NVARCHAR(MAX)`, + rollbackSql: `ALTER TABLE [${tableName}] ALTER COLUMN [name] NVARCHAR(32)`, + changeType: SchemaChangeTypeEnum.ALTER_COLUMN, + targetTableName: tableName, + isReversible: true, + summary: 'widen name', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'widen name' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + + const typeRow = await knex.raw( + `SELECT character_maximum_length FROM information_schema.columns WHERE table_name = ? AND column_name = 'name'`, + [tableName], + ); + t.is(Number((typeRow as any)[0].character_maximum_length), -1); + + const rows = (await knex(tableName).select('name').orderBy('id')) as Array<{ name: string }>; + t.deepEqual( + rows.map((r) => r.name), + ['alice', 'bob', 'carol'], + ); +}); diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-mysql-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-mysql-e2e.test.ts new file mode 100644 index 000000000..8851ce8ce --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-mysql-e2e.test.ts @@ -0,0 +1,770 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { faker } from '@faker-js/faker'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import { ValidationError } from 'class-validator'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { AICoreService } from '../../../src/ai-core/index.js'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { WinstonLogger } from '../../../src/entities/logging/winston-logger.js'; +import { + SchemaChangeStatusEnum, + SchemaChangeTypeEnum, +} from '../../../src/entities/table-schema/table-schema-change-enums.js'; +import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { dropTestTables } from '../../utils/drop-test-tables.js'; +import { getTestData } from '../../utils/get-test-data.js'; +import { getTestKnex } from '../../utils/get-test-knex.js'; +import { + createInitialTestUser, + registerUserAndReturnUserInfo, +} from '../../utils/register-user-and-return-user-info.js'; +import { setSaasEnvVariable } from '../../utils/set-saas-env-variable.js'; +import { TestUtils } from '../../utils/test.utils.js'; + +interface ProposedChange { + forwardSql: string; + rollbackSql: string; + changeType: SchemaChangeTypeEnum; + targetTableName: string; + isReversible: boolean; + summary: string; + reasoning: string; +} + +let nextProposal: ProposedChange | null = null; +let nextProposals: ProposedChange[] | null = null; + +function resolveProposals(): ProposedChange[] { + if (nextProposals && nextProposals.length > 0) return nextProposals; + if (nextProposal) return [nextProposal]; + throw new Error('Test must set nextProposal or nextProposals before invoking AI.'); +} + +function createProposalStream(proposals: ProposedChange[]) { + return { + *[Symbol.asyncIterator]() { + yield { + type: 'tool_call', + toolCall: { + id: faker.string.uuid(), + name: 'proposeSchemaChange', + arguments: { proposals }, + }, + responseId: faker.string.uuid(), + }; + }, + }; +} + +const mockAICoreService = { + streamChatWithToolsAndProvider: async () => createProposalStream(resolveProposals()), + complete: async () => 'mocked', + chat: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChat: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + chatWithTools: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithTools: async () => createProposalStream(resolveProposals()), + chatWithToolsAndProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithProvider: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + completeWithProvider: async () => 'mocked', + chatWithProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueAfterToolCall: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueStreamingAfterToolCall: async () => createProposalStream(resolveProposals()), + getDefaultProvider: () => 'openai', + setDefaultProvider: () => {}, + getAvailableProviders: () => [], +}; + +test.beforeEach(() => { + nextProposal = null; + nextProposals = null; +}); + +const mockFactory = new MockFactory(); +let app: INestApplication; +let _testUtils: TestUtils; +const testTables: Array = []; + +test.before(async () => { + setSaasEnvVariable(); + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }) + .overrideProvider(AICoreService) + .useValue(mockAICoreService) + .compile(); + + _testUtils = moduleFixture.get(TestUtils); + app = moduleFixture.createNestApplication(); + app.use(cookieParser()); + app.useGlobalFilters(new AllExceptionsFilter(app.get(WinstonLogger))); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + await createInitialTestUser(app); + app.getHttpServer().listen(0); +}); + +test.after(async () => { + try { + const connectionToTestDB = getTestData(mockFactory).connectionToMySQL; + await dropTestTables(testTables, connectionToTestDB); + await Cacher.clearAllCache(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +async function createConnection(token: string): Promise { + const connectionToTestDB = getTestData(mockFactory).connectionToMySQL; + const resp = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + if (resp.status !== 201) { + throw new Error(`Failed to create connection: ${resp.status} ${resp.text}`); + } + return JSON.parse(resp.text).id; +} + +test.serial('MySQL: generate → approve creates the table', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_t_${faker.string.alphanumeric(8).toLowerCase()}`; + testTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE \`${tableName}\` (id INT AUTO_INCREMENT PRIMARY KEY, name TEXT)`, + rollbackSql: `DROP TABLE \`${tableName}\``, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'mysql create', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create table' }); + t.is(generateResp.status, 201); + const change = JSON.parse(generateResp.text).changes[0]; + t.is(change.status, SchemaChangeStatusEnum.PENDING); + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${change.id}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).status, SchemaChangeStatusEnum.APPLIED); + + const mysqlKnexParams = { ...getTestData(mockFactory).connectionToMySQL, type: 'mysql2' as const }; + const knex = getTestKnex(mysqlKnexParams); + t.true(await knex.schema.hasTable(tableName)); +}); + +test.serial('MySQL: generate → approve → rollback removes the table', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_t_${faker.string.alphanumeric(8).toLowerCase()}`; + testTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE \`${tableName}\` (id INT AUTO_INCREMENT PRIMARY KEY)`, + rollbackSql: `DROP TABLE \`${tableName}\``, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'rollback', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + await request(app.getHttpServer()).post(`/table-schema/change/${changeId}/approve`).set('Cookie', token).send({}); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + const rolledBack = JSON.parse(rollbackResp.text); + t.is(rolledBack.status, SchemaChangeStatusEnum.ROLLED_BACK); + t.truthy(rolledBack.rolledBackAt); + + const mysqlKnexParams = { ...getTestData(mockFactory).connectionToMySQL, type: 'mysql2' as const }; + const knex = getTestKnex(mysqlKnexParams); + t.false(await knex.schema.hasTable(tableName)); + + const listResp = await request(app.getHttpServer()).get(`/table-schema/${connectionId}/changes`).set('Cookie', token); + t.is(listResp.status, 200); + const list = JSON.parse(listResp.text); + const rollbackAuditRow = list.data.find( + (c: any) => c.changeType === SchemaChangeTypeEnum.ROLLBACK && c.previousChangeId === changeId, + ); + t.truthy(rollbackAuditRow, 'Expected a linked rollback audit row'); +}); + +test.serial('MySQL: generate → reject leaves DB untouched', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_rej_${faker.string.alphanumeric(6).toLowerCase()}`; + + nextProposal = { + forwardSql: `CREATE TABLE \`${tableName}\` (id INT AUTO_INCREMENT PRIMARY KEY)`, + rollbackSql: `DROP TABLE \`${tableName}\``, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'reject test', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'will reject' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const rejectResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/reject`) + .set('Cookie', token) + .send(); + t.is(rejectResp.status, 200); + t.is(JSON.parse(rejectResp.text).status, SchemaChangeStatusEnum.REJECTED); + + const mysqlKnexParams = { ...getTestData(mockFactory).connectionToMySQL, type: 'mysql2' as const }; + const knex = getTestKnex(mysqlKnexParams); + t.false(await knex.schema.hasTable(tableName)); +}); + +test.serial('MySQL: invalid SQL marks FAILED and attempts auto-rollback', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_bad_${faker.string.alphanumeric(6).toLowerCase()}`; + + nextProposal = { + forwardSql: `CREATE TABLE \`${tableName}\` (id INVALIDTYPE)`, + rollbackSql: `DROP TABLE IF EXISTS \`${tableName}\``, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'invalid', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'bad sql' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 400); + + const getResp = await request(app.getHttpServer()).get(`/table-schema/change/${changeId}`).set('Cookie', token); + const record = JSON.parse(getResp.text); + t.is(record.status, SchemaChangeStatusEnum.FAILED); + t.true(record.autoRollbackAttempted); + t.truthy(record.executionError); + + const mysqlKnexParams = { ...getTestData(mockFactory).connectionToMySQL, type: 'mysql2' as const }; + const knex = getTestKnex(mysqlKnexParams); + t.false(await knex.schema.hasTable(tableName)); +}); + +test.serial('MySQL: rollback of PENDING change is rejected', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_pend_${faker.string.alphanumeric(6).toLowerCase()}`; + + nextProposal = { + forwardSql: `CREATE TABLE \`${tableName}\` (id INT AUTO_INCREMENT PRIMARY KEY)`, + rollbackSql: `DROP TABLE \`${tableName}\``, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'pending rollback', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'pending' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 409); +}); + +test.serial('MySQL: destructive DROP TABLE requires confirmedDestructive=true', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const seedTableName = `ra_seed_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(seedTableName); + + const mysqlKnexParams = { ...getTestData(mockFactory).connectionToMySQL, type: 'mysql2' as const }; + const knex = getTestKnex(mysqlKnexParams); + await knex.schema.createTable(seedTableName, (table) => { + table.increments(); + }); + + nextProposal = { + forwardSql: `DROP TABLE \`${seedTableName}\``, + rollbackSql: `CREATE TABLE \`${seedTableName}\` (id INT AUTO_INCREMENT PRIMARY KEY)`, + changeType: SchemaChangeTypeEnum.DROP_TABLE, + targetTableName: seedTableName, + isReversible: false, + summary: 'drop existing', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'drop it' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const firstApproveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(firstApproveResp.status, 400); + t.is(firstApproveResp.body?.type, 'destructive_confirmation_required'); + t.true(await knex.schema.hasTable(seedTableName)); + + const secondApproveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ confirmedDestructive: true }); + t.is(secondApproveResp.status, 200); + t.is(JSON.parse(secondApproveResp.text).status, SchemaChangeStatusEnum.APPLIED); + t.false(await knex.schema.hasTable(seedTableName)); +}); + +test.serial('MySQL: ADD COLUMN approve adds the column; rollback removes it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_addcol_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(tableName); + + const mysqlKnexParams = { ...getTestData(mockFactory).connectionToMySQL, type: 'mysql2' as const }; + const knex = getTestKnex(mysqlKnexParams); + await knex.schema.createTable(tableName, (table) => { + table.increments(); + }); + + nextProposal = { + forwardSql: `ALTER TABLE \`${tableName}\` ADD COLUMN \`phone\` VARCHAR(255)`, + rollbackSql: `ALTER TABLE \`${tableName}\` DROP COLUMN \`phone\``, + changeType: SchemaChangeTypeEnum.ADD_COLUMN, + targetTableName: tableName, + isReversible: true, + summary: 'add phone', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add phone column' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).status, SchemaChangeStatusEnum.APPLIED); + t.true(await knex.schema.hasColumn(tableName, 'phone')); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + t.false(await knex.schema.hasColumn(tableName, 'phone')); +}); + +test.serial('MySQL: DROP COLUMN blocked without confirmedDestructive, succeeds with it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_dropcol_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(tableName); + + const mysqlKnexParams = { ...getTestData(mockFactory).connectionToMySQL, type: 'mysql2' as const }; + const knex = getTestKnex(mysqlKnexParams); + await knex.schema.createTable(tableName, (table) => { + table.increments(); + table.string('phone', 255); + }); + + nextProposal = { + forwardSql: `ALTER TABLE \`${tableName}\` DROP COLUMN \`phone\``, + rollbackSql: `ALTER TABLE \`${tableName}\` ADD COLUMN \`phone\` VARCHAR(255)`, + changeType: SchemaChangeTypeEnum.DROP_COLUMN, + targetTableName: tableName, + isReversible: false, + summary: 'drop phone', + reasoning: 'destructive', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'drop phone column' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const firstApproveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(firstApproveResp.status, 400); + t.is(firstApproveResp.body?.type, 'destructive_confirmation_required'); + t.true(await knex.schema.hasColumn(tableName, 'phone')); + + const secondApproveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ confirmedDestructive: true }); + t.is(secondApproveResp.status, 200); + t.is(JSON.parse(secondApproveResp.text).status, SchemaChangeStatusEnum.APPLIED); + t.false(await knex.schema.hasColumn(tableName, 'phone')); +}); + +test.serial('MySQL: userModifiedSql is validated and applied in place of AI SQL', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_um_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE \`${tableName}\` (id INT AUTO_INCREMENT PRIMARY KEY, foo TEXT)`, + rollbackSql: `DROP TABLE \`${tableName}\``, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'ai original', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create table' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const editedSql = `CREATE TABLE \`${tableName}\` (id INT AUTO_INCREMENT PRIMARY KEY, bar TEXT)`; + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: editedSql }); + t.is(approveResp.status, 200); + const applied = JSON.parse(approveResp.text); + t.is(applied.status, SchemaChangeStatusEnum.APPLIED); + t.is(applied.userModifiedSql, editedSql); + + const mysqlKnexParams = { ...getTestData(mockFactory).connectionToMySQL, type: 'mysql2' as const }; + const knex = getTestKnex(mysqlKnexParams); + t.true(await knex.schema.hasColumn(tableName, 'bar')); + t.false(await knex.schema.hasColumn(tableName, 'foo')); +}); + +test.serial('MySQL: userModifiedSql with a forbidden construct is rejected', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_umbad_${faker.string.alphanumeric(6).toLowerCase()}`; + + nextProposal = { + forwardSql: `CREATE TABLE \`${tableName}\` (id INT AUTO_INCREMENT PRIMARY KEY)`, + rollbackSql: `DROP TABLE \`${tableName}\``, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'valid ai proposal', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create table' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: `TRUNCATE TABLE \`${tableName}\`` }); + t.is(approveResp.status, 400); + + const getResp = await request(app.getHttpServer()).get(`/table-schema/change/${changeId}`).set('Cookie', token); + t.is(JSON.parse(getResp.text).status, SchemaChangeStatusEnum.PENDING); + + const mysqlKnexParams = { ...getTestData(mockFactory).connectionToMySQL, type: 'mysql2' as const }; + const knex = getTestKnex(mysqlKnexParams); + t.false(await knex.schema.hasTable(tableName)); +}); + +test.serial('MySQL: ADD INDEX approve creates a named index; rollback drops it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_idx_${faker.string.alphanumeric(6).toLowerCase()}`; + const indexName = `ix_${tableName}_email`; + testTables.push(tableName); + + const mysqlKnexParams = { ...getTestData(mockFactory).connectionToMySQL, type: 'mysql2' as const }; + const knex = getTestKnex(mysqlKnexParams); + await knex.schema.createTable(tableName, (table) => { + table.increments(); + table.string('email', 255); + }); + + nextProposal = { + forwardSql: `CREATE INDEX \`${indexName}\` ON \`${tableName}\` (\`email\`)`, + rollbackSql: `DROP INDEX \`${indexName}\` ON \`${tableName}\``, + changeType: SchemaChangeTypeEnum.ADD_INDEX, + targetTableName: tableName, + isReversible: true, + summary: 'add email index', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add index' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + + const afterAdd = await knex.raw( + `SELECT COUNT(*) AS c FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = ? AND index_name = ?`, + [tableName, indexName], + ); + t.is(Number(afterAdd[0][0].c), 1); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + + const afterRollback = await knex.raw( + `SELECT COUNT(*) AS c FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = ? AND index_name = ?`, + [tableName, indexName], + ); + t.is(Number(afterRollback[0][0].c), 0); +}); + +test.serial('MySQL: ADD FOREIGN KEY cross-table reference; rollback drops it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const parentName = `ra_par_${faker.string.alphanumeric(6).toLowerCase()}`; + const childName = `ra_ch_${faker.string.alphanumeric(6).toLowerCase()}`; + const fkName = `fk_${childName}_parent`; + testTables.push(parentName, childName); + + const mysqlKnexParams = { ...getTestData(mockFactory).connectionToMySQL, type: 'mysql2' as const }; + const knex = getTestKnex(mysqlKnexParams); + await knex.schema.createTable(parentName, (table) => { + table.increments(); + }); + await knex.schema.createTable(childName, (table) => { + table.increments(); + table.integer('parent_id').unsigned().notNullable(); + }); + + nextProposal = { + forwardSql: `ALTER TABLE \`${childName}\` ADD CONSTRAINT \`${fkName}\` FOREIGN KEY (\`parent_id\`) REFERENCES \`${parentName}\` (\`id\`)`, + rollbackSql: `ALTER TABLE \`${childName}\` DROP FOREIGN KEY \`${fkName}\``, + changeType: SchemaChangeTypeEnum.ADD_FOREIGN_KEY, + targetTableName: childName, + isReversible: true, + summary: 'add fk', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add fk' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + + const afterAdd = await knex.raw( + `SELECT COUNT(*) AS c FROM information_schema.table_constraints WHERE table_schema = DATABASE() AND table_name = ? AND constraint_name = ? AND constraint_type = 'FOREIGN KEY'`, + [childName, fkName], + ); + t.is(Number(afterAdd[0][0].c), 1); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + + const afterRollback = await knex.raw( + `SELECT COUNT(*) AS c FROM information_schema.table_constraints WHERE table_schema = DATABASE() AND table_name = ? AND constraint_name = ? AND constraint_type = 'FOREIGN KEY'`, + [childName, fkName], + ); + t.is(Number(afterRollback[0][0].c), 0); +}); + +test.serial('MySQL: ALTER COLUMN widens VARCHAR → TEXT while preserving row data', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_alt_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(tableName); + + const mysqlKnexParams = { ...getTestData(mockFactory).connectionToMySQL, type: 'mysql2' as const }; + const knex = getTestKnex(mysqlKnexParams); + await knex.schema.createTable(tableName, (table) => { + table.increments(); + table.string('name', 32); + }); + await knex(tableName).insert([{ name: 'alice' }, { name: 'bob' }, { name: 'carol' }]); + + nextProposal = { + forwardSql: `ALTER TABLE \`${tableName}\` MODIFY COLUMN \`name\` TEXT`, + rollbackSql: `ALTER TABLE \`${tableName}\` MODIFY COLUMN \`name\` VARCHAR(32)`, + changeType: SchemaChangeTypeEnum.ALTER_COLUMN, + targetTableName: tableName, + isReversible: true, + summary: 'widen name to text', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'widen name' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + + const typeRow = await knex.raw( + `SELECT data_type AS dt FROM information_schema.columns WHERE table_schema = DATABASE() AND table_name = ? AND column_name = 'name'`, + [tableName], + ); + t.is(String(typeRow[0][0].dt).toLowerCase(), 'text'); + + const rows = (await knex(tableName).select('name').orderBy('id')) as Array<{ name: string }>; + t.deepEqual( + rows.map((r) => r.name), + ['alice', 'bob', 'carol'], + ); +}); + +test.serial('MySQL: multi-table batch generate + approve creates every table in order', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const products = `ra_b_products_${faker.string.alphanumeric(6).toLowerCase()}`; + const users = `ra_b_users_${faker.string.alphanumeric(6).toLowerCase()}`; + const orders = `ra_b_orders_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(orders, users, products); + + nextProposals = [ + { + forwardSql: `CREATE TABLE \`${products}\` (id INT AUTO_INCREMENT PRIMARY KEY, name TEXT)`, + rollbackSql: `DROP TABLE \`${products}\``, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: products, + isReversible: true, + summary: 'products', + reasoning: '', + }, + { + forwardSql: `CREATE TABLE \`${users}\` (id INT AUTO_INCREMENT PRIMARY KEY, email VARCHAR(255))`, + rollbackSql: `DROP TABLE \`${users}\``, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: users, + isReversible: true, + summary: 'users', + reasoning: '', + }, + { + forwardSql: `CREATE TABLE \`${orders}\` (id INT AUTO_INCREMENT PRIMARY KEY, user_id INT, product_id INT, FOREIGN KEY (user_id) REFERENCES \`${users}\`(id), FOREIGN KEY (product_id) REFERENCES \`${products}\`(id))`, + rollbackSql: `DROP TABLE \`${orders}\``, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: orders, + isReversible: true, + summary: 'orders', + reasoning: '', + }, + ]; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create products, users and orders tables' }); + t.is(generateResp.status, 201); + const batch = JSON.parse(generateResp.text); + t.is(batch.changes.length, 3); + t.deepEqual( + batch.changes.map((c: any) => c.targetTableName), + [products, users, orders], + ); + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/batch/${batch.batchId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + + const mysqlKnexParams = { ...getTestData(mockFactory).connectionToMySQL, type: 'mysql2' as const }; + const knex = getTestKnex(mysqlKnexParams); + t.true(await knex.schema.hasTable(products)); + t.true(await knex.schema.hasTable(users)); + t.true(await knex.schema.hasTable(orders)); +}); diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-oracle-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-oracle-e2e.test.ts new file mode 100644 index 000000000..149fa73a5 --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-oracle-e2e.test.ts @@ -0,0 +1,284 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { faker } from '@faker-js/faker'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import { ValidationError } from 'class-validator'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { AICoreService } from '../../../src/ai-core/index.js'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { WinstonLogger } from '../../../src/entities/logging/winston-logger.js'; +import { + SchemaChangeStatusEnum, + SchemaChangeTypeEnum, +} from '../../../src/entities/table-schema/table-schema-change-enums.js'; +import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { dropTestTables } from '../../utils/drop-test-tables.js'; +import { getTestData } from '../../utils/get-test-data.js'; +import { getTestKnex } from '../../utils/get-test-knex.js'; +import { + createInitialTestUser, + registerUserAndReturnUserInfo, +} from '../../utils/register-user-and-return-user-info.js'; +import { setSaasEnvVariable } from '../../utils/set-saas-env-variable.js'; +import { TestUtils } from '../../utils/test.utils.js'; + +interface ProposedChange { + forwardSql: string; + rollbackSql: string; + changeType: SchemaChangeTypeEnum; + targetTableName: string; + isReversible: boolean; + summary: string; + reasoning: string; +} + +let nextProposal: ProposedChange | null = null; + +function createProposalStream(proposal: ProposedChange) { + return { + *[Symbol.asyncIterator]() { + yield { + type: 'tool_call', + toolCall: { + id: faker.string.uuid(), + name: 'proposeSchemaChange', + arguments: { proposals: [proposal] }, + }, + responseId: faker.string.uuid(), + }; + }, + }; +} + +const mockAICoreService = { + streamChatWithToolsAndProvider: async () => { + if (!nextProposal) throw new Error('Test must set nextProposal.'); + return createProposalStream(nextProposal); + }, + complete: async () => 'mocked', + chat: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChat: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + chatWithTools: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithTools: async () => createProposalStream(nextProposal!), + chatWithToolsAndProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithProvider: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + completeWithProvider: async () => 'mocked', + chatWithProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueAfterToolCall: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueStreamingAfterToolCall: async () => createProposalStream(nextProposal!), + getDefaultProvider: () => 'openai', + setDefaultProvider: () => {}, + getAvailableProviders: () => [], +}; + +const mockFactory = new MockFactory(); +let app: INestApplication; +let _testUtils: TestUtils; +const testTables: Array = []; + +test.before(async () => { + setSaasEnvVariable(); + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }) + .overrideProvider(AICoreService) + .useValue(mockAICoreService) + .compile(); + + _testUtils = moduleFixture.get(TestUtils); + app = moduleFixture.createNestApplication(); + app.use(cookieParser()); + app.useGlobalFilters(new AllExceptionsFilter(app.get(WinstonLogger))); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + await createInitialTestUser(app); + app.getHttpServer().listen(0); +}); + +test.after(async () => { + try { + const connectionToTestDB = getTestData(mockFactory).connectionToOracleDB; + await dropTestTables(testTables, connectionToTestDB); + await Cacher.clearAllCache(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +async function createConnection(token: string): Promise { + const connectionToTestDB = getTestData(mockFactory).connectionToOracleDB; + const resp = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + if (resp.status !== 201) { + throw new Error(`Failed to create connection: ${resp.status} ${resp.text}`); + } + return JSON.parse(resp.text).id; +} + +function oracleKnex() { + return getTestKnex(getTestData(mockFactory).connectionToOracleDB); +} + +function randomOracleTableName(prefix: string): string { + return `${prefix}_${faker.string.alphanumeric(6).toLowerCase()}`.toUpperCase(); +} + +test.serial('Oracle: generate → approve creates the table', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomOracleTableName('RA_T'); + testTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE ${tableName} (id NUMBER PRIMARY KEY, name VARCHAR2(255))`, + rollbackSql: `DROP TABLE ${tableName}`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'oracle create', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create table' }); + t.is(generateResp.status, 201); + const change = JSON.parse(generateResp.text).changes[0]; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${change.id}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).status, SchemaChangeStatusEnum.APPLIED); + + t.true(await oracleKnex().schema.hasTable(tableName)); +}); + +async function oracleColumnExists(tableName: string, columnName: string): Promise { + const rows = await oracleKnex().raw( + `SELECT COUNT(*) AS CNT FROM USER_TAB_COLUMNS WHERE TABLE_NAME = :t AND COLUMN_NAME = :c`, + { t: tableName, c: columnName }, + ); + const cnt = (rows as any)[0]?.CNT ?? (rows as any).rows?.[0]?.CNT ?? 0; + return Number(cnt) > 0; +} + +test.serial('Oracle: ADD COLUMN approve adds the column; rollback removes it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomOracleTableName('RA_ADDCOL'); + testTables.push(tableName); + + const knex = oracleKnex(); + await knex.raw(`CREATE TABLE ${tableName} (ID NUMBER PRIMARY KEY)`); + + nextProposal = { + forwardSql: `ALTER TABLE ${tableName} ADD PHONE VARCHAR2(255)`, + rollbackSql: `ALTER TABLE ${tableName} DROP COLUMN PHONE`, + changeType: SchemaChangeTypeEnum.ADD_COLUMN, + targetTableName: tableName, + isReversible: true, + summary: 'add phone', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add phone' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.true(await oracleColumnExists(tableName, 'PHONE')); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + t.false(await oracleColumnExists(tableName, 'PHONE')); +}); + +test.serial('Oracle: ALTER COLUMN widens VARCHAR2(32) → VARCHAR2(255) preserving row data', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = randomOracleTableName('RA_ALT'); + testTables.push(tableName); + + const knex = oracleKnex(); + await knex.raw(`CREATE TABLE ${tableName} (ID NUMBER PRIMARY KEY, NAME VARCHAR2(32))`); + await knex.raw(`INSERT INTO ${tableName} (ID, NAME) VALUES (1, 'alice')`); + await knex.raw(`INSERT INTO ${tableName} (ID, NAME) VALUES (2, 'bob')`); + await knex.raw(`INSERT INTO ${tableName} (ID, NAME) VALUES (3, 'carol')`); + await knex.raw('COMMIT'); + + nextProposal = { + forwardSql: `ALTER TABLE ${tableName} MODIFY NAME VARCHAR2(255)`, + rollbackSql: `ALTER TABLE ${tableName} MODIFY NAME VARCHAR2(32)`, + changeType: SchemaChangeTypeEnum.ALTER_COLUMN, + targetTableName: tableName, + isReversible: true, + summary: 'widen name', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'widen name' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + + const colMeta = await knex.raw( + `SELECT DATA_LENGTH FROM USER_TAB_COLUMNS WHERE TABLE_NAME = :t AND COLUMN_NAME = 'NAME'`, + { t: tableName }, + ); + const lenValue = Number((colMeta as any)[0]?.DATA_LENGTH ?? (colMeta as any).rows?.[0]?.DATA_LENGTH); + t.is(lenValue, 255); + + const rows = (await knex.raw(`SELECT ID, NAME FROM ${tableName} ORDER BY ID`)) as any; + const list = Array.isArray(rows) ? rows : (rows.rows ?? []); + t.deepEqual( + list.map((r: any) => r.NAME ?? r.name), + ['alice', 'bob', 'carol'], + ); +}); diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-postgres-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-postgres-e2e.test.ts new file mode 100644 index 000000000..044ae6206 --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-postgres-e2e.test.ts @@ -0,0 +1,983 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { faker } from '@faker-js/faker'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import { ValidationError } from 'class-validator'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { AICoreService } from '../../../src/ai-core/index.js'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { WinstonLogger } from '../../../src/entities/logging/winston-logger.js'; +import { + SchemaChangeStatusEnum, + SchemaChangeTypeEnum, +} from '../../../src/entities/table-schema/table-schema-change-enums.js'; +import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { dropTestTables } from '../../utils/drop-test-tables.js'; +import { getTestData } from '../../utils/get-test-data.js'; +import { getTestKnex } from '../../utils/get-test-knex.js'; +import { + createInitialTestUser, + registerUserAndReturnUserInfo, +} from '../../utils/register-user-and-return-user-info.js'; +import { setSaasEnvVariable } from '../../utils/set-saas-env-variable.js'; +import { TestUtils } from '../../utils/test.utils.js'; + +interface ProposedChange { + forwardSql: string; + rollbackSql: string; + changeType: SchemaChangeTypeEnum; + targetTableName: string; + isReversible: boolean; + summary: string; + reasoning: string; +} + +let nextProposal: ProposedChange | null = null; +let nextProposals: ProposedChange[] | null = null; + +function resolveProposals(): ProposedChange[] { + if (nextProposals && nextProposals.length > 0) return nextProposals; + if (nextProposal) return [nextProposal]; + throw new Error('Test must set nextProposal or nextProposals before invoking AI.'); +} + +function createProposalStream(proposals: ProposedChange[]) { + return { + *[Symbol.asyncIterator]() { + yield { + type: 'tool_call', + toolCall: { + id: faker.string.uuid(), + name: 'proposeSchemaChange', + arguments: { proposals }, + }, + responseId: faker.string.uuid(), + }; + }, + }; +} + +const mockAICoreService = { + streamChatWithToolsAndProvider: async () => createProposalStream(resolveProposals()), + complete: async () => 'Mocked completion', + chat: async () => ({ content: 'Mocked chat', responseId: faker.string.uuid() }), + streamChat: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + chatWithTools: async () => ({ content: 'Mocked tools', responseId: faker.string.uuid() }), + streamChatWithTools: async () => createProposalStream(resolveProposals()), + chatWithToolsAndProvider: async () => ({ content: 'Mocked', responseId: faker.string.uuid() }), + streamChatWithProvider: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + completeWithProvider: async () => 'Mocked', + chatWithProvider: async () => ({ content: 'Mocked', responseId: faker.string.uuid() }), + continueAfterToolCall: async () => ({ content: 'Mocked', responseId: faker.string.uuid() }), + continueStreamingAfterToolCall: async () => createProposalStream(resolveProposals()), + getDefaultProvider: () => 'openai', + setDefaultProvider: () => {}, + getAvailableProviders: () => [], +}; + +test.beforeEach(() => { + nextProposal = null; + nextProposals = null; +}); + +const mockFactory = new MockFactory(); +let app: INestApplication; +let _testUtils: TestUtils; +const testTables: Array = []; + +test.before(async () => { + setSaasEnvVariable(); + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }) + .overrideProvider(AICoreService) + .useValue(mockAICoreService) + .compile(); + + _testUtils = moduleFixture.get(TestUtils); + app = moduleFixture.createNestApplication(); + app.use(cookieParser()); + app.useGlobalFilters(new AllExceptionsFilter(app.get(WinstonLogger))); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + await createInitialTestUser(app); + app.getHttpServer().listen(0); +}); + +test.after(async () => { + try { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgres; + await dropTestTables(testTables, connectionToTestDB); + await Cacher.clearAllCache(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +async function createConnection(token: string): Promise { + const connectionToTestDB = getTestData(mockFactory).connectionToPostgres; + const resp = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + if (resp.status !== 201) { + throw new Error(`Failed to create connection: ${resp.status} ${resp.text}`); + } + return JSON.parse(resp.text).id; +} + +test.serial('generate → approve creates the table in the target DB', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_t_${faker.string.alphanumeric(8).toLowerCase()}`; + testTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE "${tableName}" (id SERIAL PRIMARY KEY, name TEXT)`, + rollbackSql: `DROP TABLE "${tableName}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: `Create ${tableName}`, + reasoning: 'User asked for a new table.', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .send({ userPrompt: `create a ${tableName} table with id and name` }); + t.is(generateResp.status, 201); + const batch = JSON.parse(generateResp.text); + t.truthy(batch.batchId); + t.is(batch.changes.length, 1); + const change = batch.changes[0]; + t.is(change.status, SchemaChangeStatusEnum.PENDING); + t.is(change.targetTableName, tableName); + t.is(change.batchId, batch.batchId); + t.is(change.orderInBatch, 0); + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${change.id}/approve`) + .set('Cookie', token) + .set('Content-Type', 'application/json') + .send({}); + t.is(approveResp.status, 200); + const applied = JSON.parse(approveResp.text); + t.is(applied.status, SchemaChangeStatusEnum.APPLIED); + t.truthy(applied.appliedAt); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + t.true(await knex.schema.hasTable(tableName)); +}); + +test.serial('generate → approve → rollback removes the table and creates linked audit row', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_t_${faker.string.alphanumeric(8).toLowerCase()}`; + testTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE "${tableName}" (id SERIAL PRIMARY KEY)`, + rollbackSql: `DROP TABLE "${tableName}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: `Create ${tableName}`, + reasoning: 'Rollback test.', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create table' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + await request(app.getHttpServer()).post(`/table-schema/change/${changeId}/approve`).set('Cookie', token).send({}); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send({}); + t.is(rollbackResp.status, 200); + const rolledBack = JSON.parse(rollbackResp.text); + t.is(rolledBack.status, SchemaChangeStatusEnum.ROLLED_BACK); + t.truthy(rolledBack.rolledBackAt); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + t.false(await knex.schema.hasTable(tableName)); + + const listResp = await request(app.getHttpServer()).get(`/table-schema/${connectionId}/changes`).set('Cookie', token); + t.is(listResp.status, 200); + const list = JSON.parse(listResp.text); + const rollbackAuditRow = list.data.find( + (c: any) => c.changeType === SchemaChangeTypeEnum.ROLLBACK && c.previousChangeId === changeId, + ); + t.truthy(rollbackAuditRow, 'Expected a linked rollback audit row'); +}); + +test.serial('generate → reject leaves DB untouched', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_t_${faker.string.alphanumeric(8).toLowerCase()}`; + + nextProposal = { + forwardSql: `CREATE TABLE "${tableName}" (id SERIAL PRIMARY KEY)`, + rollbackSql: `DROP TABLE "${tableName}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'reject test', + reasoning: 'reject test', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'will reject' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const rejectResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/reject`) + .set('Cookie', token) + .send(); + t.is(rejectResp.status, 200); + t.is(JSON.parse(rejectResp.text).status, SchemaChangeStatusEnum.REJECTED); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + t.false(await knex.schema.hasTable(tableName)); +}); + +test.serial('approve with invalid SQL marks FAILED and attempts auto-rollback', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_bad_${faker.string.alphanumeric(6).toLowerCase()}`; + + nextProposal = { + forwardSql: `CREATE TABLE "${tableName}" (id INVALIDTYPE)`, + rollbackSql: `DROP TABLE IF EXISTS "${tableName}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'invalid', + reasoning: 'will fail', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'bad sql' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 400); + + const getResp = await request(app.getHttpServer()).get(`/table-schema/change/${changeId}`).set('Cookie', token); + const record = JSON.parse(getResp.text); + t.is(record.status, SchemaChangeStatusEnum.FAILED); + t.true(record.autoRollbackAttempted); + t.truthy(record.executionError); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + t.false(await knex.schema.hasTable(tableName)); +}); + +test.serial('rollback of PENDING change is rejected', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_pend_${faker.string.alphanumeric(6).toLowerCase()}`; + + nextProposal = { + forwardSql: `CREATE TABLE "${tableName}" (id SERIAL PRIMARY KEY)`, + rollbackSql: `DROP TABLE "${tableName}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'pending rollback test', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'pending' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 409); +}); + +test.serial('destructive change requires confirmedDestructive=true', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const seedTableName = `ra_seed_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(seedTableName); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + await knex.schema.createTable(seedTableName, (table) => { + table.increments(); + }); + + nextProposal = { + forwardSql: `DROP TABLE "${seedTableName}"`, + rollbackSql: `CREATE TABLE "${seedTableName}" (id SERIAL PRIMARY KEY)`, + changeType: SchemaChangeTypeEnum.DROP_TABLE, + targetTableName: seedTableName, + isReversible: false, + summary: 'destructive', + reasoning: 'user wants to drop', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'drop it' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const firstApproveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(firstApproveResp.status, 400); + t.is(firstApproveResp.body?.type, 'destructive_confirmation_required'); + + t.true(await knex.schema.hasTable(seedTableName), 'table must still exist after first attempt'); + + const secondApproveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ confirmedDestructive: true }); + t.is(secondApproveResp.status, 200); + t.is(JSON.parse(secondApproveResp.text).status, SchemaChangeStatusEnum.APPLIED); + t.false(await knex.schema.hasTable(seedTableName)); +}); + +test.serial('ADD COLUMN: approve adds the column; rollback removes it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_addcol_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(tableName); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + await knex.schema.createTable(tableName, (table) => { + table.increments(); + }); + + nextProposal = { + forwardSql: `ALTER TABLE "${tableName}" ADD COLUMN "phone" VARCHAR(255)`, + rollbackSql: `ALTER TABLE "${tableName}" DROP COLUMN "phone"`, + changeType: SchemaChangeTypeEnum.ADD_COLUMN, + targetTableName: tableName, + isReversible: true, + summary: 'add phone column', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add phone column' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + t.is(JSON.parse(approveResp.text).status, SchemaChangeStatusEnum.APPLIED); + t.true(await knex.schema.hasColumn(tableName, 'phone')); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + t.is(JSON.parse(rollbackResp.text).status, SchemaChangeStatusEnum.ROLLED_BACK); + t.false(await knex.schema.hasColumn(tableName, 'phone')); +}); + +test.serial('DROP COLUMN: blocked without confirmedDestructive, succeeds with it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_dropcol_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(tableName); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + await knex.schema.createTable(tableName, (table) => { + table.increments(); + table.string('phone', 255); + }); + + nextProposal = { + forwardSql: `ALTER TABLE "${tableName}" DROP COLUMN "phone"`, + rollbackSql: `ALTER TABLE "${tableName}" ADD COLUMN "phone" VARCHAR(255)`, + changeType: SchemaChangeTypeEnum.DROP_COLUMN, + targetTableName: tableName, + isReversible: false, + summary: 'drop phone column', + reasoning: 'destructive drop', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'drop phone column' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const firstApproveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(firstApproveResp.status, 400); + t.is(firstApproveResp.body?.type, 'destructive_confirmation_required'); + t.true(await knex.schema.hasColumn(tableName, 'phone')); + + const secondApproveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ confirmedDestructive: true }); + t.is(secondApproveResp.status, 200); + t.is(JSON.parse(secondApproveResp.text).status, SchemaChangeStatusEnum.APPLIED); + t.false(await knex.schema.hasColumn(tableName, 'phone')); +}); + +test.serial('userModifiedSql is validated and applied in place of AI SQL', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_um_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(tableName); + + nextProposal = { + forwardSql: `CREATE TABLE "${tableName}" (id SERIAL PRIMARY KEY, foo TEXT)`, + rollbackSql: `DROP TABLE "${tableName}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'ai original', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create table' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const editedSql = `CREATE TABLE "${tableName}" (id SERIAL PRIMARY KEY, bar TEXT)`; + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: editedSql }); + t.is(approveResp.status, 200); + const applied = JSON.parse(approveResp.text); + t.is(applied.status, SchemaChangeStatusEnum.APPLIED); + t.is(applied.userModifiedSql, editedSql); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + t.true(await knex.schema.hasColumn(tableName, 'bar')); + t.false(await knex.schema.hasColumn(tableName, 'foo')); +}); + +test.serial('userModifiedSql with a forbidden construct is rejected', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_umbad_${faker.string.alphanumeric(6).toLowerCase()}`; + + nextProposal = { + forwardSql: `CREATE TABLE "${tableName}" (id SERIAL PRIMARY KEY)`, + rollbackSql: `DROP TABLE "${tableName}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: tableName, + isReversible: true, + summary: 'valid ai proposal', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create table' }); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({ userModifiedSql: `GRANT ALL ON "${tableName}" TO PUBLIC` }); + t.is(approveResp.status, 400); + + const getResp = await request(app.getHttpServer()).get(`/table-schema/change/${changeId}`).set('Cookie', token); + t.is(JSON.parse(getResp.text).status, SchemaChangeStatusEnum.PENDING); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + t.false(await knex.schema.hasTable(tableName)); +}); + +test.serial('ADD INDEX approve creates a named index; rollback drops it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_idx_${faker.string.alphanumeric(6).toLowerCase()}`; + const indexName = `ix_${tableName}_email`; + testTables.push(tableName); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + await knex.schema.createTable(tableName, (table) => { + table.increments(); + table.string('email', 255); + }); + + nextProposal = { + forwardSql: `CREATE INDEX "${indexName}" ON "${tableName}" ("email")`, + rollbackSql: `DROP INDEX "${indexName}"`, + changeType: SchemaChangeTypeEnum.ADD_INDEX, + targetTableName: tableName, + isReversible: true, + summary: 'add email index', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add index' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + + const afterAdd = await knex.raw(`SELECT COUNT(*)::int AS c FROM pg_indexes WHERE indexname = ?`, [indexName]); + t.is(Number(afterAdd.rows[0].c), 1); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + + const afterRollback = await knex.raw(`SELECT COUNT(*)::int AS c FROM pg_indexes WHERE indexname = ?`, [indexName]); + t.is(Number(afterRollback.rows[0].c), 0); +}); + +test.serial('ADD FOREIGN KEY cross-table reference; rollback drops it', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const parentName = `ra_par_${faker.string.alphanumeric(6).toLowerCase()}`; + const childName = `ra_ch_${faker.string.alphanumeric(6).toLowerCase()}`; + const fkName = `fk_${childName}_parent`; + testTables.push(parentName, childName); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + await knex.schema.createTable(parentName, (table) => { + table.increments(); + }); + await knex.schema.createTable(childName, (table) => { + table.increments(); + table.integer('parent_id').notNullable(); + }); + + nextProposal = { + forwardSql: `ALTER TABLE "${childName}" ADD CONSTRAINT "${fkName}" FOREIGN KEY ("parent_id") REFERENCES "${parentName}" ("id")`, + rollbackSql: `ALTER TABLE "${childName}" DROP CONSTRAINT "${fkName}"`, + changeType: SchemaChangeTypeEnum.ADD_FOREIGN_KEY, + targetTableName: childName, + isReversible: true, + summary: 'add fk', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'add fk' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + + const afterAdd = await knex.raw(`SELECT COUNT(*)::int AS c FROM pg_constraint WHERE conname = ? AND contype = 'f'`, [ + fkName, + ]); + t.is(Number(afterAdd.rows[0].c), 1); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/rollback`) + .set('Cookie', token) + .send(); + t.is(rollbackResp.status, 200); + + const afterRollback = await knex.raw( + `SELECT COUNT(*)::int AS c FROM pg_constraint WHERE conname = ? AND contype = 'f'`, + [fkName], + ); + t.is(Number(afterRollback.rows[0].c), 0); +}); + +test.serial('ALTER COLUMN widens VARCHAR → TEXT while preserving row data', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const tableName = `ra_alt_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(tableName); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + await knex.schema.createTable(tableName, (table) => { + table.increments(); + table.string('name', 32); + }); + await knex(tableName).insert([{ name: 'alice' }, { name: 'bob' }, { name: 'carol' }]); + + nextProposal = { + forwardSql: `ALTER TABLE "${tableName}" ALTER COLUMN "name" TYPE TEXT`, + rollbackSql: `ALTER TABLE "${tableName}" ALTER COLUMN "name" TYPE VARCHAR(32)`, + changeType: SchemaChangeTypeEnum.ALTER_COLUMN, + targetTableName: tableName, + isReversible: true, + summary: 'widen name column to text', + reasoning: '', + }; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'widen name to text' }); + t.is(generateResp.status, 201); + const changeId = JSON.parse(generateResp.text).changes[0].id; + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/change/${changeId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + + const typeRow = await knex.raw( + `SELECT data_type FROM information_schema.columns WHERE table_name = ? AND column_name = 'name'`, + [tableName], + ); + t.is(typeRow.rows[0].data_type, 'text'); + + const rows = (await knex(tableName).select('name').orderBy('id')) as Array<{ name: string }>; + t.deepEqual( + rows.map((r) => r.name), + ['alice', 'bob', 'carol'], + ); +}); + +test.serial('multi-table batch: generate creates one batch with N pending changes in dependency order', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const products = `ra_products_${faker.string.alphanumeric(6).toLowerCase()}`; + const users = `ra_users_${faker.string.alphanumeric(6).toLowerCase()}`; + const orders = `ra_orders_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(orders, users, products); + + nextProposals = [ + { + forwardSql: `CREATE TABLE "${products}" (id SERIAL PRIMARY KEY, name TEXT, price NUMERIC)`, + rollbackSql: `DROP TABLE "${products}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: products, + isReversible: true, + summary: 'create products', + reasoning: '', + }, + { + forwardSql: `CREATE TABLE "${users}" (id SERIAL PRIMARY KEY, email TEXT)`, + rollbackSql: `DROP TABLE "${users}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: users, + isReversible: true, + summary: 'create users', + reasoning: '', + }, + { + forwardSql: `CREATE TABLE "${orders}" (id SERIAL PRIMARY KEY, user_id INT REFERENCES "${users}"(id), product_id INT REFERENCES "${products}"(id))`, + rollbackSql: `DROP TABLE "${orders}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: orders, + isReversible: true, + summary: 'create orders', + reasoning: '', + }, + ]; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create tables for products, users and orders' }); + t.is(generateResp.status, 201); + const batch = JSON.parse(generateResp.text); + t.truthy(batch.batchId); + t.is(batch.changes.length, 3); + t.is(batch.changes[0].targetTableName, products); + t.is(batch.changes[1].targetTableName, users); + t.is(batch.changes[2].targetTableName, orders); + t.deepEqual( + batch.changes.map((c: any) => c.orderInBatch), + [0, 1, 2], + ); + t.true(batch.changes.every((c: any) => c.batchId === batch.batchId)); + t.true(batch.changes.every((c: any) => c.status === SchemaChangeStatusEnum.PENDING)); +}); + +test.serial('multi-table batch: approve applies every change in order', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const products = `ra_b_products_${faker.string.alphanumeric(6).toLowerCase()}`; + const users = `ra_b_users_${faker.string.alphanumeric(6).toLowerCase()}`; + const orders = `ra_b_orders_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(orders, users, products); + + nextProposals = [ + { + forwardSql: `CREATE TABLE "${products}" (id SERIAL PRIMARY KEY)`, + rollbackSql: `DROP TABLE "${products}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: products, + isReversible: true, + summary: 'create products', + reasoning: '', + }, + { + forwardSql: `CREATE TABLE "${users}" (id SERIAL PRIMARY KEY)`, + rollbackSql: `DROP TABLE "${users}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: users, + isReversible: true, + summary: 'create users', + reasoning: '', + }, + { + forwardSql: `CREATE TABLE "${orders}" (id SERIAL PRIMARY KEY, user_id INT REFERENCES "${users}"(id), product_id INT REFERENCES "${products}"(id))`, + rollbackSql: `DROP TABLE "${orders}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: orders, + isReversible: true, + summary: 'create orders', + reasoning: '', + }, + ]; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create tables for products, users and orders' }); + const { batchId } = JSON.parse(generateResp.text); + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/batch/${batchId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 200); + const applied = JSON.parse(approveResp.text); + t.is(applied.changes.length, 3); + t.true(applied.changes.every((c: any) => c.status === SchemaChangeStatusEnum.APPLIED)); + t.true(applied.changes.every((c: any) => !!c.appliedAt)); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + t.true(await knex.schema.hasTable(products)); + t.true(await knex.schema.hasTable(users)); + t.true(await knex.schema.hasTable(orders)); +}); + +test.serial('multi-table batch: mid-failure auto-rolls back already-applied items', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const ok1 = `ra_f_ok1_${faker.string.alphanumeric(6).toLowerCase()}`; + const ok2 = `ra_f_ok2_${faker.string.alphanumeric(6).toLowerCase()}`; + const bad = `ra_f_bad_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(ok1, ok2, bad); + + nextProposals = [ + { + forwardSql: `CREATE TABLE "${ok1}" (id SERIAL PRIMARY KEY)`, + rollbackSql: `DROP TABLE "${ok1}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: ok1, + isReversible: true, + summary: 'ok1', + reasoning: '', + }, + { + forwardSql: `CREATE TABLE "${ok2}" (id SERIAL PRIMARY KEY)`, + rollbackSql: `DROP TABLE "${ok2}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: ok2, + isReversible: true, + summary: 'ok2', + reasoning: '', + }, + { + forwardSql: `CREATE TABLE "${bad}" (id INVALIDTYPE)`, + rollbackSql: `DROP TABLE IF EXISTS "${bad}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: bad, + isReversible: true, + summary: 'broken', + reasoning: '', + }, + ]; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'three tables but one fails' }); + const { batchId } = JSON.parse(generateResp.text); + + const approveResp = await request(app.getHttpServer()) + .post(`/table-schema/batch/${batchId}/approve`) + .set('Cookie', token) + .send({}); + t.is(approveResp.status, 400); + t.regex(approveResp.body?.message ?? '', /failed at order 2/); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + t.false(await knex.schema.hasTable(ok1)); + t.false(await knex.schema.hasTable(ok2)); + t.false(await knex.schema.hasTable(bad)); + + const getResp = await request(app.getHttpServer()).get(`/table-schema/batch/${batchId}`).set('Cookie', token); + const final = JSON.parse(getResp.text); + t.is(final.changes[0].status, SchemaChangeStatusEnum.ROLLED_BACK); + t.is(final.changes[1].status, SchemaChangeStatusEnum.ROLLED_BACK); + t.is(final.changes[2].status, SchemaChangeStatusEnum.FAILED); +}); + +test.serial('multi-table batch: reject marks every pending change REJECTED and leaves DB untouched', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const a = `ra_rj_a_${faker.string.alphanumeric(6).toLowerCase()}`; + const b = `ra_rj_b_${faker.string.alphanumeric(6).toLowerCase()}`; + + nextProposals = [ + { + forwardSql: `CREATE TABLE "${a}" (id SERIAL PRIMARY KEY)`, + rollbackSql: `DROP TABLE "${a}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: a, + isReversible: true, + summary: 'a', + reasoning: '', + }, + { + forwardSql: `CREATE TABLE "${b}" (id SERIAL PRIMARY KEY)`, + rollbackSql: `DROP TABLE "${b}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: b, + isReversible: true, + summary: 'b', + reasoning: '', + }, + ]; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'two tables to reject' }); + const { batchId } = JSON.parse(generateResp.text); + + const rejectResp = await request(app.getHttpServer()) + .post(`/table-schema/batch/${batchId}/reject`) + .set('Cookie', token) + .send({}); + t.is(rejectResp.status, 200); + const rejected = JSON.parse(rejectResp.text); + t.true(rejected.changes.every((c: any) => c.status === SchemaChangeStatusEnum.REJECTED)); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + t.false(await knex.schema.hasTable(a)); + t.false(await knex.schema.hasTable(b)); +}); + +test.serial('multi-table batch: rollback removes tables in reverse and creates linked audit rows', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createConnection(token); + const parent = `ra_rb_par_${faker.string.alphanumeric(6).toLowerCase()}`; + const child = `ra_rb_ch_${faker.string.alphanumeric(6).toLowerCase()}`; + testTables.push(child, parent); + + nextProposals = [ + { + forwardSql: `CREATE TABLE "${parent}" (id SERIAL PRIMARY KEY)`, + rollbackSql: `DROP TABLE "${parent}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: parent, + isReversible: true, + summary: 'parent', + reasoning: '', + }, + { + forwardSql: `CREATE TABLE "${child}" (id SERIAL PRIMARY KEY, parent_id INT REFERENCES "${parent}"(id))`, + rollbackSql: `DROP TABLE "${child}"`, + changeType: SchemaChangeTypeEnum.CREATE_TABLE, + targetTableName: child, + isReversible: true, + summary: 'child', + reasoning: '', + }, + ]; + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'parent + child' }); + const { batchId } = JSON.parse(generateResp.text); + + await request(app.getHttpServer()).post(`/table-schema/batch/${batchId}/approve`).set('Cookie', token).send({}); + + const knex = getTestKnex(getTestData(mockFactory).connectionToPostgres); + t.true(await knex.schema.hasTable(parent)); + t.true(await knex.schema.hasTable(child)); + + const rollbackResp = await request(app.getHttpServer()) + .post(`/table-schema/batch/${batchId}/rollback`) + .set('Cookie', token) + .send({}); + t.is(rollbackResp.status, 200); + const rolled = JSON.parse(rollbackResp.text); + t.true(rolled.changes.every((c: any) => c.status === SchemaChangeStatusEnum.ROLLED_BACK)); + + t.false(await knex.schema.hasTable(child)); + t.false(await knex.schema.hasTable(parent)); + + const listResp = await request(app.getHttpServer()).get(`/table-schema/${connectionId}/changes`).set('Cookie', token); + const list = JSON.parse(listResp.text); + const auditRows = list.data.filter((c: any) => c.changeType === SchemaChangeTypeEnum.ROLLBACK); + t.is(auditRows.length, 2); +}); diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-redis-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-redis-e2e.test.ts new file mode 100644 index 000000000..5123dd045 --- /dev/null +++ b/backend/test/ava-tests/non-saas-tests/non-saas-table-schema-redis-e2e.test.ts @@ -0,0 +1,134 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { faker } from '@faker-js/faker'; +import { INestApplication, ValidationPipe } from '@nestjs/common'; +import { Test } from '@nestjs/testing'; +import test from 'ava'; +import { ValidationError } from 'class-validator'; +import cookieParser from 'cookie-parser'; +import request from 'supertest'; +import { AICoreService } from '../../../src/ai-core/index.js'; +import { ApplicationModule } from '../../../src/app.module.js'; +import { WinstonLogger } from '../../../src/entities/logging/winston-logger.js'; +import { AllExceptionsFilter } from '../../../src/exceptions/all-exceptions.filter.js'; +import { ValidationException } from '../../../src/exceptions/custom-exceptions/validation-exception.js'; +import { Cacher } from '../../../src/helpers/cache/cacher.js'; +import { DatabaseModule } from '../../../src/shared/database/database.module.js'; +import { DatabaseService } from '../../../src/shared/database/database.service.js'; +import { MockFactory } from '../../mock.factory.js'; +import { getTestData } from '../../utils/get-test-data.js'; +import { + createInitialTestUser, + registerUserAndReturnUserInfo, +} from '../../utils/register-user-and-return-user-info.js'; +import { setSaasEnvVariable } from '../../utils/set-saas-env-variable.js'; +import { TestUtils } from '../../utils/test.utils.js'; + +// Redis is a key-value store with no schema concept; the AI table-schema feature must +// reject it at /generate before any AI call. The streamChatWithToolsAndProvider mock +// throws so any code path that reaches the AI loop fails the assertion. +const mockAICoreService = { + streamChatWithToolsAndProvider: async () => { + throw new Error('AI must not be called for unsupported dialect (redis).'); + }, + complete: async () => 'mocked', + chat: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChat: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + chatWithTools: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithTools: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + chatWithToolsAndProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + streamChatWithProvider: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + completeWithProvider: async () => 'mocked', + chatWithProvider: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueAfterToolCall: async () => ({ content: 'mocked', responseId: faker.string.uuid() }), + continueStreamingAfterToolCall: async () => ({ + *[Symbol.asyncIterator]() { + yield { type: 'text', content: 'mocked', responseId: faker.string.uuid() }; + }, + }), + getDefaultProvider: () => 'openai', + setDefaultProvider: () => {}, + getAvailableProviders: () => [], +}; + +const mockFactory = new MockFactory(); +let app: INestApplication; +let _testUtils: TestUtils; + +test.before(async () => { + setSaasEnvVariable(); + const moduleFixture = await Test.createTestingModule({ + imports: [ApplicationModule, DatabaseModule], + providers: [DatabaseService, TestUtils], + }) + .overrideProvider(AICoreService) + .useValue(mockAICoreService) + .compile(); + + _testUtils = moduleFixture.get(TestUtils); + app = moduleFixture.createNestApplication(); + app.use(cookieParser()); + app.useGlobalFilters(new AllExceptionsFilter(app.get(WinstonLogger))); + app.useGlobalPipes( + new ValidationPipe({ + exceptionFactory(validationErrors: ValidationError[] = []) { + return new ValidationException(validationErrors); + }, + }), + ); + await app.init(); + await createInitialTestUser(app); + app.getHttpServer().listen(0); +}); + +test.after(async () => { + try { + await Cacher.clearAllCache(); + await app.close(); + } catch (e) { + console.error('After tests error ' + e); + } +}); + +async function createRedisConnection(token: string): Promise { + const connectionToTestDB = getTestData(mockFactory).redisConnection; + const resp = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToTestDB) + .set('Cookie', token) + .set('Content-Type', 'application/json'); + if (resp.status !== 201) { + throw new Error(`Failed to create connection: ${resp.status} ${resp.text}`); + } + return JSON.parse(resp.text).id; +} + +test.serial('Redis: /table-schema/:id/generate returns 400 with "not yet supported"', async (t) => { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionId = await createRedisConnection(token); + + const generateResp = await request(app.getHttpServer()) + .post(`/table-schema/${connectionId}/generate`) + .set('Cookie', token) + .send({ userPrompt: 'create a users hash' }); + + t.is(generateResp.status, 400); + const body = generateResp.body ?? JSON.parse(generateResp.text || '{}'); + const message: string = body?.message ?? generateResp.text ?? ''; + t.true( + message.toLowerCase().includes('not yet supported'), + `expected "not yet supported" message, got: ${JSON.stringify(body)}`, + ); + t.true(message.toLowerCase().includes('redis'), `expected message to mention redis, got: ${JSON.stringify(body)}`); +}); diff --git a/backend/test/mock.factory.ts b/backend/test/mock.factory.ts index 4bf4ab616..b6a9b9d04 100644 --- a/backend/test/mock.factory.ts +++ b/backend/test/mock.factory.ts @@ -108,7 +108,7 @@ export class MockFactory { const dto = new CreateConnectionDto() as any; dto.title = 'Test connection to ClickHouse in Docker'; dto.type = 'clickhouse'; - dto.host = 'clickhouse-e2e-testing'; + dto.host = 'test-clickhouse-e2e-testing'; dto.port = 8123; dto.username = 'default'; dto.password = 'clickhouse_password'; diff --git a/docker-compose.tst.yml b/docker-compose.tst.yml index 91e97b28a..93985ff58 100644 --- a/docker-compose.tst.yml +++ b/docker-compose.tst.yml @@ -33,6 +33,8 @@ services: condition: service_healthy test-clickhouse-e2e-testing: condition: service_healthy + test-elasticsearch-e2e-testing: + condition: service_healthy command: ["/bin/sh", "-c", "pnpm test-all $EXTRA_ARGS"] develop: watch: @@ -222,7 +224,8 @@ services: start_period: 10s test-clickhouse-e2e-testing: - image: clickhouse/clickhouse-server:latest + image: clickhouse/clickhouse-server:24.8 + restart: unless-stopped ports: - 8123:8123 - 9000:9000 @@ -236,7 +239,39 @@ services: soft: 262144 hard: 262144 healthcheck: - test: [ "CMD", "wget", "--spider", "-q", "http://localhost:8123/ping" ] - interval: 30s - timeout: 10s - retries: 3 + test: + [ + "CMD", + "clickhouse-client", + "--user=default", + "--password=clickhouse_password", + "--query=SELECT 1", + ] + interval: 10s + timeout: 5s + retries: 20 + start_period: 60s + + test-elasticsearch-e2e-testing: + image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0 + environment: + - discovery.type=single-node + - xpack.security.enabled=true + - ELASTIC_PASSWORD=SuperSecretElasticPassword + - ES_JAVA_OPTS=-Xms512m -Xmx512m + ports: + - 9200:9200 + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: + [ + "CMD-SHELL", + "curl -s -u elastic:SuperSecretElasticPassword http://localhost:9200/_cluster/health | grep -E '\"status\":\"(green|yellow)\"'", + ] + interval: 10s + timeout: 5s + retries: 20 + start_period: 60s diff --git a/docker-compose.yml b/docker-compose.yml index dd1458726..9204586b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,7 +23,7 @@ services: - cassandra-init - test-cassandra-e2e-testing - test-redis-e2e-testing - - clickhouse-e2e-testing + - test-clickhouse-e2e-testing links: - postgres - testMySQL-e2e-testing @@ -36,7 +36,7 @@ services: - test-elasticsearch-e2e-testing - test-cassandra-e2e-testing - test-redis-e2e-testing - - clickhouse-e2e-testing + - test-clickhouse-e2e-testing command: ["pnpm", "start"] testMySQL-e2e-testing: @@ -170,11 +170,21 @@ services: - discovery.type=single-node - ELASTIC_PASSWORD=SuperSecretElasticPassword - xpack.security.enabled=true + - ES_JAVA_OPTS=-Xms512m -Xmx512m + ulimits: + memlock: + soft: -1 + hard: -1 healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9200"] - interval: 30s - timeout: 10s - retries: 3 + test: + [ + "CMD-SHELL", + "curl -s -u elastic:SuperSecretElasticPassword http://localhost:9200/_cluster/health | grep -E '\"status\":\"(green|yellow)\"'", + ] + interval: 10s + timeout: 5s + retries: 20 + start_period: 60s test-cassandra-e2e-testing: image: cassandra:5.0.4 @@ -223,8 +233,9 @@ services: timeout: 10s retries: 3 - clickhouse-e2e-testing: - image: clickhouse/clickhouse-server:latest + test-clickhouse-e2e-testing: + image: clickhouse/clickhouse-server:24.8 + restart: unless-stopped ports: - 8123:8123 - 9000:9000 @@ -238,10 +249,18 @@ services: soft: 262144 hard: 262144 healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"] - interval: 30s - timeout: 10s - retries: 3 + test: + [ + "CMD", + "clickhouse-client", + "--user=default", + "--password=clickhouse_password", + "--query=SELECT 1", + ] + interval: 10s + timeout: 5s + retries: 20 + start_period: 60s # === Postgres Proxy E2E Testing === @@ -453,10 +472,10 @@ services: - ./rocketadmin-agent/src:/app/src links: - autoadmin-ws-server - - clickhouse-e2e-testing + - test-clickhouse-e2e-testing depends_on: - autoadmin-ws-server - - clickhouse-e2e-testing + - test-clickhouse-e2e-testing environment: - REMOTE_WEBSOCKET_ADDRESS=ws://autoadmin-ws-server:8009 - APPLICATION_CONFIG_FILE_NAME=.clickhouse_test_agent_config.txt diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0d9ca9e7f..8b2981b2e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -202,6 +202,9 @@ importers: nanoid: specifier: 5.1.7 version: 5.1.7 + node-sql-parser: + specifier: ^5.3.0 + version: 5.4.0 nodemailer: specifier: ^8.0.4 version: 8.0.5 @@ -2489,6 +2492,9 @@ packages: '@types/oracledb@6.10.3': resolution: {integrity: sha512-8SCGQF/VSiXnifVFTajmRNtjHUDTNxkd9SrBrvG2A/80uwLVee5xmv/juH4hnDNTfU9qxfd6HR4Mfmg2nC3RVw==} + '@types/pegjs@0.10.6': + resolution: {integrity: sha512-eLYXDbZWXh2uxf+w8sXS8d6KSoXTswfps6fvCUuVAGN8eRpfe7h9eSRydxiSJvo9Bf+GzifsDOr9TMQlmJdmkw==} + '@types/pg-copy-streams@1.2.5': resolution: {integrity: sha512-7D6/GYW2uHIaVU6S/5omI+6RZnwlZBpLQDZAH83xX1rjxAOK0f6/deKyyUTewxqts145VIGn6XWYz1YGf50G5g==} @@ -4521,6 +4527,10 @@ packages: node-releases@2.0.37: resolution: {integrity: sha512-1h5gKZCF+pO/o3Iqt5Jp7wc9rH3eJJ0+nh/CIoiRwjRxde/hAHyLPXYN4V3CqKAbiZPSeJFSWHmJsbkicta0Eg==} + node-sql-parser@5.4.0: + resolution: {integrity: sha512-jVe6Z61gPcPjCElPZ6j8llB3wnqGcuQzefim1ERsqIakxnEy5JlzV7XKdO1KmacRG5TKwPc4vJTgSRQ0LfkbFw==} + engines: {node: '>=8'} + nodemailer@8.0.5: resolution: {integrity: sha512-0PF8Yb1yZuQfQbq+5/pZJrtF6WQcjTd5/S4JOHs9PGFxuTqoB/icwuB44pOdURHJbRKX1PPoJZtY7R4VUoCC8w==} engines: {node: '>=6.0.0'} @@ -8768,6 +8778,8 @@ snapshots: dependencies: '@types/node': 24.12.2 + '@types/pegjs@0.10.6': {} + '@types/pg-copy-streams@1.2.5': dependencies: '@types/node': 24.12.2 @@ -9185,8 +9197,7 @@ snapshots: node-addon-api: 8.7.0 node-gyp-build: 4.8.4 - big-integer@1.6.52: - optional: true + big-integer@1.6.52: {} bindings@1.5.0: dependencies: @@ -10843,6 +10854,11 @@ snapshots: node-releases@2.0.37: {} + node-sql-parser@5.4.0: + dependencies: + '@types/pegjs': 0.10.6 + big-integer: 1.6.52 + nodemailer@8.0.5: {} nofilter@3.1.0: {} diff --git a/rocketadmin-agent/.clickhouse_test_agent_config.txt b/rocketadmin-agent/.clickhouse_test_agent_config.txt index 5a5e00cf5..70f51d4bd 100644 --- a/rocketadmin-agent/.clickhouse_test_agent_config.txt +++ b/rocketadmin-agent/.clickhouse_test_agent_config.txt @@ -6,7 +6,7 @@ "azure_encryption": false, "cert": null, "database": "testdb", - "host": "clickhouse-e2e-testing", + "host": "test-clickhouse-e2e-testing", "password": "clickhouse_password", "port": 8123, "token": "CLICKHOUSE-TEST-AGENT-TOKEN",