From b38813a9437fefefb535371abb5c2b5bc2273f44 Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Wed, 28 Jan 2026 14:42:18 +0000 Subject: [PATCH 1/2] Add timestamp columns to settings and widget entities --- .../table-settings.entity.ts | 19 ++- .../personal-table-settings.entity.ts | 17 ++- .../entities/widget/table-widget.entity.ts | 130 ++++++++++-------- ...mpColumnsIntoSettingsAndWidgetsEntities.ts | 23 ++++ 4 files changed, 128 insertions(+), 61 deletions(-) create mode 100644 backend/src/migrations/1769610545842-AddTimeStampColumnsIntoSettingsAndWidgetsEntities.ts diff --git a/backend/src/entities/table-settings/common-table-settings/table-settings.entity.ts b/backend/src/entities/table-settings/common-table-settings/table-settings.entity.ts index 660631fb1..c4bc4f39f 100644 --- a/backend/src/entities/table-settings/common-table-settings/table-settings.entity.ts +++ b/backend/src/entities/table-settings/common-table-settings/table-settings.entity.ts @@ -1,5 +1,16 @@ import { Transform } from 'class-transformer'; -import { Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryGeneratedColumn, Relation, Unique } from 'typeorm'; +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + ManyToOne, + OneToMany, + PrimaryGeneratedColumn, + Relation, + Unique, + UpdateDateColumn, +} from 'typeorm'; import { ConnectionEntity } from '../../connection/connection.entity.js'; import { CustomFieldsEntity } from '../../custom-field/custom-fields.entity.js'; import { TableActionEntity } from '../../table-actions/table-actions-module/table-action.entity.js'; @@ -79,6 +90,12 @@ export class TableSettingsEntity { @Column('varchar', { default: null }) icon: string; + @CreateDateColumn({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' }) + created_at: Date; + + @UpdateDateColumn({ type: 'timestamp', nullable: true, default: null }) + updated_at: Date; + @Transform(({ value: connection }) => connection.id) @ManyToOne( (_) => ConnectionEntity, diff --git a/backend/src/entities/table-settings/personal-table-settings/personal-table-settings.entity.ts b/backend/src/entities/table-settings/personal-table-settings/personal-table-settings.entity.ts index 6794cd72f..e32fa00f8 100644 --- a/backend/src/entities/table-settings/personal-table-settings/personal-table-settings.entity.ts +++ b/backend/src/entities/table-settings/personal-table-settings/personal-table-settings.entity.ts @@ -1,4 +1,13 @@ -import { Column, Entity, JoinColumn, ManyToOne, PrimaryGeneratedColumn, Relation } from 'typeorm'; +import { + Column, + CreateDateColumn, + Entity, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, + Relation, + UpdateDateColumn, +} from 'typeorm'; import { QueryOrderingEnum } from '../../../enums/query-ordering.enum.js'; import { ConnectionEntity } from '../../connection/connection.entity.js'; import { UserEntity } from '../../user/user.entity.js'; @@ -33,6 +42,12 @@ export class PersonalTableSettingsEntity { @Column('boolean', { default: false }) original_names: boolean; + @CreateDateColumn({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' }) + created_at: Date; + + @UpdateDateColumn({ type: 'timestamp', nullable: true, default: null }) + updated_at: Date; + @ManyToOne( (_) => ConnectionEntity, (connection) => connection.personal_table_settings, diff --git a/backend/src/entities/widget/table-widget.entity.ts b/backend/src/entities/widget/table-widget.entity.ts index 4a71b0a7a..bce6ed1f1 100644 --- a/backend/src/entities/widget/table-widget.entity.ts +++ b/backend/src/entities/widget/table-widget.entity.ts @@ -1,78 +1,90 @@ import sjson from 'secure-json-parse'; import { - AfterLoad, - BeforeInsert, - BeforeUpdate, - Column, - Entity, - JoinColumn, - ManyToOne, - PrimaryGeneratedColumn, - Relation, + AfterLoad, + BeforeInsert, + BeforeUpdate, + Column, + CreateDateColumn, + Entity, + JoinColumn, + ManyToOne, + PrimaryGeneratedColumn, + Relation, + UpdateDateColumn, } from 'typeorm'; import { WidgetTypeEnum } from '../../enums/index.js'; import { TableSettingsEntity } from '../table-settings/common-table-settings/table-settings.entity.js'; @Entity('table_widget') export class TableWidgetEntity { - @PrimaryGeneratedColumn('uuid') - id: string; + @PrimaryGeneratedColumn('uuid') + id: string; - @Column() - field_name: string; + @Column() + field_name: string; - @Column({ default: null, type: 'varchar' }) - widget_type?: WidgetTypeEnum; + @Column({ default: null, type: 'varchar' }) + widget_type?: WidgetTypeEnum; - @Column('json', { default: null }) - widget_params: string; + @Column('json', { default: null }) + widget_params: string; - @Column('json', { default: null }) - widget_options: string; + @Column('json', { default: null }) + widget_options: string; - @Column({ default: null }) - name?: string; + @Column({ default: null }) + name?: string; - @Column({ default: null }) - description?: string; + @Column({ default: null }) + description?: string; - @BeforeUpdate() - stringifyOptionsOnUpdate() { - try { - if (this.widget_options) { - this.widget_options = JSON.stringify(this.widget_options); - } - } catch (e) { - console.error('-> Error widget options stringify ' + e.message); - } - } + @CreateDateColumn({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' }) + created_at: Date; - @BeforeInsert() - stringifyOptions() { - try { - if (this.widget_options) { - this.widget_options = JSON.stringify(this.widget_options); - } - } catch (e) { - console.error('-> Error widget options stringify ' + e.message); - } - } + @UpdateDateColumn({ type: 'timestamp', nullable: true, default: null }) + updated_at: Date; - @AfterLoad() - parseOptions() { - try { - if (this.widget_options) { - this.widget_options = sjson.parse(this.widget_options, null, { - protoAction: 'remove', - constructorAction: 'remove', - }); - } - } catch (e) { - console.error('-> Error widget options parse ' + e.message); - } - } + @BeforeUpdate() + stringifyOptionsOnUpdate() { + try { + if (this.widget_options) { + this.widget_options = JSON.stringify(this.widget_options); + } + } catch (e) { + console.error('-> Error widget options stringify ' + e.message); + } + } - @ManyToOne(() => TableSettingsEntity, (settings) => settings.table_widgets, { onDelete: 'CASCADE' }) - @JoinColumn() - settings: Relation; + @BeforeInsert() + stringifyOptions() { + try { + if (this.widget_options) { + this.widget_options = JSON.stringify(this.widget_options); + } + } catch (e) { + console.error('-> Error widget options stringify ' + e.message); + } + } + + @AfterLoad() + parseOptions() { + try { + if (this.widget_options) { + this.widget_options = sjson.parse(this.widget_options, null, { + protoAction: 'remove', + constructorAction: 'remove', + }); + } + } catch (e) { + console.error('-> Error widget options parse ' + e.message); + } + } + + @ManyToOne( + () => TableSettingsEntity, + (settings) => settings.table_widgets, + { onDelete: 'CASCADE' }, + ) + @JoinColumn() + settings: Relation; } diff --git a/backend/src/migrations/1769610545842-AddTimeStampColumnsIntoSettingsAndWidgetsEntities.ts b/backend/src/migrations/1769610545842-AddTimeStampColumnsIntoSettingsAndWidgetsEntities.ts new file mode 100644 index 000000000..ceaf1dc5a --- /dev/null +++ b/backend/src/migrations/1769610545842-AddTimeStampColumnsIntoSettingsAndWidgetsEntities.ts @@ -0,0 +1,23 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddTimeStampColumnsIntoSettingsAndWidgetsEntities1769610545842 implements MigrationInterface { + name = 'AddTimeStampColumnsIntoSettingsAndWidgetsEntities1769610545842'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "personal_table_settings" ADD "created_at" TIMESTAMP NOT NULL DEFAULT now()`); + await queryRunner.query(`ALTER TABLE "personal_table_settings" ADD "updated_at" TIMESTAMP DEFAULT now()`); + await queryRunner.query(`ALTER TABLE "tableSettings" ADD "created_at" TIMESTAMP NOT NULL DEFAULT now()`); + await queryRunner.query(`ALTER TABLE "tableSettings" ADD "updated_at" TIMESTAMP DEFAULT now()`); + await queryRunner.query(`ALTER TABLE "table_widget" ADD "created_at" TIMESTAMP NOT NULL DEFAULT now()`); + await queryRunner.query(`ALTER TABLE "table_widget" ADD "updated_at" TIMESTAMP DEFAULT now()`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "table_widget" DROP COLUMN "updated_at"`); + await queryRunner.query(`ALTER TABLE "table_widget" DROP COLUMN "created_at"`); + await queryRunner.query(`ALTER TABLE "tableSettings" DROP COLUMN "updated_at"`); + await queryRunner.query(`ALTER TABLE "tableSettings" DROP COLUMN "created_at"`); + await queryRunner.query(`ALTER TABLE "personal_table_settings" DROP COLUMN "updated_at"`); + await queryRunner.query(`ALTER TABLE "personal_table_settings" DROP COLUMN "created_at"`); + } +} From 973bcd651d6ec11fb89a9b7ed788eca909229daa Mon Sep 17 00:00:00 2001 From: Artem Niehrieiev Date: Wed, 28 Jan 2026 15:42:49 +0000 Subject: [PATCH 2/2] Implement batch processing for AI-generated table settings and enhance error handling --- backend/src/entities/ai/ai.service.ts | 37 +++++++++++++++-- .../shared-jobs/shared-jobs.service.ts | 40 ++++++++++++++----- 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/backend/src/entities/ai/ai.service.ts b/backend/src/entities/ai/ai.service.ts index 922530e21..0ca2286b3 100644 --- a/backend/src/entities/ai/ai.service.ts +++ b/backend/src/entities/ai/ai.service.ts @@ -27,6 +27,8 @@ interface AIResponse { tables: AIGeneratedTableSettings[]; } +const AI_BATCH_SIZE = 10; + @Injectable() export class AiService { constructor(protected readonly aiCoreService: AICoreService) {} @@ -34,11 +36,35 @@ export class AiService { public async generateNewTableSettingsWithAI( tablesInformation: Array, ): Promise> { + const allSettings: Array = []; + + for (let i = 0; i < tablesInformation.length; i += AI_BATCH_SIZE) { + const batch = tablesInformation.slice(i, i + AI_BATCH_SIZE); + try { + const batchSettings = await this.processTablesBatch(batch); + allSettings.push(...batchSettings); + } catch (error) { + console.warn(`Batch processing failed, falling back to individual table processing: ${error.message}`); + for (const tableInfo of batch) { + try { + const singleTableSettings = await this.processTablesBatch([tableInfo]); + allSettings.push(...singleTableSettings); + } catch (singleError) { + console.error(`Error processing AI for table "${tableInfo.table_name}": ${singleError.message}`); + } + } + } + } + + return allSettings; + } + + private async processTablesBatch(tablesInformation: Array): Promise> { const prompt = this.buildPrompt(tablesInformation); const aiResponse = await this.aiCoreService.completeWithProvider(AIProviderType.BEDROCK, prompt, { temperature: 0.3, }); - const parsedResponse = this.parseAIResponse(aiResponse); + const parsedResponse = this.parseAIResponse(aiResponse, tablesInformation); return this.buildTableSettingsEntities(parsedResponse, tablesInformation); } @@ -131,13 +157,14 @@ Respond ONLY with valid JSON in this exact format (no markdown, no explanations) }`; } - private parseAIResponse(aiResponse: string): AIResponse { + private parseAIResponse(aiResponse: string, tablesInformation: Array): AIResponse { const cleanedResponse = cleanAIJsonResponse(aiResponse); + const tableNames = tablesInformation.map((t) => t.table_name); try { return JSON.parse(cleanedResponse) as AIResponse; } catch (error) { - throw new Error(`Failed to parse AI response: ${error.message}`); + throw new Error(`Failed to parse AI response for tables [${tableNames.join(', ')}]: ${error.message}`); } } @@ -156,7 +183,9 @@ Respond ONLY with valid JSON in this exact format (no markdown, no explanations) settings.readonly_fields = this.filterValidColumns(tableSettings.readonly_fields, validColumnNames); settings.columns_view = this.filterValidColumns(tableSettings.columns_view, validColumnNames); settings.ordering = this.mapOrdering(tableSettings.ordering); - settings.ordering_field = validColumnNames.includes(tableSettings.ordering_field) ? tableSettings.ordering_field : null; + settings.ordering_field = validColumnNames.includes(tableSettings.ordering_field) + ? tableSettings.ordering_field + : null; settings.table_widgets = tableSettings.widgets .filter((w) => validColumnNames.includes(w.field_name)) .map((widgetData) => { diff --git a/backend/src/entities/shared-jobs/shared-jobs.service.ts b/backend/src/entities/shared-jobs/shared-jobs.service.ts index 04983fe2a..4f9ab5453 100644 --- a/backend/src/entities/shared-jobs/shared-jobs.service.ts +++ b/backend/src/entities/shared-jobs/shared-jobs.service.ts @@ -51,24 +51,44 @@ export class SharedJobsService { } const queue = new PQueue({ concurrency: 4 }); - const tablesInformation = await Promise.all( + const tablesInformationResults = await Promise.all( tablesToScan.map((table) => queue.add(async () => { - const structure = await dao.getTableStructure(table.tableName, null); - const primaryColumns = await dao.getTablePrimaryColumns(table.tableName, null); - const foreignKeys = await dao.getTableForeignKeys(table.tableName, null); - return { - table_name: table.tableName, - structure, - primaryColumns, - foreignKeys, - }; + try { + const structure = await dao.getTableStructure(table.tableName, null); + const primaryColumns = await dao.getTablePrimaryColumns(table.tableName, null); + const foreignKeys = await dao.getTableForeignKeys(table.tableName, null); + return { + table_name: table.tableName, + structure, + primaryColumns, + foreignKeys, + }; + } catch (error) { + console.error(`Error getting table information for "${table.tableName}": ${error.message}`); + return null; + } }), ), ); + const tablesInformation = tablesInformationResults.filter((info) => info !== null); + + if (tablesInformation.length === 0) { + console.info(`No valid tables to process for connection with id "${connection.id}"`); + return; + } + + console.info(`Processing ${tablesInformation.length} tables with AI for connection "${connection.id}"`); const generatedTableSettings = await this.aiService.generateNewTableSettingsWithAI(tablesInformation); + if (generatedTableSettings.length === 0) { + console.info(`No table settings generated by AI for connection with id "${connection.id}"`); + return; + } + + console.info(`AI generated settings for ${generatedTableSettings.length} tables`); + const widgetsByTable = new Map>(); for (const setting of generatedTableSettings) { if (setting.table_widgets && setting.table_widgets.length > 0) {