Skip to content

Commit 6345c2e

Browse files
ArtuomkaArtuomka
authored
Merge pull request #1540 from rocket-admin/backend_ai_table_settings_fix
Backend ai table settings fix
2 parents 8559b5e + 973bcd6 commit 6345c2e

6 files changed

Lines changed: 191 additions & 75 deletions

File tree

‎backend/src/entities/ai/ai.service.ts‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,18 +27,44 @@ interface AIResponse {
2727
tables: AIGeneratedTableSettings[];
2828
}
2929

30+
const AI_BATCH_SIZE = 10;
31+
3032
@Injectable()
3133
export class AiService {
3234
constructor(protected readonly aiCoreService: AICoreService) {}
3335

3436
public async generateNewTableSettingsWithAI(
3537
tablesInformation: Array<TableInformation>,
3638
): Promise<Array<TableSettingsEntity>> {
39+
const allSettings: Array<TableSettingsEntity> = [];
40+
41+
for (let i = 0; i < tablesInformation.length; i += AI_BATCH_SIZE) {
42+
const batch = tablesInformation.slice(i, i + AI_BATCH_SIZE);
43+
try {
44+
const batchSettings = await this.processTablesBatch(batch);
45+
allSettings.push(...batchSettings);
46+
} catch (error) {
47+
console.warn(`Batch processing failed, falling back to individual table processing: ${error.message}`);
48+
for (const tableInfo of batch) {
49+
try {
50+
const singleTableSettings = await this.processTablesBatch([tableInfo]);
51+
allSettings.push(...singleTableSettings);
52+
} catch (singleError) {
53+
console.error(`Error processing AI for table "${tableInfo.table_name}": ${singleError.message}`);
54+
}
55+
}
56+
}
57+
}
58+
59+
return allSettings;
60+
}
61+
62+
private async processTablesBatch(tablesInformation: Array<TableInformation>): Promise<Array<TableSettingsEntity>> {
3763
const prompt = this.buildPrompt(tablesInformation);
3864
const aiResponse = await this.aiCoreService.completeWithProvider(AIProviderType.BEDROCK, prompt, {
3965
temperature: 0.3,
4066
});
41-
const parsedResponse = this.parseAIResponse(aiResponse);
67+
const parsedResponse = this.parseAIResponse(aiResponse, tablesInformation);
4268
return this.buildTableSettingsEntities(parsedResponse, tablesInformation);
4369
}
4470

@@ -131,13 +157,14 @@ Respond ONLY with valid JSON in this exact format (no markdown, no explanations)
131157
}`;
132158
}
133159

134-
private parseAIResponse(aiResponse: string): AIResponse {
160+
private parseAIResponse(aiResponse: string, tablesInformation: Array<TableInformation>): AIResponse {
135161
const cleanedResponse = cleanAIJsonResponse(aiResponse);
162+
const tableNames = tablesInformation.map((t) => t.table_name);
136163

137164
try {
138165
return JSON.parse(cleanedResponse) as AIResponse;
139166
} catch (error) {
140-
throw new Error(`Failed to parse AI response: ${error.message}`);
167+
throw new Error(`Failed to parse AI response for tables [${tableNames.join(', ')}]: ${error.message}`);
141168
}
142169
}
143170

@@ -156,7 +183,9 @@ Respond ONLY with valid JSON in this exact format (no markdown, no explanations)
156183
settings.readonly_fields = this.filterValidColumns(tableSettings.readonly_fields, validColumnNames);
157184
settings.columns_view = this.filterValidColumns(tableSettings.columns_view, validColumnNames);
158185
settings.ordering = this.mapOrdering(tableSettings.ordering);
159-
settings.ordering_field = validColumnNames.includes(tableSettings.ordering_field) ? tableSettings.ordering_field : null;
186+
settings.ordering_field = validColumnNames.includes(tableSettings.ordering_field)
187+
? tableSettings.ordering_field
188+
: null;
160189
settings.table_widgets = tableSettings.widgets
161190
.filter((w) => validColumnNames.includes(w.field_name))
162191
.map((widgetData) => {

‎backend/src/entities/shared-jobs/shared-jobs.service.ts‎

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -51,24 +51,44 @@ export class SharedJobsService {
5151
}
5252

5353
const queue = new PQueue({ concurrency: 4 });
54-
const tablesInformation = await Promise.all(
54+
const tablesInformationResults = await Promise.all(
5555
tablesToScan.map((table) =>
5656
queue.add(async () => {
57-
const structure = await dao.getTableStructure(table.tableName, null);
58-
const primaryColumns = await dao.getTablePrimaryColumns(table.tableName, null);
59-
const foreignKeys = await dao.getTableForeignKeys(table.tableName, null);
60-
return {
61-
table_name: table.tableName,
62-
structure,
63-
primaryColumns,
64-
foreignKeys,
65-
};
57+
try {
58+
const structure = await dao.getTableStructure(table.tableName, null);
59+
const primaryColumns = await dao.getTablePrimaryColumns(table.tableName, null);
60+
const foreignKeys = await dao.getTableForeignKeys(table.tableName, null);
61+
return {
62+
table_name: table.tableName,
63+
structure,
64+
primaryColumns,
65+
foreignKeys,
66+
};
67+
} catch (error) {
68+
console.error(`Error getting table information for "${table.tableName}": ${error.message}`);
69+
return null;
70+
}
6671
}),
6772
),
6873
);
6974

75+
const tablesInformation = tablesInformationResults.filter((info) => info !== null);
76+
77+
if (tablesInformation.length === 0) {
78+
console.info(`No valid tables to process for connection with id "${connection.id}"`);
79+
return;
80+
}
81+
82+
console.info(`Processing ${tablesInformation.length} tables with AI for connection "${connection.id}"`);
7083
const generatedTableSettings = await this.aiService.generateNewTableSettingsWithAI(tablesInformation);
7184

85+
if (generatedTableSettings.length === 0) {
86+
console.info(`No table settings generated by AI for connection with id "${connection.id}"`);
87+
return;
88+
}
89+
90+
console.info(`AI generated settings for ${generatedTableSettings.length} tables`);
91+
7292
const widgetsByTable = new Map<string, Array<TableWidgetEntity>>();
7393
for (const setting of generatedTableSettings) {
7494
if (setting.table_widgets && setting.table_widgets.length > 0) {

‎backend/src/entities/table-settings/common-table-settings/table-settings.entity.ts‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,16 @@
11
import { Transform } from 'class-transformer';
2-
import { Column, Entity, JoinColumn, ManyToOne, OneToMany, PrimaryGeneratedColumn, Relation, Unique } from 'typeorm';
2+
import {
3+
Column,
4+
CreateDateColumn,
5+
Entity,
6+
JoinColumn,
7+
ManyToOne,
8+
OneToMany,
9+
PrimaryGeneratedColumn,
10+
Relation,
11+
Unique,
12+
UpdateDateColumn,
13+
} from 'typeorm';
314
import { ConnectionEntity } from '../../connection/connection.entity.js';
415
import { CustomFieldsEntity } from '../../custom-field/custom-fields.entity.js';
516
import { TableActionEntity } from '../../table-actions/table-actions-module/table-action.entity.js';
@@ -79,6 +90,12 @@ export class TableSettingsEntity {
7990
@Column('varchar', { default: null })
8091
icon: string;
8192

93+
@CreateDateColumn({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' })
94+
created_at: Date;
95+
96+
@UpdateDateColumn({ type: 'timestamp', nullable: true, default: null })
97+
updated_at: Date;
98+
8299
@Transform(({ value: connection }) => connection.id)
83100
@ManyToOne(
84101
(_) => ConnectionEntity,

‎backend/src/entities/table-settings/personal-table-settings/personal-table-settings.entity.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,13 @@
1-
import { Column, Entity, JoinColumn, ManyToOne, PrimaryGeneratedColumn, Relation } from 'typeorm';
1+
import {
2+
Column,
3+
CreateDateColumn,
4+
Entity,
5+
JoinColumn,
6+
ManyToOne,
7+
PrimaryGeneratedColumn,
8+
Relation,
9+
UpdateDateColumn,
10+
} from 'typeorm';
211
import { QueryOrderingEnum } from '../../../enums/query-ordering.enum.js';
312
import { ConnectionEntity } from '../../connection/connection.entity.js';
413
import { UserEntity } from '../../user/user.entity.js';
@@ -33,6 +42,12 @@ export class PersonalTableSettingsEntity {
3342
@Column('boolean', { default: false })
3443
original_names: boolean;
3544

45+
@CreateDateColumn({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' })
46+
created_at: Date;
47+
48+
@UpdateDateColumn({ type: 'timestamp', nullable: true, default: null })
49+
updated_at: Date;
50+
3651
@ManyToOne(
3752
(_) => ConnectionEntity,
3853
(connection) => connection.personal_table_settings,

‎backend/src/entities/widget/table-widget.entity.ts‎

Lines changed: 71 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -1,78 +1,90 @@
11
import sjson from 'secure-json-parse';
22
import {
3-
AfterLoad,
4-
BeforeInsert,
5-
BeforeUpdate,
6-
Column,
7-
Entity,
8-
JoinColumn,
9-
ManyToOne,
10-
PrimaryGeneratedColumn,
11-
Relation,
3+
AfterLoad,
4+
BeforeInsert,
5+
BeforeUpdate,
6+
Column,
7+
CreateDateColumn,
8+
Entity,
9+
JoinColumn,
10+
ManyToOne,
11+
PrimaryGeneratedColumn,
12+
Relation,
13+
UpdateDateColumn,
1214
} from 'typeorm';
1315
import { WidgetTypeEnum } from '../../enums/index.js';
1416
import { TableSettingsEntity } from '../table-settings/common-table-settings/table-settings.entity.js';
1517

1618
@Entity('table_widget')
1719
export class TableWidgetEntity {
18-
@PrimaryGeneratedColumn('uuid')
19-
id: string;
20+
@PrimaryGeneratedColumn('uuid')
21+
id: string;
2022

21-
@Column()
22-
field_name: string;
23+
@Column()
24+
field_name: string;
2325

24-
@Column({ default: null, type: 'varchar' })
25-
widget_type?: WidgetTypeEnum;
26+
@Column({ default: null, type: 'varchar' })
27+
widget_type?: WidgetTypeEnum;
2628

27-
@Column('json', { default: null })
28-
widget_params: string;
29+
@Column('json', { default: null })
30+
widget_params: string;
2931

30-
@Column('json', { default: null })
31-
widget_options: string;
32+
@Column('json', { default: null })
33+
widget_options: string;
3234

33-
@Column({ default: null })
34-
name?: string;
35+
@Column({ default: null })
36+
name?: string;
3537

36-
@Column({ default: null })
37-
description?: string;
38+
@Column({ default: null })
39+
description?: string;
3840

39-
@BeforeUpdate()
40-
stringifyOptionsOnUpdate() {
41-
try {
42-
if (this.widget_options) {
43-
this.widget_options = JSON.stringify(this.widget_options);
44-
}
45-
} catch (e) {
46-
console.error('-> Error widget options stringify ' + e.message);
47-
}
48-
}
41+
@CreateDateColumn({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' })
42+
created_at: Date;
4943

50-
@BeforeInsert()
51-
stringifyOptions() {
52-
try {
53-
if (this.widget_options) {
54-
this.widget_options = JSON.stringify(this.widget_options);
55-
}
56-
} catch (e) {
57-
console.error('-> Error widget options stringify ' + e.message);
58-
}
59-
}
44+
@UpdateDateColumn({ type: 'timestamp', nullable: true, default: null })
45+
updated_at: Date;
6046

61-
@AfterLoad()
62-
parseOptions() {
63-
try {
64-
if (this.widget_options) {
65-
this.widget_options = sjson.parse(this.widget_options, null, {
66-
protoAction: 'remove',
67-
constructorAction: 'remove',
68-
});
69-
}
70-
} catch (e) {
71-
console.error('-> Error widget options parse ' + e.message);
72-
}
73-
}
47+
@BeforeUpdate()
48+
stringifyOptionsOnUpdate() {
49+
try {
50+
if (this.widget_options) {
51+
this.widget_options = JSON.stringify(this.widget_options);
52+
}
53+
} catch (e) {
54+
console.error('-> Error widget options stringify ' + e.message);
55+
}
56+
}
7457

75-
@ManyToOne(() => TableSettingsEntity, (settings) => settings.table_widgets, { onDelete: 'CASCADE' })
76-
@JoinColumn()
77-
settings: Relation<TableSettingsEntity>;
58+
@BeforeInsert()
59+
stringifyOptions() {
60+
try {
61+
if (this.widget_options) {
62+
this.widget_options = JSON.stringify(this.widget_options);
63+
}
64+
} catch (e) {
65+
console.error('-> Error widget options stringify ' + e.message);
66+
}
67+
}
68+
69+
@AfterLoad()
70+
parseOptions() {
71+
try {
72+
if (this.widget_options) {
73+
this.widget_options = sjson.parse(this.widget_options, null, {
74+
protoAction: 'remove',
75+
constructorAction: 'remove',
76+
});
77+
}
78+
} catch (e) {
79+
console.error('-> Error widget options parse ' + e.message);
80+
}
81+
}
82+
83+
@ManyToOne(
84+
() => TableSettingsEntity,
85+
(settings) => settings.table_widgets,
86+
{ onDelete: 'CASCADE' },
87+
)
88+
@JoinColumn()
89+
settings: Relation<TableSettingsEntity>;
7890
}

‎backend/src/migrations/1769610545842-AddTimeStampColumnsIntoSettingsAndWidgetsEntities.ts‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import { MigrationInterface, QueryRunner } from 'typeorm';
2+
3+
export class AddTimeStampColumnsIntoSettingsAndWidgetsEntities1769610545842 implements MigrationInterface {
4+
name = 'AddTimeStampColumnsIntoSettingsAndWidgetsEntities1769610545842';
5+
6+
public async up(queryRunner: QueryRunner): Promise<void> {
7+
await queryRunner.query(`ALTER TABLE "personal_table_settings" ADD "created_at" TIMESTAMP NOT NULL DEFAULT now()`);
8+
await queryRunner.query(`ALTER TABLE "personal_table_settings" ADD "updated_at" TIMESTAMP DEFAULT now()`);
9+
await queryRunner.query(`ALTER TABLE "tableSettings" ADD "created_at" TIMESTAMP NOT NULL DEFAULT now()`);
10+
await queryRunner.query(`ALTER TABLE "tableSettings" ADD "updated_at" TIMESTAMP DEFAULT now()`);
11+
await queryRunner.query(`ALTER TABLE "table_widget" ADD "created_at" TIMESTAMP NOT NULL DEFAULT now()`);
12+
await queryRunner.query(`ALTER TABLE "table_widget" ADD "updated_at" TIMESTAMP DEFAULT now()`);
13+
}
14+
15+
public async down(queryRunner: QueryRunner): Promise<void> {
16+
await queryRunner.query(`ALTER TABLE "table_widget" DROP COLUMN "updated_at"`);
17+
await queryRunner.query(`ALTER TABLE "table_widget" DROP COLUMN "created_at"`);
18+
await queryRunner.query(`ALTER TABLE "tableSettings" DROP COLUMN "updated_at"`);
19+
await queryRunner.query(`ALTER TABLE "tableSettings" DROP COLUMN "created_at"`);
20+
await queryRunner.query(`ALTER TABLE "personal_table_settings" DROP COLUMN "updated_at"`);
21+
await queryRunner.query(`ALTER TABLE "personal_table_settings" DROP COLUMN "created_at"`);
22+
}
23+
}

0 commit comments

Comments
 (0)