[sync] fix: flatten multi-select array_unique rollups (#1504) (#2813)

tea-artist · nichenqin · web-flow · commit cf3540836c98 · 2026-03-24T11:35:03.000+08:00
Synced from teableio/teable-ee@5105c9b Co-authored-by: nichenqin <nichenqin@hotmail.com>
diff --git a/apps/nestjs-backend/src/features/record/query-builder/field-cte-visitor.ts b/apps/nestjs-backend/src/features/record/query-builder/field-cte-visitor.ts
@@ -294,10 +294,14 @@ class FieldCteSelectionVisitor implements IFieldVisitor<IFieldSelectName> {
     rowPresenceExpr?: string
   ): string {
     const functionName = parseRollupFunctionName(expression);
+    const shouldFlattenNestedArray =
+      functionName === 'array_unique' &&
+      ((targetField?.isMultipleCellValue ?? false) || (targetField?.isConditionalLookup ?? false));
     return this.dialect.rollupAggregate(functionName, fieldExpression, {
       targetField,
       orderByField,
       rowPresenceExpr,
+      flattenNestedArray: shouldFlattenNestedArray,
     });
   }
 
@@ -1259,7 +1263,7 @@ export class FieldCteVisitor implements IFieldVisitor<ICteResult> {
   ): string {
     const fn = parseRollupFunctionName(rollupExpression);
     const shouldFlattenNestedArray =
-      fn === 'array_compact' &&
+      (fn === 'array_compact' || fn === 'array_unique') &&
       ((targetField?.isMultipleCellValue ?? false) || (targetField?.isConditionalLookup ?? false));
     return this.dialect.rollupAggregate(fn, fieldExpression, {
       targetField,
diff --git a/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.ts b/apps/nestjs-backend/src/features/record/query-builder/providers/pg-record-query-dialect.ts
@@ -22,6 +22,34 @@ export class PgRecordQueryDialect implements IRecordQueryDialectProvider {
 
   constructor(private readonly knex: Knex) {}
 
+  private buildDistinctFlattenedJsonArray(baseAggregate: string): string {
+    return `(SELECT jsonb_agg(to_jsonb(v.val))
+      FROM (
+        SELECT DISTINCT val
+        FROM (
+          SELECT leaf #>> '{}' AS val
+          FROM jsonb_array_elements(COALESCE(${baseAggregate}, '[]'::jsonb)) AS row_elem(elem)
+          CROSS JOIN LATERAL jsonb_array_elements(
+            CASE
+              WHEN jsonb_typeof(row_elem.elem) = 'array' THEN row_elem.elem
+              ELSE jsonb_build_array(row_elem.elem)
+            END
+          ) AS leaf_elem(leaf)
+        ) AS flattened
+        WHERE val IS NOT NULL AND val <> ''
+        ORDER BY val
+      ) AS v)`;
+  }
+
+  private normalizeSingleValueJsonArray(expr: string): string {
+    return `(CASE
+      WHEN ${expr} IS NULL THEN '[]'::jsonb
+      WHEN jsonb_typeof(to_jsonb(${expr})) = 'array' THEN to_jsonb(${expr})
+      WHEN jsonb_typeof(to_jsonb(${expr})) = 'null' THEN '[]'::jsonb
+      ELSE jsonb_build_array(to_jsonb(${expr}))
+    END)`;
+  }
+
   toText(expr: string): string {
     return `(${expr})::TEXT`;
   }
@@ -504,6 +532,12 @@ export class PgRecordQueryDialect implements IRecordQueryDialectProvider {
           ? `STRING_AGG(${fieldExpression}::text, ', ' ORDER BY ${orderByField})`
           : `STRING_AGG(${fieldExpression}::text, ', ')`;
       case 'array_unique':
+        if (flattenNestedArray) {
+          const baseAggregate = orderByField
+            ? `jsonb_agg(to_jsonb(${fieldExpression}) ORDER BY ${orderByField}) FILTER (WHERE ${fieldExpression} IS NOT NULL)`
+            : `jsonb_agg(to_jsonb(${fieldExpression})) FILTER (WHERE ${fieldExpression} IS NOT NULL)`;
+          return this.buildDistinctFlattenedJsonArray(baseAggregate);
+        }
         return `json_agg(DISTINCT ${fieldExpression})`;
       case 'array_compact': {
         const buildAggregate = (expr: string) =>
@@ -572,6 +606,15 @@ export class PgRecordQueryDialect implements IRecordQueryDialectProvider {
       case 'xor':
         return `(COALESCE((${fieldExpression})::boolean, false))`;
       case 'array_unique':
+        if (
+          requiresJsonArray &&
+          (options.targetField.isMultipleCellValue || options.targetField.isConditionalLookup)
+        ) {
+          return this.normalizeSingleValueJsonArray(fieldExpression);
+        }
+        return !requiresJsonArray
+          ? `${fieldExpression}`
+          : `(CASE WHEN ${fieldExpression} IS NULL THEN '[]'::jsonb ELSE jsonb_build_array(${fieldExpression}) END)`;
       case 'array_compact':
         if (!requiresJsonArray) {
           return `${fieldExpression}`;
diff --git a/apps/nestjs-backend/src/features/record/query-builder/providers/sqlite-record-query-dialect.ts b/apps/nestjs-backend/src/features/record/query-builder/providers/sqlite-record-query-dialect.ts
@@ -183,6 +183,26 @@ export class SqliteRecordQueryDialect implements IRecordQueryDialectProvider {
       case 'concatenate':
         return `GROUP_CONCAT(${fieldExpression}, ', ')`;
       case 'array_unique':
+        if (opts.flattenNestedArray) {
+          return `(WITH outer_values AS (
+              SELECT value
+              FROM json_each(COALESCE(json_group_array(${fieldExpression}), json('[]')))
+            ),
+            flattened AS (
+              SELECT inner_elem.value AS value
+              FROM outer_values
+              JOIN json_each(
+                CASE
+                  WHEN json_valid(outer_values.value) AND json_type(outer_values.value) = 'array'
+                    THEN outer_values.value
+                  ELSE json_array(outer_values.value)
+                END
+              ) AS inner_elem
+            )
+            SELECT json_group_array(DISTINCT value)
+            FROM flattened
+            WHERE value IS NOT NULL AND CAST(value AS TEXT) <> '')`;
+        }
         return `json_group_array(DISTINCT ${fieldExpression})`;
       case 'array_compact':
         return `json_group_array(CASE WHEN ${fieldExpression} IS NOT NULL AND CAST(${fieldExpression} AS TEXT) <> '' THEN ${fieldExpression} END)`;
@@ -215,6 +235,19 @@ export class SqliteRecordQueryDialect implements IRecordQueryDialectProvider {
       case 'xor':
         return `(CASE WHEN ${fieldExpression} THEN 1 ELSE 0 END)`;
       case 'array_unique':
+        if (
+          requiresJsonArray &&
+          (options.targetField.isMultipleCellValue || options.targetField.isConditionalLookup)
+        ) {
+          return `(CASE
+            WHEN ${fieldExpression} IS NULL THEN json('[]')
+            WHEN json_valid(${fieldExpression}) AND json_type(${fieldExpression}) = 'array' THEN ${fieldExpression}
+            ELSE json_array(${fieldExpression})
+          END)`;
+        }
+        return !requiresJsonArray
+          ? `${fieldExpression}`
+          : `(CASE WHEN ${fieldExpression} IS NULL THEN json('[]') ELSE json_array(${fieldExpression}) END)`;
       case 'array_compact':
         if (!requiresJsonArray) {
           return `${fieldExpression}`;
diff --git a/apps/nestjs-backend/test/rollup.e2e-spec.ts b/apps/nestjs-backend/test/rollup.e2e-spec.ts
@@ -658,6 +658,74 @@ describe('OpenAPI Rollup field (e2e)', () => {
     }
   });
 
+  it('flattens and deduplicates multiple select values with array_unique when rolling up', async () => {
+    const projects = await createTable(baseId, {
+      name: 'rollup_multiselect_projects_unique',
+      fields: [
+        { name: 'Name', type: FieldType.SingleLineText } as IFieldRo,
+        {
+          name: 'Tags',
+          type: FieldType.MultipleSelect,
+          options: {
+            choices: [
+              { name: 'A', color: Colors.Yellow },
+              { name: 'B', color: Colors.Orange },
+              { name: 'C', color: Colors.Green },
+            ],
+          },
+        } as IFieldRo,
+      ],
+      records: [
+        { fields: { Name: 'P1', Tags: ['A', 'B'] } },
+        { fields: { Name: 'P2', Tags: ['B'] } },
+        { fields: { Name: 'P3', Tags: ['B', 'C'] } },
+      ],
+    });
+
+    const departments = await createTable(baseId, {
+      name: 'rollup_multiselect_departments_unique',
+      fields: [{ name: 'Dept', type: FieldType.SingleLineText } as IFieldRo],
+      records: [{ fields: { Dept: 'Ops' } }],
+    });
+
+    try {
+      const projectLink = await createField(departments.id, {
+        name: 'Projects',
+        type: FieldType.Link,
+        options: {
+          relationship: Relationship.OneMany,
+          foreignTableId: projects.id,
+        },
+      } as IFieldRo);
+
+      await updateRecordField(departments.id, departments.records[0].id, projectLink.id, [
+        { id: projects.records[0].id },
+        { id: projects.records[1].id },
+        { id: projects.records[2].id },
+      ]);
+
+      const tagsField = getFieldByName(projects.fields, 'Tags');
+      const rollup = await createField(departments.id, {
+        name: 'project_tags_unique',
+        type: FieldType.Rollup,
+        options: {
+          expression: 'array_unique({values})',
+        },
+        lookupOptions: {
+          foreignTableId: projects.id,
+          linkFieldId: projectLink.id,
+          lookupFieldId: tagsField.id,
+        },
+      } as IFieldRo);
+
+      const record = await getRecord(departments.id, departments.records[0].id);
+      expect(record.fields[rollup.id]).toEqual(['A', 'B', 'C']);
+    } finally {
+      await permanentDeleteTable(baseId, departments.id);
+      await permanentDeleteTable(baseId, projects.id);
+    }
+  });
+
   describe('rollup expression coverage', () => {
     const baseId = globalThis.testConfig.baseId;
     const isForceV2 = process.env.FORCE_V2_ALL === 'true';
diff --git a/packages/v2/adapter-table-repository-postgres/src/record/query-builder/computed/ComputedTableRecordQueryBuilder.spec.ts b/packages/v2/adapter-table-repository-postgres/src/record/query-builder/computed/ComputedTableRecordQueryBuilder.spec.ts
@@ -1829,6 +1829,26 @@ describe('ComputedTableRecordQueryBuilder', () => {
                     ) as "col_rollup" from "bseaaaaaaaaaaaaaaaa"."tblffffffffffffffff" as "f" where "f"."__fk_fldssssssssssssssss" = "t"."__id") as "lat_fldkkkkkkkkkkkkkkkk_0" on true"
       `);
     });
+
+    test('rollup array_unique flattens multi-value field entries before deduplication', () => {
+      const db = createTestDb();
+      const { mainTable, foreignTable, foreignTableId } =
+        createMultiValueRollupTable('array_unique({values})');
+
+      const foreignTables = new Map([[foreignTableId.toString(), foreignTable]]);
+      const { sql } = compileQuery(
+        db,
+        new ComputedTableRecordQueryBuilder(db, { foreignTables, typeValidationStrategy }).from(
+          mainTable
+        )
+      );
+
+      expect(sql).toContain('jsonb_agg(to_jsonb(v.val))');
+      expect(sql).toContain('SELECT DISTINCT val');
+      expect(sql).toContain('jsonb_array_elements(COALESCE(jsonb_agg("f"."col_tags"');
+      expect(sql).toContain('FILTER (WHERE "f"."col_tags" IS NOT NULL)');
+    });
+
     test('returns NULL rollup when rollup foreign table mismatches link foreign table', () => {
       const db = createTestDb();
       const baseId = BaseId.create(BASE_ID)._unsafeUnwrap();
diff --git a/packages/v2/adapter-table-repository-postgres/src/record/query-builder/computed/ComputedTableRecordQueryBuilder.ts b/packages/v2/adapter-table-repository-postgres/src/record/query-builder/computed/ComputedTableRecordQueryBuilder.ts
@@ -1422,6 +1422,12 @@ export class ComputedTableRecordQueryBuilder implements ITableRecordQueryBuilder
                 : sql`jsonb_agg(to_jsonb(${colRef})) FILTER (WHERE ${colRef} IS NOT NULL)`;
               return ok(this.buildDistinctNestedJsonTextArrayExpr(baseAggregate));
             }
+            if (isMultipleValue) {
+              const baseAggregate = orderByExpr
+                ? sql`jsonb_agg(${colRef} ORDER BY ${orderByExpr}) FILTER (WHERE ${colRef} IS NOT NULL)`
+                : sql`jsonb_agg(${colRef}) FILTER (WHERE ${colRef} IS NOT NULL)`;
+              return ok(this.buildDistinctNestedJsonTextArrayExpr(baseAggregate));
+            }
             return ok(sql`json_agg(DISTINCT ${colRef})`);
           }
           case 'array_compact({values})': {
diff --git a/packages/v2/e2e/src/rollup-arrayunique-multiselect.e2e.spec.ts b/packages/v2/e2e/src/rollup-arrayunique-multiselect.e2e.spec.ts
@@ -0,0 +1,142 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import { beforeAll, describe, expect, test } from 'vitest';
+
+import { getSharedTestContext, type SharedTestContext } from './shared/globalTestContext';
+
+describe('v2 rollup ARRAYUNIQUE over multi-select values (e2e)', () => {
+  let ctx: SharedTestContext;
+  let fieldIdCounter = 0;
+
+  const createFieldId = () => {
+    const suffix = fieldIdCounter.toString(36).padStart(16, '0');
+    fieldIdCounter += 1;
+    return `fld${suffix}`;
+  };
+
+  beforeAll(async () => {
+    ctx = await getSharedTestContext();
+  });
+
+  test('flattens nested multi-select values before deduplicating rollup output', async () => {
+    const foreignPrimaryFieldId = createFieldId();
+    const foreignTagsFieldId = createFieldId();
+    const hostPrimaryFieldId = createFieldId();
+    const hostLinkFieldId = createFieldId();
+    const hostRollupFieldId = createFieldId();
+
+    let foreignTableId: string | undefined;
+    let hostTableId: string | undefined;
+
+    try {
+      const foreignTable = await ctx.createTable({
+        baseId: ctx.baseId,
+        name: 'Rollup ArrayUnique Foreign',
+        fields: [
+          {
+            type: 'singleLineText',
+            id: foreignPrimaryFieldId,
+            name: 'Name',
+            isPrimary: true,
+          },
+          {
+            type: 'multipleSelect',
+            id: foreignTagsFieldId,
+            name: 'Tags',
+            options: {
+              choices: [
+                { id: 'optA', name: 'A', color: 'blue' },
+                { id: 'optB', name: 'B', color: 'green' },
+                { id: 'optC', name: 'C', color: 'red' },
+              ],
+            },
+          },
+        ],
+      });
+      foreignTableId = foreignTable.id;
+
+      const foreignRecord1 = await ctx.createRecord(foreignTable.id, {
+        [foreignPrimaryFieldId]: 'P1',
+        [foreignTagsFieldId]: ['A', 'B'],
+      });
+      const foreignRecord2 = await ctx.createRecord(foreignTable.id, {
+        [foreignPrimaryFieldId]: 'P2',
+        [foreignTagsFieldId]: ['B'],
+      });
+      const foreignRecord3 = await ctx.createRecord(foreignTable.id, {
+        [foreignPrimaryFieldId]: 'P3',
+        [foreignTagsFieldId]: ['B', 'C'],
+      });
+
+      const hostTable = await ctx.createTable({
+        baseId: ctx.baseId,
+        name: 'Rollup ArrayUnique Host',
+        fields: [
+          {
+            type: 'singleLineText',
+            id: hostPrimaryFieldId,
+            name: 'Name',
+            isPrimary: true,
+          },
+        ],
+      });
+      hostTableId = hostTable.id;
+
+      await ctx.createField({
+        baseId: ctx.baseId,
+        tableId: hostTable.id,
+        field: {
+          type: 'link',
+          id: hostLinkFieldId,
+          name: 'Projects',
+          options: {
+            relationship: 'manyMany',
+            foreignTableId: foreignTable.id,
+            lookupFieldId: foreignPrimaryFieldId,
+            isOneWay: true,
+          },
+        },
+      });
+
+      await ctx.createField({
+        baseId: ctx.baseId,
+        tableId: hostTable.id,
+        field: {
+          type: 'rollup',
+          id: hostRollupFieldId,
+          name: 'Unique Tags',
+          options: {
+            expression: 'array_unique({values})',
+          },
+          config: {
+            linkFieldId: hostLinkFieldId,
+            foreignTableId: foreignTable.id,
+            lookupFieldId: foreignTagsFieldId,
+          },
+        },
+      });
+
+      const hostRecord = await ctx.createRecord(hostTable.id, {
+        [hostPrimaryFieldId]: 'Ops',
+        [hostLinkFieldId]: [
+          { id: foreignRecord1.id },
+          { id: foreignRecord2.id },
+          { id: foreignRecord3.id },
+        ],
+      });
+
+      await ctx.drainOutbox();
+
+      const hostRecords = await ctx.listRecords(hostTable.id);
+      const updatedHostRecord = hostRecords.find((record) => record.id === hostRecord.id);
+
+      expect(updatedHostRecord?.fields[hostRollupFieldId]).toEqual(['A', 'B', 'C']);
+    } finally {
+      if (hostTableId) {
+        await ctx.deleteTable(hostTableId).catch(() => undefined);
+      }
+      if (foreignTableId) {
+        await ctx.deleteTable(foreignTableId).catch(() => undefined);
+      }
+    }
+  });
+});

Original file line number	Diff line number	Diff line change
`@@ -1422,6 +1422,12 @@ export class ComputedTableRecordQueryBuilder implements ITableRecordQueryBuilder`
`1422`	`1422`	: sql`jsonb_agg(to_jsonb(${colRef})) FILTER (WHERE ${colRef} IS NOT NULL)`;
`1423`	`1423`	`return ok(this.buildDistinctNestedJsonTextArrayExpr(baseAggregate));`
`1424`	`1424`	`}`
	`1425`	`+ if (isMultipleValue) {`
	`1426`	`+ const baseAggregate = orderByExpr`
	`1427`	+ ? sql`jsonb_agg(${colRef} ORDER BY ${orderByExpr}) FILTER (WHERE ${colRef} IS NOT NULL)`
	`1428`	+ : sql`jsonb_agg(${colRef}) FILTER (WHERE ${colRef} IS NOT NULL)`;
	`1429`	`+ return ok(this.buildDistinctNestedJsonTextArrayExpr(baseAggregate));`
	`1430`	`+ }`
`1425`	`1431`	return ok(sql`json_agg(DISTINCT ${colRef})`);
`1426`	`1432`	`}`
`1427`	`1433`	`case 'array_compact({values})': {`